From 77f6be411d7098535add0c80064bf07587533a86 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:37:38 +0100 Subject: [PATCH 01/36] Created base proxmox os plugin --- dissect/target/plugin.py | 1 + .../os/unix/linux/debian/proxmox/__init__.py | 0 .../os/unix/linux/debian/proxmox/_os.py | 29 +++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 dissect/target/plugins/os/unix/linux/debian/proxmox/__init__.py create mode 100644 dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py diff --git a/dissect/target/plugin.py b/dissect/target/plugin.py index 5118a4df6..63db7313d 100644 --- a/dissect/target/plugin.py +++ b/dissect/target/plugin.py @@ -66,6 +66,7 @@ class OperatingSystem(StrEnum): IOS = "ios" FORTIOS = "fortios" CITRIX = "citrix-netscaler" + PROXMOX = "proxmox" def export(*args, **kwargs) -> Callable: diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/__init__.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py new file mode 100644 index 000000000..8ebd3c90d --- /dev/null +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -0,0 +1,29 @@ +import logging +from typing import Optional + +from dissect.target.filesystem import Filesystem +from dissect.target.plugins.os.unix._os import OperatingSystem, export +from dissect.target.plugins.os.unix.linux._os import LinuxPlugin +from dissect.target.helpers.record import TargetRecordDescriptor +from dissect.target.target import Target + +log = logging.getLogger(__name__) + +PROXMOX_PACKAGE_NAME="proxmox-ve" + + +class ProxmoxPlugin(LinuxPlugin): + def __init__(self, target: Target): + super().__init__(target) + + @classmethod + def detect(cls, target: Target) -> Optional[Filesystem]: + for fs in target.filesystems: + if (fs.exists("/etc/pve") or fs.exists("/var/lib/pve")): + return fs + return None + + @export(property=True) + def os(self) -> str: + return OperatingSystem.PROXMOX.value + From 3c45dffd97c5a2433753d80ea8f02d3b56cfd9ae Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:41:15 +0100 Subject: [PATCH 02/36] Implemented proxmox version retrieval --- .../target/plugins/os/unix/linux/debian/proxmox/_os.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 8ebd3c90d..996ff61fe 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -23,6 +23,15 @@ def detect(cls, target: Target) -> Optional[Filesystem]: return fs return None + @export(property=True) + def version(self) -> str: + """Returns Proxmox VE version with underlying os release""" + + for pkg in self.target.dpkg.status(): + if pkg.name == PROXMOX_PACKAGE_NAME: + distro_name = self._os_release.get("PRETTY_NAME", "") + return f"{pkg.name} {pkg.version} ({distro_name})" + @export(property=True) def os(self) -> str: return OperatingSystem.PROXMOX.value From 76cb75331262583d7db0bc664df52eccc9b608d5 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:24:30 +0100 Subject: [PATCH 03/36] Implemented ability to parse, setup and mount pmxcfs from database. --- .../os/unix/linux/debian/proxmox/_os.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 996ff61fe..890575bb8 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -1,7 +1,12 @@ +from __future__ import annotations + import logging +from io import BytesIO from typing import Optional -from dissect.target.filesystem import Filesystem +from dissect.sql import sqlite3 + +from dissect.target.filesystem import Filesystem, VirtualFilesystem from dissect.target.plugins.os.unix._os import OperatingSystem, export from dissect.target.plugins.os.unix.linux._os import LinuxPlugin from dissect.target.helpers.record import TargetRecordDescriptor @@ -10,6 +15,8 @@ log = logging.getLogger(__name__) PROXMOX_PACKAGE_NAME="proxmox-ve" +FILETREE_TABLE_NAME="tree" +PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" class ProxmoxPlugin(LinuxPlugin): @@ -23,6 +30,17 @@ def detect(cls, target: Target) -> Optional[Filesystem]: return fs return None + @classmethod + def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: + obj = super().create(target, sysvol) + # [PERSONAL TO REMOVE] Modifies target / executescode before initializing the class + pmxcfs = _create_pmxcfs(sysvol.path(PMXCFS_DATABASE_PATH).open("rb")) + target.fs.mount("/etc/pve", pmxcfs) + + ipdb.set_trace() + + return obj + @export(property=True) def version(self) -> str: """Returns Proxmox VE version with underlying os release""" @@ -36,3 +54,28 @@ def version(self) -> str: def os(self) -> str: return OperatingSystem.PROXMOX.value +def _create_pmxcfs(fh): + db = sqlite3.SQLite3(fh) + filetree_table = db.table(FILETREE_TABLE_NAME) + # columns = filetree_table.columns # For implementing fs with propper stat data later + rows = filetree_table.rows() + + fs_entries = [] + for row in rows: + fs_entries.append(row) + fs_entries.sort(key=lambda entry: (entry.parent, entry.inode), reverse=True) + + vfs = VirtualFilesystem() + for entry in fs_entries: # might add dir mapping if deemed necessary + if entry.type == 8: # Type 8 file | Type 4 dir + path = entry.name + parent = entry.parent + content = entry.data + + for file in fs_entries: + if file.inode == parent and file.inode != 0: + path = f"{file.name}/{path}" + else: + vfs.map_file_fh(f"/{path}", BytesIO(content or b"")) + + return vfs \ No newline at end of file From 84df21ee921c51b11eac0b2a774113d68b2be649 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:04:43 +0100 Subject: [PATCH 04/36] Added helper functions of vm listing --- .../os/unix/linux/debian/proxmox/_os.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 890575bb8..3450675c5 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import logging from io import BytesIO from typing import Optional @@ -78,4 +79,20 @@ def _create_pmxcfs(fh): else: vfs.map_file_fh(f"/{path}", BytesIO(content or b"")) - return vfs \ No newline at end of file + return vfs + +def _parse_vm_configuration(conf) -> list: + file = conf.open() + parsed_lines = {} + for line in file: + key, value = line.split(b': ') + parsed_lines[key] = value.replace(b'\n', b'') + return parsed_lines + +def _is_disk(config_value: str) -> str | None: + disk = re.match(r"^(sata|scsi|ide)[0-9]+$", config_value) + return True if disk else None + +def _get_disk_name(config_value: str) -> str | None: + disk = re.search(r"vm-[0-9]+-disk-[0-9]+", config_value) + return disk if disk else None \ No newline at end of file From ca59cebfdd2059c5886165032a16c56c7bfeb9d8 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:06:26 +0100 Subject: [PATCH 05/36] W.I.P. VM listing function --- .../os/unix/linux/debian/proxmox/_os.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 3450675c5..93e1311c6 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import pathlib import logging from io import BytesIO from typing import Optional @@ -20,6 +21,16 @@ PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" +VirtualMachineRecord = TargetRecordDescriptor( + "proxmox/vm", + [ + ("string", "id"), + ("string", "name"), + ("path", "disk"), + ], +) + + class ProxmoxPlugin(LinuxPlugin): def __init__(self, target: Target): super().__init__(target) @@ -35,6 +46,7 @@ def detect(cls, target: Target) -> Optional[Filesystem]: def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: obj = super().create(target, sysvol) # [PERSONAL TO REMOVE] Modifies target / executescode before initializing the class + obj = super().create(target, sysvol) pmxcfs = _create_pmxcfs(sysvol.path(PMXCFS_DATABASE_PATH).open("rb")) target.fs.mount("/etc/pve", pmxcfs) @@ -51,6 +63,25 @@ def version(self) -> str: distro_name = self._os_release.get("PRETTY_NAME", "") return f"{pkg.name} {pkg.version} ({distro_name})" + @export(property=VirtualMachineRecord) + def vms(self) -> Iterator[VirtualMachineRecord]: + configs = self.target.fs.path("/etc/pve/qemu-server") + for config in configs.iterdir(): + + parsed_config = _parse_vm_configuration(config) + for option in parsed_config: + + if _is_disk(option.decode()): + id_num = pathlib.Path(config).stem + # TypeError: expected str, bytes or os.PathLike object, not Match + yield VirtualMachineRecord( + id=id_num, + name=parsed_config[b'name'], + disk=_get_disk_name(parsed_config[option].decode()) + ) + + return None + @export(property=True) def os(self) -> str: return OperatingSystem.PROXMOX.value From 350b5e076e9286698ea4ce80bdbafc3b21854c5c Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:52:33 +0100 Subject: [PATCH 06/36] Refractoring of helper functions and completing vm listing function --- .../os/unix/linux/debian/proxmox/_os.py | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 93e1311c6..ae85b1f1f 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -26,7 +26,8 @@ [ ("string", "id"), ("string", "name"), - ("path", "disk"), + ("string", "storage_id"), + ("string", "disk"), ], ) @@ -54,39 +55,38 @@ def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: return obj + @export(property=True) + def os(self) -> str: + return OperatingSystem.PROXMOX.value + @export(property=True) def version(self) -> str: - """Returns Proxmox VE version with underlying os release""" + """Returns Proxmox VE version with underlying os release""" - for pkg in self.target.dpkg.status(): - if pkg.name == PROXMOX_PACKAGE_NAME: - distro_name = self._os_release.get("PRETTY_NAME", "") - return f"{pkg.name} {pkg.version} ({distro_name})" + for pkg in self.target.dpkg.status(): + if pkg.name == PROXMOX_PACKAGE_NAME: + distro_name = self._os_release.get("PRETTY_NAME", "") + return f"{pkg.name} {pkg.version} ({distro_name})" - @export(property=VirtualMachineRecord) + @export(record=VirtualMachineRecord) def vms(self) -> Iterator[VirtualMachineRecord]: + # ipdb.set_trace() + # Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs configs = self.target.fs.path("/etc/pve/qemu-server") for config in configs.iterdir(): - parsed_config = _parse_vm_configuration(config) for option in parsed_config: - - if _is_disk(option.decode()): - id_num = pathlib.Path(config).stem - # TypeError: expected str, bytes or os.PathLike object, not Match + if _is_disk_device(option.decode()): + vm_id = pathlib.Path(config).stem + config_value = parsed_config[option].decode() yield VirtualMachineRecord( - id=id_num, - name=parsed_config[b'name'], - disk=_get_disk_name(parsed_config[option].decode()) + id=vm_id, + name=parsed_config[b'name'].decode(), + storage_id=_get_storage_ID(config_value), + disk=_get_disk_name(config_value) ) - return None - - @export(property=True) - def os(self) -> str: - return OperatingSystem.PROXMOX.value - -def _create_pmxcfs(fh): +def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) filetree_table = db.table(FILETREE_TABLE_NAME) # columns = filetree_table.columns # For implementing fs with propper stat data later @@ -120,10 +120,14 @@ def _parse_vm_configuration(conf) -> list: parsed_lines[key] = value.replace(b'\n', b'') return parsed_lines -def _is_disk(config_value: str) -> str | None: +def _is_disk_device(config_value: str) -> str | None: disk = re.match(r"^(sata|scsi|ide)[0-9]+$", config_value) return True if disk else None +def _get_storage_ID(config_value: str) -> str | None: + storage_id = config_value.split(":") + return storage_id[0] if storage_id else None + def _get_disk_name(config_value: str) -> str | None: disk = re.search(r"vm-[0-9]+-disk-[0-9]+", config_value) - return disk if disk else None \ No newline at end of file + return disk.group(0) if disk else None \ No newline at end of file From 264e58857ccbebf0d1e0141e58940182498ea587 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:22:02 +0100 Subject: [PATCH 07/36] Made child plugin for loading vm --- dissect/target/plugins/child/proxmox.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 dissect/target/plugins/child/proxmox.py diff --git a/dissect/target/plugins/child/proxmox.py b/dissect/target/plugins/child/proxmox.py new file mode 100644 index 000000000..3f14d2aea --- /dev/null +++ b/dissect/target/plugins/child/proxmox.py @@ -0,0 +1,21 @@ +from dissect.target.exceptions import UnsupportedPluginError +from dissect.target.helpers.record import ChildTargetRecord +from dissect.target.plugin import ChildTargetPlugin + + +class ProxmoxChildTargetPlugin(ChildTargetPlugin): + """Child target plugin that yields from the VM listing.""" + + __type__ = "proxmox" + + def check_compatible(self) -> None: + if self.target.os != "proxmox": + raise UnsupportedPluginError("Not an promox operating system") + + def list_children(self): + for vm in self.target.vm_list(): + yield ChildTargetRecord( + type=self.__type__, + path=vm.path, + _target=self.target, + ) From 40bdb322c0f05fdeb1c98bd7c1fd5be5b9c0e635 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:23:11 +0100 Subject: [PATCH 08/36] Modified proxmox plugin to facilitate child loading --- .../plugins/os/unix/linux/debian/proxmox/_os.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index ae85b1f1f..60d964efe 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -19,6 +19,8 @@ PROXMOX_PACKAGE_NAME="proxmox-ve" FILETREE_TABLE_NAME="tree" PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" +# Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs +VM_CONFIG_PATH="/etc/pve/qemu-server" VirtualMachineRecord = TargetRecordDescriptor( @@ -28,6 +30,7 @@ ("string", "name"), ("string", "storage_id"), ("string", "disk"), + ("path", "path"), ], ) @@ -69,10 +72,8 @@ def version(self) -> str: return f"{pkg.name} {pkg.version} ({distro_name})" @export(record=VirtualMachineRecord) - def vms(self) -> Iterator[VirtualMachineRecord]: - # ipdb.set_trace() - # Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs - configs = self.target.fs.path("/etc/pve/qemu-server") + def vm_list(self) -> Iterator[VirtualMachineRecord]: + configs = self.target.fs.path(VM_CONFIG_PATH) for config in configs.iterdir(): parsed_config = _parse_vm_configuration(config) for option in parsed_config: @@ -83,7 +84,8 @@ def vms(self) -> Iterator[VirtualMachineRecord]: id=vm_id, name=parsed_config[b'name'].decode(), storage_id=_get_storage_ID(config_value), - disk=_get_disk_name(config_value) + disk=_get_disk_name(config_value), + path=VM_CONFIG_PATH + f"/{vm_id}.conf", ) def _create_pmxcfs(fh) -> VirtualFilesystem: From b36b1b9edc53f5184a6daa3650bb28d387aa26bb Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:50:32 +0200 Subject: [PATCH 09/36] Created function that adds lvm devices to target fs --- dissect/target/plugins/os/unix/_os.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dissect/target/plugins/os/unix/_os.py b/dissect/target/plugins/os/unix/_os.py index 81710ecc5..b89d70c95 100644 --- a/dissect/target/plugins/os/unix/_os.py +++ b/dissect/target/plugins/os/unix/_os.py @@ -20,6 +20,7 @@ class UnixPlugin(OSPlugin): def __init__(self, target: Target): super().__init__(target) self._add_mounts() + self._add_lvm_devices() self._hostname_dict = self._parse_hostname_string() self._hosts_dict = self._parse_hosts_string() self._os_release = self._parse_os_release() @@ -230,6 +231,19 @@ def _add_mounts(self) -> None: self.target.log.debug("Mounting %s (%s) at %s", fs, fs.volume, mount_point) self.target.fs.mount(mount_point, fs) + def _add_lvm_devices(self) -> None: + """Parses and mounts lvm devices from external target to local target fs""" + vfs = VirtualFilesystem() + for volume in self.target.volumes.entries: + if isinstance(volume.vs, lvm.LvmVolumeSystem) and "disk" in volume.name: + vg_name = volume.vs.lvm.vg.name + + for logical_volume in volume.vs.lvm.vg.lv: + lv_name = logical_volume.name + vfs.map_file_fh(f"/{vg_name}/{lv_name}", BufferedStream(volume)) + + self.target.fs.mount("/dev", vfs) + def _parse_os_release(self, glob: Optional[str] = None) -> dict[str, str]: """Parse files containing Unix version information. From b58fa368a29b0fad9e75cbe92ef1e0bf1708612d Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:43:31 +0200 Subject: [PATCH 10/36] Added missing deps --- dissect/target/plugins/os/unix/_os.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dissect/target/plugins/os/unix/_os.py b/dissect/target/plugins/os/unix/_os.py index b89d70c95..9f30222dc 100644 --- a/dissect/target/plugins/os/unix/_os.py +++ b/dissect/target/plugins/os/unix/_os.py @@ -6,13 +6,17 @@ from struct import unpack from typing import Iterator, Optional, Union -from dissect.target.filesystem import Filesystem +from dissect.util.stream import BufferedStream + +from dissect.target.filesystem import Filesystem, VirtualFilesystem from dissect.target.helpers.fsutil import TargetPath from dissect.target.helpers.record import UnixUserRecord from dissect.target.helpers.utils import parse_options_string from dissect.target.plugin import OperatingSystem, OSPlugin, arg, export from dissect.target.target import Target +from dissect.target.volumes import lvm + log = logging.getLogger(__name__) From 7755b183f76a9369c6e0ae2692afa11e37265cf2 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:46:41 +0200 Subject: [PATCH 11/36] Refractored logic for proxmox loader --- dissect/target/loader.py | 1 + dissect/target/plugins/child/proxmox.py | 2 +- .../os/unix/linux/debian/proxmox/_os.py | 32 ++++--------------- 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/dissect/target/loader.py b/dissect/target/loader.py index de1c04df0..b64ccd081 100644 --- a/dissect/target/loader.py +++ b/dissect/target/loader.py @@ -203,4 +203,5 @@ def open(item: Union[str, Path], *args, **kwargs) -> Loader: register("smb", "SmbLoader") register("cb", "CbLoader") register("cyber", "CyberLoader") +register("proxmox", "ProxmoxLoader") register("multiraw", "MultiRawLoader") # Should be last diff --git a/dissect/target/plugins/child/proxmox.py b/dissect/target/plugins/child/proxmox.py index 3f14d2aea..17611c2b1 100644 --- a/dissect/target/plugins/child/proxmox.py +++ b/dissect/target/plugins/child/proxmox.py @@ -16,6 +16,6 @@ def list_children(self): for vm in self.target.vm_list(): yield ChildTargetRecord( type=self.__type__, - path=vm.path, + path=vm.config_path, _target=self.target, ) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 60d964efe..80154b797 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -27,10 +27,7 @@ "proxmox/vm", [ ("string", "id"), - ("string", "name"), - ("string", "storage_id"), - ("string", "disk"), - ("path", "path"), + ("string", "config_path"), ], ) @@ -48,14 +45,11 @@ def detect(cls, target: Target) -> Optional[Filesystem]: @classmethod def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: - obj = super().create(target, sysvol) # [PERSONAL TO REMOVE] Modifies target / executescode before initializing the class obj = super().create(target, sysvol) pmxcfs = _create_pmxcfs(sysvol.path(PMXCFS_DATABASE_PATH).open("rb")) target.fs.mount("/etc/pve", pmxcfs) - ipdb.set_trace() - return obj @export(property=True) @@ -75,18 +69,10 @@ def version(self) -> str: def vm_list(self) -> Iterator[VirtualMachineRecord]: configs = self.target.fs.path(VM_CONFIG_PATH) for config in configs.iterdir(): - parsed_config = _parse_vm_configuration(config) - for option in parsed_config: - if _is_disk_device(option.decode()): - vm_id = pathlib.Path(config).stem - config_value = parsed_config[option].decode() - yield VirtualMachineRecord( - id=vm_id, - name=parsed_config[b'name'].decode(), - storage_id=_get_storage_ID(config_value), - disk=_get_disk_name(config_value), - path=VM_CONFIG_PATH + f"/{vm_id}.conf", - ) + yield VirtualMachineRecord( + id=pathlib.Path(config).stem, + config_path=config, + ) def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) @@ -114,13 +100,7 @@ def _create_pmxcfs(fh) -> VirtualFilesystem: return vfs -def _parse_vm_configuration(conf) -> list: - file = conf.open() - parsed_lines = {} - for line in file: - key, value = line.split(b': ') - parsed_lines[key] = value.replace(b'\n', b'') - return parsed_lines + def _is_disk_device(config_value: str) -> str | None: disk = re.match(r"^(sata|scsi|ide)[0-9]+$", config_value) From c7a613d23eb17244d03f2942ecf300b028570d2a Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:50:04 +0200 Subject: [PATCH 12/36] Created proxmox loader (w.i.p.) --- dissect/target/loaders/proxmox.py | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 dissect/target/loaders/proxmox.py diff --git a/dissect/target/loaders/proxmox.py b/dissect/target/loaders/proxmox.py new file mode 100644 index 000000000..558d4f727 --- /dev/null +++ b/dissect/target/loaders/proxmox.py @@ -0,0 +1,58 @@ +from __future__ import annotations +from pathlib import Path +import re + +from dissect.target.loader import Loader + + + +class ProxmoxLoader(Loader): + """Load Proxmox disk data onto target disks. + + The method proxmox uses to store disk data varies on multiple factors such as + filesystem used, available storage space and other factors. This information is + stored within a config file in the filesystem. + + This loader attains the necessary information to find the disk data on the + filesystem and appends it to the target filesystem's disks. + """ + + def __init__(self, path, **kwargs): + path = path.resolve() + super().__init__(path) + + @staticmethod + def detect(path) -> bool: + return path.suffix.lower() == ".conf" + + def map(self, target): + parsed_config = self._parse_vm_configuration(self.path) + + for option in parsed_config: + config_value = parsed_config[option] + vm_disk = _get_vm_disk_name(config_value) + + if _is_disk_device(option) and vm_disk is not None: + vm_id = self.path.stem + name = parsed_config['name'] + + + def _parse_vm_configuration(self, conf) -> list: + lines = conf.read_text().split("\n") + lines.remove("") # Removes any trailing empty lines in file + parsed_lines = {} + + for line in lines: + key, value = line.split(': ') + parsed_lines[key] = value + + return parsed_lines + +def _is_disk_device(config_value: str) -> bool | None: + disk = re.match(r"^(sata|scsi|ide)[0-9]+$", config_value) + return True if disk else None + +def _get_vm_disk_name(config_value: str) -> str | None: + """Retrieves the disk device name from vm""" + disk = re.search(r"vm-[0-9]+-disk-[0-9]+(,|.+?,)", config_value) + return disk.group(0).replace(",", "") if disk else None \ No newline at end of file From c75ede546d5214cea13048bcce3537fe305edf48 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 1 May 2024 14:44:00 +0200 Subject: [PATCH 13/36] Added disk mapping functionality in loader --- dissect/target/loaders/proxmox.py | 14 +++++++- .../os/unix/linux/debian/proxmox/_os.py | 34 +++++++++++-------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/dissect/target/loaders/proxmox.py b/dissect/target/loaders/proxmox.py index 558d4f727..214b8785d 100644 --- a/dissect/target/loaders/proxmox.py +++ b/dissect/target/loaders/proxmox.py @@ -2,6 +2,7 @@ from pathlib import Path import re +from dissect.target.containers.raw import RawContainer from dissect.target.loader import Loader @@ -33,9 +34,16 @@ def map(self, target): vm_disk = _get_vm_disk_name(config_value) if _is_disk_device(option) and vm_disk is not None: + disk_interface = option vm_id = self.path.stem name = parsed_config['name'] + storage_id = _get_storage_ID(config_value) + path = self.path.joinpath("/dev/pve/", vm_disk) + try: + target.disks.add(RawContainer(path.open("rb"))) + except Exception: + target.log.exception("Failed to load block device: %s", vm_disk) def _parse_vm_configuration(self, conf) -> list: lines = conf.read_text().split("\n") @@ -55,4 +63,8 @@ def _is_disk_device(config_value: str) -> bool | None: def _get_vm_disk_name(config_value: str) -> str | None: """Retrieves the disk device name from vm""" disk = re.search(r"vm-[0-9]+-disk-[0-9]+(,|.+?,)", config_value) - return disk.group(0).replace(",", "") if disk else None \ No newline at end of file + return disk.group(0).replace(",", "") if disk else None + +def _get_storage_ID(config_value: str) -> str | None: + storage_id = config_value.split(":") + return storage_id[0] if storage_id else None \ No newline at end of file diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 80154b797..c232f9625 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re import pathlib import logging @@ -14,6 +15,8 @@ from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.target import Target +import ipdb + log = logging.getLogger(__name__) PROXMOX_PACKAGE_NAME="proxmox-ve" @@ -39,6 +42,7 @@ def __init__(self, target: Target): @classmethod def detect(cls, target: Target) -> Optional[Filesystem]: for fs in target.filesystems: + # [REMINDER FOR REPORT] both filesystems are checked for redundancy (in case fs is damaged/missing stuff) if (fs.exists("/etc/pve") or fs.exists("/var/lib/pve")): return fs return None @@ -49,6 +53,7 @@ def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: obj = super().create(target, sysvol) pmxcfs = _create_pmxcfs(sysvol.path(PMXCFS_DATABASE_PATH).open("rb")) target.fs.mount("/etc/pve", pmxcfs) + # ipdb.set_trace() return obj @@ -67,12 +72,27 @@ def version(self) -> str: @export(record=VirtualMachineRecord) def vm_list(self) -> Iterator[VirtualMachineRecord]: + import ipdb configs = self.target.fs.path(VM_CONFIG_PATH) for config in configs.iterdir(): yield VirtualMachineRecord( id=pathlib.Path(config).stem, config_path=config, ) + # ipdb.set_trace() + # parsed_config = _parse_vm_configuration(config) + # for option in parsed_config: + # config_value = parsed_config[option].decode() + # vm_disk = _get_vm_disk_name(config_value) + # if _is_disk_device(option.decode()) and vm_disk is not None: + # vm_id = pathlib.Path(config).stem + # yield VirtualMachineRecord( + # id=vm_id, + # name=parsed_config[b'name'].decode(), + # storage_id=_get_storage_ID(config_value), + # disk=vm_disk, # TODO: Maybe remove + # config_path=config, + # ) def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) @@ -99,17 +119,3 @@ def _create_pmxcfs(fh) -> VirtualFilesystem: vfs.map_file_fh(f"/{path}", BytesIO(content or b"")) return vfs - - - -def _is_disk_device(config_value: str) -> str | None: - disk = re.match(r"^(sata|scsi|ide)[0-9]+$", config_value) - return True if disk else None - -def _get_storage_ID(config_value: str) -> str | None: - storage_id = config_value.split(":") - return storage_id[0] if storage_id else None - -def _get_disk_name(config_value: str) -> str | None: - disk = re.search(r"vm-[0-9]+-disk-[0-9]+", config_value) - return disk.group(0) if disk else None \ No newline at end of file From d2ccd26568790ab2522d5061ae5e7c18955258ae Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 1 May 2024 14:44:48 +0200 Subject: [PATCH 14/36] Fixed Bug causing lvm volumes to be added twice per taget volume --- dissect/target/plugins/os/unix/_os.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dissect/target/plugins/os/unix/_os.py b/dissect/target/plugins/os/unix/_os.py index 9f30222dc..a6935aa17 100644 --- a/dissect/target/plugins/os/unix/_os.py +++ b/dissect/target/plugins/os/unix/_os.py @@ -238,13 +238,9 @@ def _add_mounts(self) -> None: def _add_lvm_devices(self) -> None: """Parses and mounts lvm devices from external target to local target fs""" vfs = VirtualFilesystem() - for volume in self.target.volumes.entries: + for volume in self.target.volumes: if isinstance(volume.vs, lvm.LvmVolumeSystem) and "disk" in volume.name: - vg_name = volume.vs.lvm.vg.name - - for logical_volume in volume.vs.lvm.vg.lv: - lv_name = logical_volume.name - vfs.map_file_fh(f"/{vg_name}/{lv_name}", BufferedStream(volume)) + vfs.map_file_fh(f"{volume.raw.vg.name}/{volume.raw.name}", BufferedStream(volume)) self.target.fs.mount("/dev", vfs) From f4a676663ab8b5792d1713e5d8eb80235c2cd863 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 15 May 2024 16:48:11 +0200 Subject: [PATCH 15/36] Re-implemented Breadth-first search logic into pmxcfs creation --- .../os/unix/linux/debian/proxmox/_os.py | 69 ++++++++++--------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index c232f9625..88cf09b58 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -22,7 +22,7 @@ PROXMOX_PACKAGE_NAME="proxmox-ve" FILETREE_TABLE_NAME="tree" PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" -# Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs +# TODO: Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs VM_CONFIG_PATH="/etc/pve/qemu-server" @@ -42,18 +42,15 @@ def __init__(self, target: Target): @classmethod def detect(cls, target: Target) -> Optional[Filesystem]: for fs in target.filesystems: - # [REMINDER FOR REPORT] both filesystems are checked for redundancy (in case fs is damaged/missing stuff) if (fs.exists("/etc/pve") or fs.exists("/var/lib/pve")): return fs return None @classmethod def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: - # [PERSONAL TO REMOVE] Modifies target / executescode before initializing the class obj = super().create(target, sysvol) pmxcfs = _create_pmxcfs(sysvol.path(PMXCFS_DATABASE_PATH).open("rb")) target.fs.mount("/etc/pve", pmxcfs) - # ipdb.set_trace() return obj @@ -72,50 +69,56 @@ def version(self) -> str: @export(record=VirtualMachineRecord) def vm_list(self) -> Iterator[VirtualMachineRecord]: - import ipdb configs = self.target.fs.path(VM_CONFIG_PATH) for config in configs.iterdir(): yield VirtualMachineRecord( id=pathlib.Path(config).stem, config_path=config, ) - # ipdb.set_trace() - # parsed_config = _parse_vm_configuration(config) - # for option in parsed_config: - # config_value = parsed_config[option].decode() - # vm_disk = _get_vm_disk_name(config_value) - # if _is_disk_device(option.decode()) and vm_disk is not None: - # vm_id = pathlib.Path(config).stem - # yield VirtualMachineRecord( - # id=vm_id, - # name=parsed_config[b'name'].decode(), - # storage_id=_get_storage_ID(config_value), - # disk=vm_disk, # TODO: Maybe remove - # config_path=config, - # ) def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) filetree_table = db.table(FILETREE_TABLE_NAME) - # columns = filetree_table.columns # For implementing fs with propper stat data later rows = filetree_table.rows() - fs_entries = [] for row in rows: fs_entries.append(row) - fs_entries.sort(key=lambda entry: (entry.parent, entry.inode), reverse=True) + fs_entries.sort(key=lambda entry: (entry.parent, entry.inode)) vfs = VirtualFilesystem() - for entry in fs_entries: # might add dir mapping if deemed necessary - if entry.type == 8: # Type 8 file | Type 4 dir - path = entry.name - parent = entry.parent - content = entry.data - - for file in fs_entries: - if file.inode == parent and file.inode != 0: - path = f"{file.name}/{path}" - else: - vfs.map_file_fh(f"/{path}", BytesIO(content or b"")) + for entry in fs_entries: + content = entry.data + if entry.parent == 0: # Root entries do not require parent check + vfs.map_file_fh(f"/{entry.name}", BytesIO(content or b"")) + continue + else: + entry_chain = _get_entry_parent_chain(fs_entries, entry) + path = _create_fs_path(entry_chain) + vfs.map_file_fh(f"/{path}", BytesIO(content or b"")) return vfs + +def _get_entry_parent_chain(fs_entries: list, entry: sqlite3[Row]) -> list[sqlite3[Row]]: + """Looks through the list of inodes (fs_entries) and retrieves parent inode of each node until root is found.""" + + inode_chain = [entry] + target = entry + for entry in fs_entries: + if target.inode != 0: + inode_chain.append(entry) + target = entry + else: + return inode_chain + +def _create_fs_path(entry_chain: list[sqlite3[Row]]) -> str: + """Creates a full path out of a sorted list of file entries""" + + entry_chain.sort(key=lambda entry: (entry.parent)) + entry_names = [] + for entry in entry_chain: + if entry.inode != 0: + entry_names.append(entry.name) + + path = "/".join(entry_names) + + return path From 32bf4e3a91b02fbb910ff6d9b7af99a64d922562 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 15 May 2024 17:01:30 +0200 Subject: [PATCH 16/36] Temporary changes to test proof-of-concept --- .../plugins/os/unix/linux/debian/proxmox/_os.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 88cf09b58..7f50ea671 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -22,8 +22,8 @@ PROXMOX_PACKAGE_NAME="proxmox-ve" FILETREE_TABLE_NAME="tree" PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" -# TODO: Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs -VM_CONFIG_PATH="/etc/pve/qemu-server" +# VM_CONFIG_PATH="/etc/pve/qemu-server" # TODO: Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs +VM_CONFIG_PATH="/etc/pve/" # TODO: properly implement pmxcfs creation and revert to propper path VirtualMachineRecord = TargetRecordDescriptor( @@ -71,10 +71,11 @@ def version(self) -> str: def vm_list(self) -> Iterator[VirtualMachineRecord]: configs = self.target.fs.path(VM_CONFIG_PATH) for config in configs.iterdir(): - yield VirtualMachineRecord( - id=pathlib.Path(config).stem, - config_path=config, - ) + if pathlib.Path(config).suffix == ".conf": # TODO: Remove if statement once pmxcfs is propperly implemented + yield VirtualMachineRecord( + id=pathlib.Path(config).stem, + config_path=config, + ) def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) @@ -94,7 +95,7 @@ def _create_pmxcfs(fh) -> VirtualFilesystem: else: entry_chain = _get_entry_parent_chain(fs_entries, entry) path = _create_fs_path(entry_chain) - vfs.map_file_fh(f"/{path}", BytesIO(content or b"")) + vfs.map_file_fh(f"{path}", BytesIO(content or b"")) # TODO: revert root (/) on string once pmxcfs has been propperly implemented return vfs From 3c6b10e65e336f332fc0c37cf5a89c9d4f6557f3 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:08:29 +0200 Subject: [PATCH 17/36] Implemented file & directory fs mounting with metadata --- .../os/unix/linux/debian/proxmox/_os.py | 92 ++++++++++++++++--- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 7f50ea671..a4e79a2f5 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -2,6 +2,7 @@ import os import re +import stat import pathlib import logging from io import BytesIO @@ -9,7 +10,8 @@ from dissect.sql import sqlite3 -from dissect.target.filesystem import Filesystem, VirtualFilesystem +from dissect.target.filesystem import Filesystem, VirtualFilesystem, VirtualFile, VirtualDirectory +from dissect.target.helpers import fsutil from dissect.target.plugins.os.unix._os import OperatingSystem, export from dissect.target.plugins.os.unix.linux._os import LinuxPlugin from dissect.target.helpers.record import TargetRecordDescriptor @@ -81,21 +83,32 @@ def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) filetree_table = db.table(FILETREE_TABLE_NAME) rows = filetree_table.rows() - fs_entries = [] + fs_entries = {} for row in rows: - fs_entries.append(row) - fs_entries.sort(key=lambda entry: (entry.parent, entry.inode)) + fs_entries[row.inode] = row + # fs_entries.sort(key=lambda entry: (entry.parent, entry.inode)) vfs = VirtualFilesystem() - for entry in fs_entries: - content = entry.data + for entry in fs_entries.values(): if entry.parent == 0: # Root entries do not require parent check - vfs.map_file_fh(f"/{entry.name}", BytesIO(content or b"")) - continue + path = entry.name + else: + # import ipdb; ipdb.set_trace() + parts = [] + current = entry + while current.parent != 0: + parts.append(current.name) + current = fs_entries[current.parent] + + path = "/".join(parts[::-1]) + if entry.type == 4: + fsentry = ProxmoxConfigDirectoryEntry(vfs, path, entry) + elif entry.type == 8: + fsentry = ProxmoxConfigFileEntry(vfs, path, entry) else: - entry_chain = _get_entry_parent_chain(fs_entries, entry) - path = _create_fs_path(entry_chain) - vfs.map_file_fh(f"{path}", BytesIO(content or b"")) # TODO: revert root (/) on string once pmxcfs has been propperly implemented + raise ValueError(f"Unknown pmxcfs file type: {entry.type}") + + vfs.map_file_entry(path, fsentry) return vfs @@ -123,3 +136,60 @@ def _create_fs_path(entry_chain: list[sqlite3[Row]]) -> str: path = "/".join(entry_names) return path + + +class ProxmoxConfigFileEntry(VirtualFile): + def open(self) -> BinaryIO: + """Returns file handle (file-like object).""" + # if self.entry is not a directory, but a file + return BytesIO(self.entry.data or b"") + + def stat(self, follow_symlinks: bool = True) -> fsutil.stat_result: + """Return the stat information of this entry.""" + return self.lstat() + + def lstat(self) -> fsutil.stat_result: + """Return the stat information of the given path, without resolving links.""" + # ['mode', 'addr', 'dev', 'nlink', 'uid', 'gid', 'size', 'atime', 'mtime', 'ctime'] + return fsutil.stat_result( + [ + stat.S_IFREG | 0o777, + self.entry.inode, + id(self.fs), + 1, + 0, + 0, + len(self.entry.data) if self.entry.data else 0, + 0, + self.entry.mtime, + 0, + ] + ) + + +class ProxmoxConfigDirectoryEntry(VirtualDirectory): + def __init__(self, fs: VirtualFilesystem, path: str, entry: sqlite3.Row): + super().__init__(fs, path) + self.entry = entry + + def stat(self, follow_symlinks: bool = True) -> fsutil.stat_result: + """Return the stat information of this entry.""" + return self.lstat() + + def lstat(self) -> fsutil.stat_result: + """Return the stat information of the given path, without resolving links.""" + # ['mode', 'addr', 'dev', 'nlink', 'uid', 'gid', 'size', 'atime', 'mtime', 'ctime'] + return fsutil.stat_result( + [ + stat.S_IFDIR | 0o777, + self.entry.inode, + id(self.fs), + 1, + 0, + 0, + 0, + 0, + self.entry.mtime, + 0, + ] + ) \ No newline at end of file From 9d72586c908a80ea5721999ae6f30889831ac14e Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:51:32 +0200 Subject: [PATCH 18/36] Fixed bug causing pmxcfs root directories to be empty --- dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index a4e79a2f5..937e81370 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -84,21 +84,22 @@ def _create_pmxcfs(fh) -> VirtualFilesystem: filetree_table = db.table(FILETREE_TABLE_NAME) rows = filetree_table.rows() fs_entries = {} + + # index entries on their inodes for row in rows: fs_entries[row.inode] = row - # fs_entries.sort(key=lambda entry: (entry.parent, entry.inode)) vfs = VirtualFilesystem() for entry in fs_entries.values(): if entry.parent == 0: # Root entries do not require parent check path = entry.name else: - # import ipdb; ipdb.set_trace() parts = [] current = entry while current.parent != 0: parts.append(current.name) current = fs_entries[current.parent] + parts.append(current.name) # appends the missing root parent path = "/".join(parts[::-1]) if entry.type == 4: From 28e6e6ef8d0d534b894940376267a629c03d5c76 Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:36:34 +0200 Subject: [PATCH 19/36] Updated code to work with properly implemented pmxcfs --- .../os/unix/linux/debian/proxmox/_os.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 937e81370..a831f01ae 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -17,15 +17,13 @@ from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.target import Target -import ipdb log = logging.getLogger(__name__) PROXMOX_PACKAGE_NAME="proxmox-ve" FILETREE_TABLE_NAME="tree" PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" -# VM_CONFIG_PATH="/etc/pve/qemu-server" # TODO: Change to /etc/pve/nodes/pve/qemu-server once pmxcfs func has been reworked to properly map fs -VM_CONFIG_PATH="/etc/pve/" # TODO: properly implement pmxcfs creation and revert to propper path +PROXMOX_NODES_PATH="/etc/pve/nodes" VirtualMachineRecord = TargetRecordDescriptor( @@ -71,13 +69,18 @@ def version(self) -> str: @export(record=VirtualMachineRecord) def vm_list(self) -> Iterator[VirtualMachineRecord]: - configs = self.target.fs.path(VM_CONFIG_PATH) + configs = self.target.fs.path(self.vm_configs_path) for config in configs.iterdir(): - if pathlib.Path(config).suffix == ".conf": # TODO: Remove if statement once pmxcfs is propperly implemented - yield VirtualMachineRecord( - id=pathlib.Path(config).stem, - config_path=config, - ) + yield VirtualMachineRecord( + id=pathlib.Path(config).stem, + config_path=config, + ) + + @export(property=True) + def vm_configs_path(self) -> str: + """Returns path containing VM configurations of the target pve node""" + + return f"{PROXMOX_NODES_PATH}/{self.hostname}/qemu-server" def _create_pmxcfs(fh) -> VirtualFilesystem: db = sqlite3.SQLite3(fh) From d47aff6416bf2d1750598d44449d9692b176603b Mon Sep 17 00:00:00 2001 From: otnxSl <171164394+otnxSl@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:34:34 +0200 Subject: [PATCH 20/36] Cleaned up code a bit --- dissect/target/loaders/proxmox.py | 2 +- .../os/unix/linux/debian/proxmox/_os.py | 25 ------------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/dissect/target/loaders/proxmox.py b/dissect/target/loaders/proxmox.py index 214b8785d..e83987490 100644 --- a/dissect/target/loaders/proxmox.py +++ b/dissect/target/loaders/proxmox.py @@ -62,7 +62,7 @@ def _is_disk_device(config_value: str) -> bool | None: def _get_vm_disk_name(config_value: str) -> str | None: """Retrieves the disk device name from vm""" - disk = re.search(r"vm-[0-9]+-disk-[0-9]+(,|.+?,)", config_value) + disk = re.search(r"vm-[0-9]+-disk-[0-9]+", config_value) return disk.group(0).replace(",", "") if disk else None def _get_storage_ID(config_value: str) -> str | None: diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index a831f01ae..db1ce3777 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -116,31 +116,6 @@ def _create_pmxcfs(fh) -> VirtualFilesystem: return vfs -def _get_entry_parent_chain(fs_entries: list, entry: sqlite3[Row]) -> list[sqlite3[Row]]: - """Looks through the list of inodes (fs_entries) and retrieves parent inode of each node until root is found.""" - - inode_chain = [entry] - target = entry - for entry in fs_entries: - if target.inode != 0: - inode_chain.append(entry) - target = entry - else: - return inode_chain - -def _create_fs_path(entry_chain: list[sqlite3[Row]]) -> str: - """Creates a full path out of a sorted list of file entries""" - - entry_chain.sort(key=lambda entry: (entry.parent)) - entry_names = [] - for entry in entry_chain: - if entry.inode != 0: - entry_names.append(entry.name) - - path = "/".join(entry_names) - - return path - class ProxmoxConfigFileEntry(VirtualFile): def open(self) -> BinaryIO: From 6358c4634317e2569478e5eaee8958b81f14b6f6 Mon Sep 17 00:00:00 2001 From: Luke Paris Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 21/36] Fixed KeyError when loading Windows targets over SMB (#726) this is in regards with low-privileged user credentials --- dissect/target/plugins/os/windows/_os.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dissect/target/plugins/os/windows/_os.py b/dissect/target/plugins/os/windows/_os.py index 0a0851f73..8c6cd90e7 100644 --- a/dissect/target/plugins/os/windows/_os.py +++ b/dissect/target/plugins/os/windows/_os.py @@ -21,7 +21,7 @@ def __init__(self, target: Target): self.add_mounts() target.props["sysvol_drive"] = next( - (mnt for mnt, fs in target.fs.mounts.items() if fs is target.fs.mounts["sysvol"] and mnt != "sysvol"), + (mnt for mnt, fs in target.fs.mounts.items() if fs is target.fs.mounts.get("sysvol") and mnt != "sysvol"), None, ) @@ -78,13 +78,16 @@ def add_mounts(self) -> None: self.target.log.warning("Failed to map drive letters") self.target.log.debug("", exc_info=e) + sysvol_drive = self.target.fs.mounts.get("sysvol") # Fallback mount the sysvol to C: if we didn't manage to mount it to any other drive letter - if operator.countOf(self.target.fs.mounts.values(), self.target.fs.mounts["sysvol"]) == 1: + if sysvol_drive and operator.countOf(self.target.fs.mounts.values(), sysvol_drive) == 1: if "c:" not in self.target.fs.mounts: self.target.log.debug("Unable to determine drive letter of sysvol, falling back to C:") - self.target.fs.mount("c:", self.target.fs.mounts["sysvol"]) + self.target.fs.mount("c:", sysvol_drive) else: self.target.log.warning("Unknown drive letter for sysvol") + else: + self.target.log.warning("No sysvol drive found") @export(property=True) def hostname(self) -> Optional[str]: From dbe5869bddc64ba408732505a9a8c53102a87e7c Mon Sep 17 00:00:00 2001 From: cecinestpasunepipe <110607403+cecinestpasunepipe@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 22/36] Add glob/dump function for config tree (#728) (DIS-2163) --- .../target/plugins/os/unix/etc/__init__.py | 0 dissect/target/plugins/os/unix/etc/etc.py | 77 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 dissect/target/plugins/os/unix/etc/__init__.py create mode 100644 dissect/target/plugins/os/unix/etc/etc.py diff --git a/dissect/target/plugins/os/unix/etc/__init__.py b/dissect/target/plugins/os/unix/etc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dissect/target/plugins/os/unix/etc/etc.py b/dissect/target/plugins/os/unix/etc/etc.py new file mode 100644 index 000000000..ba546aabc --- /dev/null +++ b/dissect/target/plugins/os/unix/etc/etc.py @@ -0,0 +1,77 @@ +import fnmatch +import logging +import re +from pathlib import Path +from typing import Iterator + +from dissect.target import Target +from dissect.target.helpers.record import TargetRecordDescriptor +from dissect.target.plugin import arg, export +from dissect.target.plugins.general.config import ( + ConfigurationEntry, + ConfigurationTreePlugin, +) + +UnixConfigTreeRecord = TargetRecordDescriptor( + "unix/config", + [ + ("path", "source"), + ("path", "path"), + ("string", "key"), + ("string[]", "value"), + ], +) + +log = logging.getLogger(__name__) + + +class EtcTree(ConfigurationTreePlugin): + __namespace__ = "etc" + + def __init__(self, target: Target): + super().__init__(target, "/etc") + + def _sub(self, items: ConfigurationEntry, entry: Path, pattern: str) -> Iterator[UnixConfigTreeRecord]: + index = 0 + config_entry = items + if not isinstance(items, dict): + items = items.as_dict() + + for raw_key, value in items.items(): + key = re.sub(r"[\n\r\t]", "", raw_key) + path = Path(entry) / Path(key) + + if isinstance(value, dict): + yield from self._sub(value, path, pattern) + continue + + if not isinstance(value, list): + value = [str(value)] + + if fnmatch.fnmatch(path, pattern): + data = { + "_target": self.target, + "source": self.target.fs.path(config_entry.entry.path), + "path": path, + "key": key, + "value": value, + } + if value == [""]: + data["key"] = index + data["value"] = [key] + index += 1 + + yield UnixConfigTreeRecord(**data) + + @export(record=UnixConfigTreeRecord) + @arg("--glob", dest="pattern", required=False, default="*", type=str, help="Glob-style pattern to search for") + def etc(self, pattern: str) -> Iterator[UnixConfigTreeRecord]: + for entry, subs, items in self.config_fs.walk("/"): + for item in items: + try: + config_object = self.get(str(Path(entry) / Path(item))) + if isinstance(config_object, ConfigurationEntry): + yield from self._sub(config_object, Path(entry) / Path(item), pattern) + except Exception: + log.warning("Could not open configuration item: %s", item) + pass From 9640951b71fe1ad869cf565fe233afee56b8c8d3 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 23/36] Fix edge case where unix history path is a directory (#727) --- dissect/target/plugins/os/unix/history.py | 2 +- tests/plugins/os/unix/test_history.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dissect/target/plugins/os/unix/history.py b/dissect/target/plugins/os/unix/history.py index edc1910ee..2798cdc98 100644 --- a/dissect/target/plugins/os/unix/history.py +++ b/dissect/target/plugins/os/unix/history.py @@ -52,7 +52,7 @@ def _find_history_files(self) -> List[Tuple[str, TargetPath, UnixUserRecord]]: for user_details in self.target.user_details.all_with_home(): for shell, history_relative_path in self.COMMAND_HISTORY_RELATIVE_PATHS: history_path = user_details.home_path.joinpath(history_relative_path) - if history_path.exists(): + if history_path.is_file(): history_files.append((shell, history_path, user_details.user)) return history_files diff --git a/tests/plugins/os/unix/test_history.py b/tests/plugins/os/unix/test_history.py index 09993c3f0..06e71dae6 100644 --- a/tests/plugins/os/unix/test_history.py +++ b/tests/plugins/os/unix/test_history.py @@ -5,6 +5,8 @@ from dissect.util.ts import from_unix from flow.record.fieldtypes import datetime as dt +from dissect.target import Target +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.os.unix.history import CommandHistoryPlugin @@ -214,3 +216,22 @@ def test_commandhistory_database_history(target_unix_users, fs_unix, db_type, db assert results[i].command == line assert results[i].shell == db_type assert results[i].source.as_posix() == f"/root/{db_file}" + + +def test_commandhistory_is_directory(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None: + commandhistory_data = """test""" + + fs_unix.map_file_fh( + "/root/.zsh_history", + BytesIO(textwrap.dedent(commandhistory_data).encode()), + ) + + fs_unix.makedirs("/root/.bash_history") + results = list(target_unix_users.commandhistory()) + + assert len(results) == 1 + + assert results[0].ts is None + assert results[0].command == "test" + assert results[0].shell == "zsh" + assert results[0].source.as_posix() == "/root/.zsh_history" From 480317fb997e9718b6941521ae77d6a99a53305f Mon Sep 17 00:00:00 2001 From: pyrco <105293448+pyrco@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 24/36] Bump dissect.ctruct dependency to version 4 (#731) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c907b8e1c..090ec490b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ "defusedxml", - "dissect.cstruct>=4.dev,<5", + "dissect.cstruct>=4,<5", "dissect.eventlog>=3,<4", "dissect.evidence>=3,<4", "dissect.hypervisor>=3,<4", From 823dc77f370e820516a1d74e4ef80150158238ea Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 25/36] Correctly detect Windows 11 builds (#714) --- dissect/target/plugins/os/windows/_os.py | 14 +++++++++++--- tests/plugins/os/windows/test__os.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/dissect/target/plugins/os/windows/_os.py b/dissect/target/plugins/os/windows/_os.py index 8c6cd90e7..5d5b55fdf 100644 --- a/dissect/target/plugins/os/windows/_os.py +++ b/dissect/target/plugins/os/windows/_os.py @@ -247,13 +247,21 @@ def _part_str(parts: dict[str, Any], name: str) -> str: if any(map(lambda value: value is not None, version_parts.values())): version = [] + nt_version = _part_str(version_parts, "CurrentVersion") + build_version = _part_str(version_parts, "CurrentBuildNumber") prodcut_name = _part_str(version_parts, "ProductName") - version.append(prodcut_name) - nt_version = _part_str(version_parts, "CurrentVersion") + # CurrentBuildNumber >= 22000 on NT 10.0 indicates Windows 11. + # https://learn.microsoft.com/en-us/windows/release-health/windows11-release-information + try: + if nt_version == "10.0" and int(build_version) >= 22_000: + prodcut_name = prodcut_name.replace("Windows 10", "Windows 11") + except ValueError: + pass + + version.append(prodcut_name) version.append(f"(NT {nt_version})") - build_version = _part_str(version_parts, "CurrentBuildNumber") ubr = version_parts["UBR"] if ubr: build_version = f"{build_version}.{ubr}" diff --git a/tests/plugins/os/windows/test__os.py b/tests/plugins/os/windows/test__os.py index 683b4c17b..8cc9b6422 100644 --- a/tests/plugins/os/windows/test__os.py +++ b/tests/plugins/os/windows/test__os.py @@ -202,6 +202,26 @@ def test_windowsplugin__nt_version( ], " (NT ) 5678", ), + ( + [ + ("ProductName", "Windows 10 Pro"), + ("CurrentMajorVersionNumber", 10), + ("CurrentMinorVersionNumber", 0), + ("CurrentBuildNumber", 19_045), + ("UBR", 1234), + ], + "Windows 10 Pro (NT 10.0) 19045.1234", + ), + ( + [ + ("ProductName", "Windows 10 Enterprise"), + ("CurrentMajorVersionNumber", 10), + ("CurrentMinorVersionNumber", 0), + ("CurrentBuildNumber", 22_000), + ("UBR", 1234), + ], + "Windows 11 Enterprise (NT 10.0) 22000.1234", + ), ], ) def test_windowsplugin_version( From f7abd55206640d216c393bd5499bf59ef35a5d5e Mon Sep 17 00:00:00 2001 From: Miauwkeru Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 26/36] Fix EOF read error for char arrays in a BEEF0004 shellbag (#730) --- .../plugins/os/windows/regf/shellbags.py | 13 +++-- .../plugins/os/windows/regf/test_shellbags.py | 48 +++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 tests/plugins/os/windows/regf/test_shellbags.py diff --git a/dissect/target/plugins/os/windows/regf/shellbags.py b/dissect/target/plugins/os/windows/regf/shellbags.py index 0dd648b9c..14be6320e 100644 --- a/dissect/target/plugins/os/windows/regf/shellbags.py +++ b/dissect/target/plugins/os/windows/regf/shellbags.py @@ -907,17 +907,20 @@ def __init__(self, buf): self.file_reference = c_bag.uint64(fh) c_bag.uint64(fh) if version >= 3: - long_len = c_bag.uint16(fh) + # Start of strings + localized_name_offset = c_bag.uint16(fh) if version >= 9: c_bag.uint32(fh) if version >= 8: c_bag.uint32(fh) if version >= 3: self.long_name = c_bag.wchar[None](fh) - if 3 <= version < 7 and long_len > 0: - self.localized_name = c_bag.char[long_len](fh) - if version >= 7 and long_len > 0: - self.localized_name = c_bag.wchar[long_len](fh) + + if 3 <= version < 7 and localized_name_offset > 0: + self.localized_name = c_bag.char[None](fh) + + if version >= 7 and localized_name_offset > 0: + self.localized_name = c_bag.wchar[None](fh) class EXTENSION_BLOCK_BEEF0005(EXTENSION_BLOCK): # noqa diff --git a/tests/plugins/os/windows/regf/test_shellbags.py b/tests/plugins/os/windows/regf/test_shellbags.py new file mode 100644 index 000000000..03795558b --- /dev/null +++ b/tests/plugins/os/windows/regf/test_shellbags.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import datetime + +import pytest + +from dissect.target.plugins.os.windows.regf.shellbags import parse_shell_item_list + + +@pytest.mark.parametrize( + "shellbag, name, modification_time, localized_name", + [ + ( + ( + b"X\x001\x00\x00\x00\x00\x00WX\xc1I\x11\x00MENUST~1\x00\x00@\x00\x03\x00\x04\x00\xef\xbeWXZBWXZB\x14" + b"\x00*\x00M\x00e\x00n\x00u\x00 \x00S\x00t\x00a\x00r\x00t\x00\x00\x00@shell32.dll,-21786\x00\x18\x00" + b"\x00\x00" + ), + "Menu Start", + datetime.datetime(2024, 2, 23, 9, 14, 2), + b"@shell32.dll,-21786", + ), + ( + ( + b"x\x001\x00\x00\x00\x00\x00\x17W\x0bk\x11\x00Users\x00d\x00\t\x00\x04\x00\xef\xbe\xa7T,*\x91X\xe6R." + b"\x00\x00\x00\n\x08\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00:\x00\x00\x00\x00\x00A" + b"\x06\x85\x00U\x00s\x00e\x00r\x00s\x00\x00\x00@\x00s\x00h\x00e\x00l\x00l\x003\x002\x00.\x00d\x00l" + b"\x00l\x00,\x00-\x002\x001\x008\x001\x003\x00\x00\x00\x14\x00\x00\x00" + ), + "Users", + datetime.datetime(2023, 8, 23, 13, 24, 22), + "@shell32.dll,-21813", + ), + ], + ids=["char", "wchar"], +) +def test_parse_shell_item_list( + shellbag: bytes, name: str, modification_time: datetime.datetime, localized_name: str | bytes +) -> None: + bag = next(parse_shell_item_list(shellbag)) + + assert bag.name == name + assert bag.modification_time == modification_time + + extension = bag.extensions[0] + + assert extension.long_name == name + assert extension.localized_name == localized_name From 55fd035ddae96d198dbf6143bdc22bbe018b5769 Mon Sep 17 00:00:00 2001 From: cecinestpasunepipe <110607403+cecinestpasunepipe@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 27/36] Add username and password options to MQTT loader (#732) (DIS-3240) --- dissect/target/loaders/mqtt.py | 15 ++++++++++++++- tests/loaders/test_mqtt.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dissect/target/loaders/mqtt.py b/dissect/target/loaders/mqtt.py index cf87f4973..5deecc7e5 100644 --- a/dissect/target/loaders/mqtt.py +++ b/dissect/target/loaders/mqtt.py @@ -10,6 +10,7 @@ import urllib from dataclasses import dataclass from functools import lru_cache +from getpass import getpass from pathlib import Path from struct import pack, unpack_from from threading import Thread @@ -270,19 +271,25 @@ class Broker: case = None bytes_received = 0 monitor = False + username = None + password = None diskinfo = {} index = {} topo = {} factor = 1 - def __init__(self, broker: Broker, port: str, key: str, crt: str, ca: str, case: str, **kwargs): + def __init__( + self, broker: Broker, port: str, key: str, crt: str, ca: str, case: str, username: str, password: str, **kwargs + ): self.broker_host = broker self.broker_port = int(port) self.private_key_file = key self.certificate_file = crt self.cacert_file = ca self.case = case + self.username = username + self.password = password self.command = kwargs.get("command", None) def clear_cache(self) -> None: @@ -393,6 +400,7 @@ def connect(self) -> None: tls_version=ssl.PROTOCOL_TLS, ciphers=None, ) + self.mqtt_client.username_pw_set(self.username, self.password) self.mqtt_client.tls_insecure_set(True) # merely having the correct cert is ok self.mqtt_client.on_connect = self._on_connect self.mqtt_client.on_message = self._on_message @@ -411,6 +419,8 @@ def connect(self) -> None: @arg("--mqtt-ca", dest="ca", help="certificate authority file") @arg("--mqtt-command", dest="command", help="direct command to client(s)") @arg("--mqtt-diag", action="store_true", dest="diag", help="show MQTT diagnostic information") +@arg("--mqtt-username", dest="username", help="Username for connection") +@arg("--mqtt-password", action="store_true", dest="password", help="Ask for password before connecting") class MQTTLoader(Loader): """Load remote targets through a broker.""" @@ -435,7 +445,10 @@ def find_all(path: Path, **kwargs) -> Iterator[str]: if cls.broker is None: if (uri := kwargs.get("parsed_path")) is None: raise LoaderError("No URI connection details have been passed.") + options = dict(urllib.parse.parse_qsl(uri.query, keep_blank_values=True)) + if options.get("password"): + options["password"] = getpass() cls.broker = Broker(**options) cls.broker.connect() num_peers = int(options.get("peers", 1)) diff --git a/tests/loaders/test_mqtt.py b/tests/loaders/test_mqtt.py index 15ba108c3..d221ed44a 100644 --- a/tests/loaders/test_mqtt.py +++ b/tests/loaders/test_mqtt.py @@ -114,7 +114,7 @@ def test_remote_loader_stream( ) -> None: from dissect.target.loaders.mqtt import Broker - broker = Broker("0.0.0.0", "1884", "key", "crt", "ca", "case1") + broker = Broker("0.0.0.0", "1884", "key", "crt", "ca", "case1", "user", "pass") broker.connect() broker.mqtt_client.fill_disks(disks) broker.mqtt_client.hostnames = hosts From 2c8870346dc356497c9b8ab393eddf205e6cc917 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 28/36] Make ESXi Plugin work without crypto and fix vm_inventory (#697) --- dissect/target/plugins/os/unix/esxi/_os.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/dissect/target/plugins/os/unix/esxi/_os.py b/dissect/target/plugins/os/unix/esxi/_os.py index c46b005ff..a177b35f5 100644 --- a/dissect/target/plugins/os/unix/esxi/_os.py +++ b/dissect/target/plugins/os/unix/esxi/_os.py @@ -17,9 +17,14 @@ from dissect.target.helpers.fsutil import TargetPath try: - from dissect.hypervisor.util.envelope import Envelope, KeyStore + from dissect.hypervisor.util.envelope import ( + HAS_PYCRYPTODOME, + HAS_PYSTANDALONE, + Envelope, + KeyStore, + ) - HAS_ENVELOPE = True + HAS_ENVELOPE = HAS_PYCRYPTODOME or HAS_PYSTANDALONE except ImportError: HAS_ENVELOPE = False @@ -70,7 +75,8 @@ def __init__(self, target: Target): def _cfg(self, path: str) -> Optional[str]: if not self._config: - raise ValueError("No ESXi config!") + self.target.log.warning("No ESXi config!") + return None value_name = path.strip("/").split("/")[-1] obj = _traverse(path, self._config) @@ -95,13 +101,13 @@ def detect(cls, target: Target) -> Optional[Filesystem]: def create(cls, target: Target, sysvol: Filesystem) -> ESXiPlugin: cfg = parse_boot_cfg(sysvol.path("boot.cfg").open("rt")) + # Mount all the visor tars in individual filesystem layers + _mount_modules(target, sysvol, cfg) + # Create a root layer for the "local state" filesystem # This stores persistent configuration data local_layer = target.fs.append_layer() - # Mount all the visor tars in individual filesystem layers - _mount_modules(target, sysvol, cfg) - # Mount the local.tgz to the local state layer _mount_local(target, local_layer) From d8df205465c0a92514590256889562eb0a3ecb05 Mon Sep 17 00:00:00 2001 From: Erik Schamper <1254028+Schamper@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 29/36] Fix visual bugs in cyber (#738) --- dissect/target/helpers/cyber.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dissect/target/helpers/cyber.py b/dissect/target/helpers/cyber.py index a44117ddd..dd26b54db 100644 --- a/dissect/target/helpers/cyber.py +++ b/dissect/target/helpers/cyber.py @@ -137,7 +137,7 @@ def nms(buf: str, color: Optional[Color] = None, mask_space: bool = False, mask_ if ( ("\n" in char or "\r\n" in char) - or (not mask_space and char == " " and not is_indent and not mask_indent) + or (not mask_space and char == " " and not is_indent) or (not mask_indent and is_indent) ): if "\n" in char: @@ -189,7 +189,7 @@ def nms(buf: str, color: Optional[Color] = None, mask_space: bool = False, mask_ if ( ("\n" in char or "\r\n" in char) - or (not mask_space and char == " " and not is_indent and not mask_indent) + or (not mask_space and char == " " and not is_indent) or (not mask_indent and is_indent) ): if "\n" in char: @@ -268,6 +268,8 @@ def matrix(buf: str, color: Optional[Color] = None, **kwargs) -> None: if cur_ansi: char = cur_ansi + char + "\033[0m" + if end_ansi: + cur_ansi = "" if "\n" in char or "\r\n" in char: char = " " + char From fa17a5ee0d81f4353e28bf7441a2945b6a092a09 Mon Sep 17 00:00:00 2001 From: Erik Schamper <1254028+Schamper@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 30/36] Improve type hint in Defender plugin (#739) --- dissect/target/plugins/os/windows/defender.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dissect/target/plugins/os/windows/defender.py b/dissect/target/plugins/os/windows/defender.py index 1a433cceb..7ea065dd0 100644 --- a/dissect/target/plugins/os/windows/defender.py +++ b/dissect/target/plugins/os/windows/defender.py @@ -7,7 +7,7 @@ from typing import Any, BinaryIO, Generator, Iterable, Iterator, TextIO, Union import dissect.util.ts as ts -from dissect.cstruct import Structure, cstruct +from dissect.cstruct import cstruct from flow.record import Record from dissect.target import plugin @@ -357,7 +357,7 @@ def __init__(self, fh: BinaryIO): resource_info = c_defender.QuarantineEntrySection2(resource_buf) # List holding all quarantine entry resources that belong to this quarantine entry. - self.resources = [] + self.resources: list[QuarantineEntryResource] = [] for offset in resource_info.EntryOffsets: resource_buf.seek(offset) @@ -393,7 +393,7 @@ def __init__(self, fh: BinaryIO): # Move pointer offset += 4 + field.Size - def _add_field(self, field: Structure): + def _add_field(self, field: c_defender.QuarantineEntryResourceField) -> None: if field.Identifier == FIELD_IDENTIFIER.CQuaResDataID_File: self.resource_id = field.Data.hex().upper() elif field.Identifier == FIELD_IDENTIFIER.PhysicalPath: From 7b953946902cb427111c9990a9347e44823bca0e Mon Sep 17 00:00:00 2001 From: cecinestpasunepipe <110607403+cecinestpasunepipe@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 31/36] Fix issue with MPLogs (#742) --- dissect/target/plugins/os/windows/defender.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dissect/target/plugins/os/windows/defender.py b/dissect/target/plugins/os/windows/defender.py index 7ea065dd0..17c999588 100644 --- a/dissect/target/plugins/os/windows/defender.py +++ b/dissect/target/plugins/os/windows/defender.py @@ -627,6 +627,9 @@ def _mplog_block( if suffix.search(mplog_line): break match = pattern.match(block) + if not match: + return + data = match.groupdict() data["_target"] = self.target data["source_log"] = source From 1bd787f9631d7f8fe6bd2d53775f44614129f222 Mon Sep 17 00:00:00 2001 From: cecinestpasunepipe <110607403+cecinestpasunepipe@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 32/36] Use target logger in etc-plugin (#741) --- dissect/target/plugins/os/unix/etc/etc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dissect/target/plugins/os/unix/etc/etc.py b/dissect/target/plugins/os/unix/etc/etc.py index ba546aabc..b6689eb5f 100644 --- a/dissect/target/plugins/os/unix/etc/etc.py +++ b/dissect/target/plugins/os/unix/etc/etc.py @@ -1,5 +1,4 @@ import fnmatch -import logging import re from pathlib import Path from typing import Iterator @@ -22,8 +21,6 @@ ], ) -log = logging.getLogger(__name__) - class EtcTree(ConfigurationTreePlugin): __namespace__ = "etc" @@ -73,5 +70,5 @@ def etc(self, pattern: str) -> Iterator[UnixConfigTreeRecord]: if isinstance(config_object, ConfigurationEntry): yield from self._sub(config_object, Path(entry) / Path(item), pattern) except Exception: - log.warning("Could not open configuration item: %s", item) + self.target.log.warning("Could not open configuration item: %s", item) pass From 629dfcb0362678884d5e489d31c6a863e2be2c85 Mon Sep 17 00:00:00 2001 From: Miauwkeru Date: Tue, 9 Jul 2024 22:43:50 +0200 Subject: [PATCH 33/36] Fix TargetPath instances for configutil.parse (#743) --- dissect/target/helpers/configutil.py | 6 +++--- tests/helpers/test_configutil.py | 28 +++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/dissect/target/helpers/configutil.py b/dissect/target/helpers/configutil.py index 8be76beda..bb808c656 100644 --- a/dissect/target/helpers/configutil.py +++ b/dissect/target/helpers/configutil.py @@ -748,13 +748,13 @@ def parse(path: Union[FilesystemEntry, TargetPath], hint: Optional[str] = None, FileNotFoundError: If the ``path`` is not a file. """ - if not path.is_file(follow_symlinks=True): - raise FileNotFoundError(f"Could not parse {path} as a dictionary.") - entry = path if isinstance(path, TargetPath): entry = path.get() + if not entry.is_file(follow_symlinks=True): + raise FileNotFoundError(f"Could not parse {path} as a dictionary.") + options = ParserOptions(*args, **kwargs) return parse_config(entry, hint, options) diff --git a/tests/helpers/test_configutil.py b/tests/helpers/test_configutil.py index b97cfe95b..49fe54d4e 100644 --- a/tests/helpers/test_configutil.py +++ b/tests/helpers/test_configutil.py @@ -1,10 +1,13 @@ +from __future__ import annotations + import textwrap from io import StringIO from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Union import pytest +from dissect.target.exceptions import FileNotFoundError from dissect.target.helpers.configutil import ( ConfigurationParser, Default, @@ -12,9 +15,14 @@ Json, ScopeManager, SystemD, + parse, ) from tests._utils import absolute_path +if TYPE_CHECKING: + from dissect.target import Target + from dissect.target.filesystem import VirtualFilesystem + def parse_data(parser_type: type[ConfigurationParser], data_to_read: str, *args, **kwargs) -> dict: """Initializes parser_type as a parser which parses ``data_to_read``""" @@ -258,3 +266,21 @@ def test_json_syntax(data_string: str, expected_data: Union[dict, list]) -> None parser.parse_file(StringIO(data_string)) assert parser.parsed_data == expected_data + + +def test_parse(target_linux: Target, fs_linux: VirtualFilesystem, tmp_path: Path) -> None: + # File does not exist on the system in the first place + with pytest.raises(FileNotFoundError): + parse(target_linux.fs.path("/path/to/file")) + + file_path = tmp_path.joinpath("path/to/file") + file_path.parent.mkdir(parents=True) + file_path.touch() + + fs_linux.map_dir("/", tmp_path.absolute()) + + # Trying to read a directory + with pytest.raises(FileNotFoundError): + parse(target_linux.fs.path("/path/to")) + + parse(target_linux.fs.path("/path/to/file")) From 63dccd672551ea5ed98600728aaa5fd453b319a1 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:57:53 +0200 Subject: [PATCH 34/36] Refactor --- dissect/target/loaders/proxmox.py | 98 ++++++------ dissect/target/plugin.py | 17 +- dissect/target/plugins/child/proxmox.py | 10 +- dissect/target/plugins/os/unix/_os.py | 24 +-- .../os/unix/linux/debian/proxmox/_os.py | 150 +++++++----------- .../os/unix/linux/debian/proxmox/vm.py | 29 ++++ 6 files changed, 163 insertions(+), 165 deletions(-) create mode 100644 dissect/target/plugins/os/unix/linux/debian/proxmox/vm.py diff --git a/dissect/target/loaders/proxmox.py b/dissect/target/loaders/proxmox.py index e83987490..9d8dd2f80 100644 --- a/dissect/target/loaders/proxmox.py +++ b/dissect/target/loaders/proxmox.py @@ -1,70 +1,68 @@ from __future__ import annotations -from pathlib import Path + import re +from pathlib import Path -from dissect.target.containers.raw import RawContainer +from dissect.target import container from dissect.target.loader import Loader +from dissect.target.target import Target +RE_VOLUME_ID = re.compile(r"(?:file=)?([^:]+):([^,]+)") class ProxmoxLoader(Loader): - """Load Proxmox disk data onto target disks. - - The method proxmox uses to store disk data varies on multiple factors such as - filesystem used, available storage space and other factors. This information is - stored within a config file in the filesystem. + """Loader for Proxmox VM configuration files. - This loader attains the necessary information to find the disk data on the - filesystem and appends it to the target filesystem's disks. + Proxmox uses volume identifiers in the format of ``storage_id:volume_id``. The ``storage_id`` maps to a + storage configuration in ``/etc/pve/storage.cfg``. The ``volume_id`` is the name of the volume within + that configuration. + + This loader currently does not support parsing the storage configuration, so it will attempt to open the + volume directly from the same directory as the configuration file, or from ``/dev/pve/`` (default LVM config). + If the volume is not found, it will log a warning. """ - def __init__(self, path, **kwargs): + def __init__(self, path: Path, **kwargs): path = path.resolve() - super().__init__(path) + super().__init__(path) + self.base_dir = path.parent @staticmethod - def detect(path) -> bool: - return path.suffix.lower() == ".conf" - - def map(self, target): - parsed_config = self._parse_vm_configuration(self.path) - - for option in parsed_config: - config_value = parsed_config[option] - vm_disk = _get_vm_disk_name(config_value) + def detect(path: Path) -> bool: + if path.suffix.lower() != ".conf": + return False - if _is_disk_device(option) and vm_disk is not None: - disk_interface = option - vm_id = self.path.stem - name = parsed_config['name'] - storage_id = _get_storage_ID(config_value) + with path.open("rb") as fh: + lines = fh.read(512).split(b"\n") + needles = [b"cpu:", b"memory:", b"name:"] + return all(any(needle in line for line in lines) for needle in needles) - path = self.path.joinpath("/dev/pve/", vm_disk) - try: - target.disks.add(RawContainer(path.open("rb"))) - except Exception: - target.log.exception("Failed to load block device: %s", vm_disk) - - def _parse_vm_configuration(self, conf) -> list: - lines = conf.read_text().split("\n") - lines.remove("") # Removes any trailing empty lines in file - parsed_lines = {} + def map(self, target: Target) -> None: + with self.path.open("rt") as fh: + for line in fh: + if not (line := line.strip()): + continue - for line in lines: - key, value = line.split(': ') - parsed_lines[key] = value - - return parsed_lines + key, value = line.split(":", 1) + value = value.strip() -def _is_disk_device(config_value: str) -> bool | None: - disk = re.match(r"^(sata|scsi|ide)[0-9]+$", config_value) - return True if disk else None + if key.startswith(("scsi", "sata", "ide", "virtio")) and key[-1].isdigit(): + # https://pve.proxmox.com/wiki/Storage + if match := RE_VOLUME_ID.match(value): + storage_id, volume_id = match.groups() -def _get_vm_disk_name(config_value: str) -> str | None: - """Retrieves the disk device name from vm""" - disk = re.search(r"vm-[0-9]+-disk-[0-9]+", config_value) - return disk.group(0).replace(",", "") if disk else None + # TODO: parse the storage information from /etc/pve/storage.cfg + # For now, let's try a few assumptions + disk_path = None + if (path := self.base_dir.joinpath(volume_id)).exists(): + disk_path = path + elif (path := self.base_dir.joinpath("/dev/pve/").joinpath(volume_id)).exists(): + disk_path = path -def _get_storage_ID(config_value: str) -> str | None: - storage_id = config_value.split(":") - return storage_id[0] if storage_id else None \ No newline at end of file + if disk_path: + try: + target.disks.add(container.open(disk_path)) + except Exception: + target.log.exception("Failed to open disk: %s", disk_path) + else: + target.log.warning("Unable to find disk: %s:%s", storage_id, volume_id) diff --git a/dissect/target/plugin.py b/dissect/target/plugin.py index e46511de8..1b28e5e6d 100644 --- a/dissect/target/plugin.py +++ b/dissect/target/plugin.py @@ -2,6 +2,7 @@ See dissect/target/plugins/general/example.py for an example plugin. """ + from __future__ import annotations import fnmatch @@ -55,18 +56,18 @@ class OperatingSystem(StrEnum): - LINUX = "linux" - WINDOWS = "windows" - ESXI = "esxi" + ANDROID = "android" BSD = "bsd" + CITRIX = "citrix-netscaler" + ESXI = "esxi" + FORTIOS = "fortios" + IOS = "ios" + LINUX = "linux" OSX = "osx" + PROXMOX = "proxmox" UNIX = "unix" - ANDROID = "android" VYOS = "vyos" - IOS = "ios" - FORTIOS = "fortios" - CITRIX = "citrix-netscaler" - PROXMOX = "proxmox" + WINDOWS = "windows" def export(*args, **kwargs) -> Callable: diff --git a/dissect/target/plugins/child/proxmox.py b/dissect/target/plugins/child/proxmox.py index 17611c2b1..914fc789c 100644 --- a/dissect/target/plugins/child/proxmox.py +++ b/dissect/target/plugins/child/proxmox.py @@ -1,3 +1,5 @@ +from typing import Iterator + from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.record import ChildTargetRecord from dissect.target.plugin import ChildTargetPlugin @@ -10,12 +12,12 @@ class ProxmoxChildTargetPlugin(ChildTargetPlugin): def check_compatible(self) -> None: if self.target.os != "proxmox": - raise UnsupportedPluginError("Not an promox operating system") + raise UnsupportedPluginError("Not a Proxmox operating system") - def list_children(self): - for vm in self.target.vm_list(): + def list_children(self) -> Iterator[ChildTargetRecord]: + for vm in self.target.vmlist(): yield ChildTargetRecord( type=self.__type__, - path=vm.config_path, + path=vm.path, _target=self.target, ) diff --git a/dissect/target/plugins/os/unix/_os.py b/dissect/target/plugins/os/unix/_os.py index a6935aa17..8d803db05 100644 --- a/dissect/target/plugins/os/unix/_os.py +++ b/dissect/target/plugins/os/unix/_os.py @@ -6,8 +6,6 @@ from struct import unpack from typing import Iterator, Optional, Union -from dissect.util.stream import BufferedStream - from dissect.target.filesystem import Filesystem, VirtualFilesystem from dissect.target.helpers.fsutil import TargetPath from dissect.target.helpers.record import UnixUserRecord @@ -15,8 +13,6 @@ from dissect.target.plugin import OperatingSystem, OSPlugin, arg, export from dissect.target.target import Target -from dissect.target.volumes import lvm - log = logging.getLogger(__name__) @@ -24,7 +20,7 @@ class UnixPlugin(OSPlugin): def __init__(self, target: Target): super().__init__(target) self._add_mounts() - self._add_lvm_devices() + self._add_devices() self._hostname_dict = self._parse_hostname_string() self._hosts_dict = self._parse_hosts_string() self._os_release = self._parse_os_release() @@ -235,15 +231,19 @@ def _add_mounts(self) -> None: self.target.log.debug("Mounting %s (%s) at %s", fs, fs.volume, mount_point) self.target.fs.mount(mount_point, fs) - def _add_lvm_devices(self) -> None: - """Parses and mounts lvm devices from external target to local target fs""" + def _add_devices(self) -> None: + """Add some virtual block devices to the target. + + Currently only adds LVM devices. + """ vfs = VirtualFilesystem() + for volume in self.target.volumes: - if isinstance(volume.vs, lvm.LvmVolumeSystem) and "disk" in volume.name: - vfs.map_file_fh(f"{volume.raw.vg.name}/{volume.raw.name}", BufferedStream(volume)) - - self.target.fs.mount("/dev", vfs) - + if volume.vs and volume.vs.__type__ == "lvm": + vfs.map_file_fh(f"{volume.raw.vg.name}/{volume.raw.name}", volume) + + self.target.fs.mount("/dev", vfs, ignore_existing=False) + def _parse_os_release(self, glob: Optional[str] = None) -> dict[str, str]: """Parse files containing Unix version information. diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index db1ce3777..3ba60a863 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -1,138 +1,110 @@ from __future__ import annotations -import os -import re import stat -import pathlib -import logging from io import BytesIO -from typing import Optional +from typing import BinaryIO from dissect.sql import sqlite3 +from dissect.util.stream import BufferedStream -from dissect.target.filesystem import Filesystem, VirtualFilesystem, VirtualFile, VirtualDirectory +from dissect.target.filesystem import ( + Filesystem, + VirtualDirectory, + VirtualFile, + VirtualFilesystem, +) from dissect.target.helpers import fsutil from dissect.target.plugins.os.unix._os import OperatingSystem, export -from dissect.target.plugins.os.unix.linux._os import LinuxPlugin -from dissect.target.helpers.record import TargetRecordDescriptor +from dissect.target.plugins.os.unix.linux.debian._os import DebianPlugin from dissect.target.target import Target -log = logging.getLogger(__name__) - -PROXMOX_PACKAGE_NAME="proxmox-ve" -FILETREE_TABLE_NAME="tree" -PMXCFS_DATABASE_PATH="/var/lib/pve-cluster/config.db" -PROXMOX_NODES_PATH="/etc/pve/nodes" - - -VirtualMachineRecord = TargetRecordDescriptor( - "proxmox/vm", - [ - ("string", "id"), - ("string", "config_path"), - ], -) - - -class ProxmoxPlugin(LinuxPlugin): - def __init__(self, target: Target): - super().__init__(target) - +class ProxmoxPlugin(DebianPlugin): @classmethod - def detect(cls, target: Target) -> Optional[Filesystem]: + def detect(cls, target: Target) -> Filesystem | None: for fs in target.filesystems: - if (fs.exists("/etc/pve") or fs.exists("/var/lib/pve")): + if fs.exists("/etc/pve") or fs.exists("/var/lib/pve"): return fs + return None @classmethod def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: obj = super().create(target, sysvol) - pmxcfs = _create_pmxcfs(sysvol.path(PMXCFS_DATABASE_PATH).open("rb")) - target.fs.mount("/etc/pve", pmxcfs) - return obj + with target.fs.path("/var/lib/pve-cluster/config.db").open("rb") as fh: + vfs = _create_pmxcfs(fh, obj.hostname) - @export(property=True) - def os(self) -> str: - return OperatingSystem.PROXMOX.value + target.fs.mount("/etc/pve", vfs) + + return obj @export(property=True) def version(self) -> str: - """Returns Proxmox VE version with underlying os release""" + """Returns Proxmox VE version with underlying OS release.""" for pkg in self.target.dpkg.status(): - if pkg.name == PROXMOX_PACKAGE_NAME: + if pkg.name == "proxmox-ve": distro_name = self._os_release.get("PRETTY_NAME", "") return f"{pkg.name} {pkg.version} ({distro_name})" - @export(record=VirtualMachineRecord) - def vm_list(self) -> Iterator[VirtualMachineRecord]: - configs = self.target.fs.path(self.vm_configs_path) - for config in configs.iterdir(): - yield VirtualMachineRecord( - id=pathlib.Path(config).stem, - config_path=config, - ) - @export(property=True) - def vm_configs_path(self) -> str: - """Returns path containing VM configurations of the target pve node""" + def os(self) -> str: + return OperatingSystem.PROXMOX.value - return f"{PROXMOX_NODES_PATH}/{self.hostname}/qemu-server" -def _create_pmxcfs(fh) -> VirtualFilesystem: +DT_DIR = 4 +DT_REG = 8 + + +def _create_pmxcfs(fh: BinaryIO, hostname: str | None = None) -> VirtualFilesystem: + # https://pve.proxmox.com/wiki/Proxmox_Cluster_File_System_(pmxcfs) db = sqlite3.SQLite3(fh) - filetree_table = db.table(FILETREE_TABLE_NAME) - rows = filetree_table.rows() - fs_entries = {} - # index entries on their inodes - for row in rows: - fs_entries[row.inode] = row + entries = {} + for row in db.table("tree").rows(): + entries[row.inode] = row vfs = VirtualFilesystem() - for entry in fs_entries.values(): - if entry.parent == 0: # Root entries do not require parent check - path = entry.name - else: - parts = [] - current = entry - while current.parent != 0: - parts.append(current.name) - current = fs_entries[current.parent] - parts.append(current.name) # appends the missing root parent - - path = "/".join(parts[::-1]) - if entry.type == 4: - fsentry = ProxmoxConfigDirectoryEntry(vfs, path, entry) - elif entry.type == 8: - fsentry = ProxmoxConfigFileEntry(vfs, path, entry) + for entry in entries.values(): + if entry.type == DT_DIR: + cls = ProxmoxConfigDirectoryEntry + elif entry.type == DT_REG: + cls = ProxmoxConfigFileEntry else: raise ValueError(f"Unknown pmxcfs file type: {entry.type}") - vfs.map_file_entry(path, fsentry) + parts = [] + current = entry + while current.parent != 0: + parts.append(current.name) + current = entries[current.parent] + parts.append(current.name) - return vfs + path = "/".join(parts[::-1]) + vfs.map_file_entry(path, cls(vfs, path, entry)) + + if hostname: + node_root = vfs.path(f"nodes/{hostname}") + vfs.symlink(str(node_root), "local") + vfs.symlink(str(node_root / "lxc"), "lxc") + vfs.symlink(str(node_root / "openvz"), "openvz") + vfs.symlink(str(node_root / "qemu-server"), "qemu-server") + + # TODO: .version, .members, .vmlist, maybe .clusterlog and .rrd? + + return vfs class ProxmoxConfigFileEntry(VirtualFile): def open(self) -> BinaryIO: - """Returns file handle (file-like object).""" - # if self.entry is not a directory, but a file - return BytesIO(self.entry.data or b"") - - def stat(self, follow_symlinks: bool = True) -> fsutil.stat_result: - """Return the stat information of this entry.""" - return self.lstat() + return BufferedStream(BytesIO(self.entry.data or b"")) def lstat(self) -> fsutil.stat_result: - """Return the stat information of the given path, without resolving links.""" # ['mode', 'addr', 'dev', 'nlink', 'uid', 'gid', 'size', 'atime', 'mtime', 'ctime'] return fsutil.stat_result( [ - stat.S_IFREG | 0o777, + stat.S_IFREG | 0o640, self.entry.inode, id(self.fs), 1, @@ -151,16 +123,12 @@ def __init__(self, fs: VirtualFilesystem, path: str, entry: sqlite3.Row): super().__init__(fs, path) self.entry = entry - def stat(self, follow_symlinks: bool = True) -> fsutil.stat_result: - """Return the stat information of this entry.""" - return self.lstat() - def lstat(self) -> fsutil.stat_result: """Return the stat information of the given path, without resolving links.""" # ['mode', 'addr', 'dev', 'nlink', 'uid', 'gid', 'size', 'atime', 'mtime', 'ctime'] return fsutil.stat_result( [ - stat.S_IFDIR | 0o777, + stat.S_IFDIR | 0o755, self.entry.inode, id(self.fs), 1, @@ -171,4 +139,4 @@ def lstat(self) -> fsutil.stat_result: self.entry.mtime, 0, ] - ) \ No newline at end of file + ) diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/vm.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/vm.py new file mode 100644 index 000000000..29c72b4c9 --- /dev/null +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/vm.py @@ -0,0 +1,29 @@ +from typing import Iterator + +from dissect.target.exceptions import UnsupportedPluginError +from dissect.target.helpers.record import TargetRecordDescriptor +from dissect.target.plugin import Plugin, export + +VirtualMachineRecord = TargetRecordDescriptor( + "proxmox/vm", + [ + ("string", "path"), + ], +) + + +class VirtualMachinePlugin(Plugin): + """Plugin to list Proxmox virtual machines.""" + + def check_compatible(self) -> None: + if self.target.os != "proxmox": + raise UnsupportedPluginError("Not a Proxmox operating system") + + @export(record=VirtualMachineRecord) + def vmlist(self) -> Iterator[VirtualMachineRecord]: + """List Proxmox virtual machines on this node.""" + for config in self.target.fs.path("/etc/pve/qemu-server").iterdir(): + yield VirtualMachineRecord( + path=config, + _target=self.target, + ) From 66dcefb71584c3e64c205456842587217b751dc2 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Mon, 26 Aug 2024 15:43:07 +0200 Subject: [PATCH 35/36] Use a layer instead of a mount for /dev --- dissect/target/plugins/os/unix/_os.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dissect/target/plugins/os/unix/_os.py b/dissect/target/plugins/os/unix/_os.py index 181472dbf..e6bfc98c1 100644 --- a/dissect/target/plugins/os/unix/_os.py +++ b/dissect/target/plugins/os/unix/_os.py @@ -6,7 +6,7 @@ from struct import unpack from typing import Iterator, Optional, Union -from dissect.target.filesystem import Filesystem, VirtualFilesystem +from dissect.target.filesystem import Filesystem from dissect.target.helpers.fsutil import TargetPath from dissect.target.helpers.record import UnixUserRecord from dissect.target.helpers.utils import parse_options_string @@ -250,13 +250,11 @@ def _add_devices(self) -> None: Currently only adds LVM devices. """ - vfs = VirtualFilesystem() + vfs = self.target.fs.append_layer() for volume in self.target.volumes: if volume.vs and volume.vs.__type__ == "lvm": - vfs.map_file_fh(f"{volume.raw.vg.name}/{volume.raw.name}", volume) - - self.target.fs.mount("/dev", vfs, ignore_existing=False) + vfs.map_file_fh(f"/dev/{volume.raw.vg.name}/{volume.raw.name}", volume) def _parse_os_release(self, glob: Optional[str] = None) -> dict[str, str]: """Parse files containing Unix version information. From c73987b12e44d4d9d3f01098097383b4745e50b9 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:10:10 +0200 Subject: [PATCH 36/36] Add unit test --- .../os/unix/linux/debian/proxmox/_os.py | 7 +- .../os/unix/linux/debian/proxmox/__init__.py | 0 .../os/unix/linux/debian/proxmox/test__os.py | 79 +++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 tests/plugins/os/unix/linux/debian/proxmox/__init__.py create mode 100644 tests/plugins/os/unix/linux/debian/proxmox/test__os.py diff --git a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py index 3ba60a863..8dccf6eed 100644 --- a/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py +++ b/dissect/target/plugins/os/unix/linux/debian/proxmox/_os.py @@ -32,10 +32,11 @@ def detect(cls, target: Target) -> Filesystem | None: def create(cls, target: Target, sysvol: Filesystem) -> ProxmoxPlugin: obj = super().create(target, sysvol) - with target.fs.path("/var/lib/pve-cluster/config.db").open("rb") as fh: - vfs = _create_pmxcfs(fh, obj.hostname) + if (config_db := target.fs.path("/var/lib/pve-cluster/config.db")).exists(): + with config_db.open("rb") as fh: + vfs = _create_pmxcfs(fh, obj.hostname) - target.fs.mount("/etc/pve", vfs) + target.fs.mount("/etc/pve", vfs) return obj diff --git a/tests/plugins/os/unix/linux/debian/proxmox/__init__.py b/tests/plugins/os/unix/linux/debian/proxmox/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/plugins/os/unix/linux/debian/proxmox/test__os.py b/tests/plugins/os/unix/linux/debian/proxmox/test__os.py new file mode 100644 index 000000000..07abfe9ec --- /dev/null +++ b/tests/plugins/os/unix/linux/debian/proxmox/test__os.py @@ -0,0 +1,79 @@ +from io import BytesIO + +from dissect.target.filesystem import VirtualFilesystem +from dissect.target.plugins.os.unix.linux.debian.proxmox._os import ProxmoxPlugin +from dissect.target.target import Target +from tests._utils import absolute_path + + +def test_proxmox_os(target_bare: Target) -> None: + fs = VirtualFilesystem() + + fs.map_file_fh("/etc/hostname", BytesIO(b"pve")) + fs.map_file( + "/var/lib/pve-cluster/config.db", absolute_path("_data/plugins/os/unix/linux/debian/proxmox/_os/config.db") + ) + fs.makedirs("/etc/pve") + fs.makedirs("/var/lib/pve") + + target_bare.filesystems.add(fs) + + assert ProxmoxPlugin.detect(target_bare) + target_bare._os_plugin = ProxmoxPlugin + target_bare.apply() + + assert target_bare.os == "proxmox" + assert sorted(list(map(str, target_bare.fs.path("/etc/pve").rglob("*")))) == [ + "/etc/pve/__version__", + "/etc/pve/authkey.pub", + "/etc/pve/authkey.pub.old", + "/etc/pve/corosync.conf", + "/etc/pve/datacenter.cfg", + "/etc/pve/firewall", + "/etc/pve/ha", + "/etc/pve/local", + "/etc/pve/lxc", + "/etc/pve/mapping", + "/etc/pve/nodes", + "/etc/pve/nodes/pve", + "/etc/pve/nodes/pve-btrfs", + "/etc/pve/nodes/pve-btrfs/lrm_status", + "/etc/pve/nodes/pve-btrfs/lrm_status.tmp.971", + "/etc/pve/nodes/pve-btrfs/lxc", + "/etc/pve/nodes/pve-btrfs/openvz", + "/etc/pve/nodes/pve-btrfs/priv", + "/etc/pve/nodes/pve-btrfs/pve-ssl.key", + "/etc/pve/nodes/pve-btrfs/pve-ssl.pem", + "/etc/pve/nodes/pve-btrfs/qemu-server", + "/etc/pve/nodes/pve-btrfs/ssh_known_hosts", + "/etc/pve/nodes/pve/lrm_status", + "/etc/pve/nodes/pve/lxc", + "/etc/pve/nodes/pve/openvz", + "/etc/pve/nodes/pve/priv", + "/etc/pve/nodes/pve/pve-ssl.key", + "/etc/pve/nodes/pve/pve-ssl.pem", + "/etc/pve/nodes/pve/qemu-server", + "/etc/pve/nodes/pve/qemu-server/100.conf", + "/etc/pve/nodes/pve/ssh_known_hosts", + "/etc/pve/openvz", + "/etc/pve/priv", + "/etc/pve/priv/acme", + "/etc/pve/priv/authkey.key", + "/etc/pve/priv/authorized_keys", + "/etc/pve/priv/known_hosts", + "/etc/pve/priv/lock", + "/etc/pve/priv/pve-root-ca.key", + "/etc/pve/priv/pve-root-ca.srl", + "/etc/pve/pve-root-ca.pem", + "/etc/pve/pve-www.key", + "/etc/pve/qemu-server", + "/etc/pve/sdn", + "/etc/pve/storage.cfg", + "/etc/pve/user.cfg", + "/etc/pve/virtual-guest", + "/etc/pve/vzdump.cron", + ] + + vmlist = list(target_bare.vmlist()) + assert len(vmlist) == 1 + assert vmlist[0].path == "/etc/pve/qemu-server/100.conf"