🗓 Feb 27, 2024 7:21:35 PM

🐙 merge argument in for_each
✨ register method
✨ get/set_register methods
📔 docs added/updated
🤖 types added/updated
🔥 rename substituion to sub
✨ addition method
✨ substitution method
🐙 regex_search works with bytes also
🧪 tests added/updated

chepy/core.py

@@ -96,6 +96,8 @@ def __init__(self, *data):
         self.read_file = self.load_file
         #: Holds all the methods that are called/chained and their args
         self._stack = list()
+        #: Holds register values
+        self._registers = dict()
 
         #: Log level
         self.log_level = logging.INFO
@@ -296,7 +298,11 @@ def fork(self, methods: List[Tuple[Union[str, object], dict]]):
                     self.states[i] = getattr(self, method_name)().o
         return self
 
-    def for_each(self, methods: List[Tuple[Union[str, object], dict]]):
+    def for_each(
+        self,
+        methods: List[Tuple[Union[str, object], dict]],
+        merge: Union[str, bytes, None] = None,
+    ):
         """Run multiple methods on current state if it is a list
 
         Method names in a list of tuples. If using in the cli,
@@ -305,6 +311,7 @@ def for_each(self, methods: List[Tuple[Union[str, object], dict]]):
         Args:
             methods (List[Tuple[Union[str, object], dict]]): Required.
                 List of tuples
+            merge (Union[str, bytes, None]): Merge data with. Defaults to None
 
         Returns:
             Chepy: The Chepy object.
@@ -340,7 +347,17 @@ def for_each(self, methods: List[Tuple[Union[str, object], dict]]):
                     ).o  # pragma: no cover
                 else:
                     hold[i] = getattr(self, method_name)().o
-        self.state = hold
+        if merge is not None:
+            if len(hold) > 0:
+                merge = self._to_bytes(merge)
+                if isinstance(hold[0], str):  # pragma: no cover
+                    self.state = merge.decode().join(hold)
+                elif isinstance(hold[0], bytes):
+                    self.state = merge.join(hold)
+                else:  # pragma: no cover
+                    self.state = hold
+        else:
+            self.state = hold
         return self
 
     @ChepyDecorators.call_stack
@@ -686,7 +703,7 @@ def _convert_to_int(self) -> int:
             raise NotImplementedError
 
     def _get_nested_value(self, data, key, split_by="."):
-        """Get a dict value based on a string key with dot notation. Supports array indexing. 
+        """Get a dict value based on a string key with dot notation. Supports array indexing.
         If split_by is None or "", returns only the first key
 
         Args:
@@ -1451,3 +1468,95 @@ def cb(data):
             return self
         self.state = callback_function(self.state)
         return self
+
+    def register(
+        self,
+        pattern: Union[str, bytes],
+        ignore_case: bool = False,
+        multiline: bool = False,
+        dotall: bool = False,
+        unicode: bool = False,
+        extended: bool = False,
+    ):
+        """Extract data from the input and store it in registers. Regular expression capture groups are used to select the data to extract.
+
+        Args:
+            pattern (Union[str, bytes]): Required. The regex pattern to search by
+            ignore_case (bool, optional): Set case insensitive flag. Defaults to False.
+            multiline (bool, optional): ^/$ match start/end. Defaults to False.
+            dotall (bool, optional): `.` matches newline. Defaults to False.
+            unicode (bool, optional): Match unicode characters. Defaults to False.
+            extended (bool, optional): Ignore whitespace. Defaults to False.
+
+        Returns:
+            Chepy: The Chepy object.
+
+        Examples:
+            >>> c = Chepy("hello world")
+            >>> c.register("(hello)\s(world)")
+            >>> c._registers
+            {'$R0': 'hello', '$R1': 'world'}
+        """
+        # regex flags
+        flags = 0
+        if ignore_case:
+            flags += re.IGNORECASE
+        if multiline:
+            flags += re.MULTILINE
+        if dotall:
+            flags += re.DOTALL
+        if unicode:
+            flags += re.UNICODE
+        if extended:  # pragma: no cover
+            flags += re.X
+
+        r = re.compile(pattern, flags=flags)
+        old_state = self.state
+
+        if isinstance(pattern, bytes):
+            matches = r.findall(self._convert_to_bytes())
+        else:
+            matches = r.findall(self._convert_to_str())
+
+        # there are matches which is a list of tuples
+        if len(matches) > 0:
+            # if there is only one match, it will be a list. else, it is a list with a tuple
+            if isinstance(matches[0], tuple):  # pragma: no cover
+                matches = matches[0]
+            for i in range(len(matches)):
+                # only add non empty
+                if matches[i]:
+                    self._registers[f"$R{i}"] = matches[i]
+
+        self.state = old_state
+        return self
+
+    def get_register(self, key: str) -> Union[str, bytes]:
+        """Get a value from registers by key
+
+        Args:
+            key (str): Key
+
+        Raises:
+            ValueError: If key does not exist
+
+        Returns:
+            Union[str, bytes]: Value of register
+        """
+        v = self._registers.get(key)
+        if v is None:  # pragma: no cover
+            raise ValueError("Key not found in registers")
+        return v
+
+    def set_register(self, key: str, val: Union[str, bytes]):
+        """Set the value of a register
+
+        Args:
+            key (str): Key
+            val (Union[str, bytes]): Value
+
+        Returns:
+            Chepy: The Chepy object.
+        """
+        self._registers[key] = val
+        return self

chepy/core.pyi

@@ -20,6 +20,7 @@ class ChepyCore:
     read_file: Any = ...
     log_level: Any = ...
     log_format: str = ...
+    _registers: Dict[str, Union[str, bytes]] = ...
     _log: logging.Logger = ...
     def __init__(self, *data: Any) -> None: ...
     def _convert_to_bytes(self) -> bytes: ...
@@ -35,7 +36,7 @@ class ChepyCore:
     @state.setter
     def state(self: ChepyCoreT, val: Any) -> None: ...
     def fork(self: ChepyCoreT, methods: List[Union[Tuple[Union[str, Callable], dict], Tuple[Union[str, Callable],]]]) -> ChepyCoreT: ...
-    def for_each(self: ChepyCoreT, methods: List[Union[Tuple[Union[str, Callable], dict], Tuple[Union[str, Callable],]]]) -> ChepyCoreT: ...
+    def for_each(self: ChepyCoreT, methods: List[Union[Tuple[Union[str, Callable], dict], Tuple[Union[str, Callable],]]], merge: Union[str, bytes, None]=None) -> ChepyCoreT: ...
     def set_state(self: ChepyCoreT, data: Any) -> ChepyCoreT: ...
     def create_state(self: ChepyCoreT): ...
     def copy_state(self: ChepyCoreT, index: int=...) -> ChepyCoreT: ...
@@ -80,3 +81,6 @@ class ChepyCore:
     def set_plugin_path(self: ChepyCoreT, path: str) -> None: ...
     def subsection(self: ChepyCoreT, pattern: str, methods: List[Tuple[Union[str, object], dict]], group: int=...) -> ChepyCoreT: ...
     def callback(self: ChepyCoreT, callback_function: Callable[[Any], Any]) -> ChepyCoreT: ...
+    def register(self: ChepyCoreT, pattern: Union[str, bytes], ignore_case: bool=False, multiline: bool=False, dotall: bool=False, unicode: bool=False, extended: bool=False) -> ChepyCoreT: ...
+    def get_register(self: ChepyCoreT, key: str) -> Union[str, bytes]: ...
+    def set_register(self, key: str, val: Union[str, bytes]) -> ChepyCoreT: ...

chepy/modules/aritmeticlogic.py

@@ -1,9 +1,11 @@
 import binascii
 import statistics
 from typing import TypeVar, Union
+from functools import reduce as functools_reduce
 
 from ..core import ChepyCore, ChepyDecorators
 from .exceptions import StateNotList
+from .internal.helpers import detect_delimiter
 
 
 AritmeticLogicT = TypeVar("AritmeticLogicT", bound="AritmeticLogic")
@@ -70,8 +72,33 @@ def add(self, n: int) -> AritmeticLogicT:
         return self
 
     @ChepyDecorators.call_stack
-    def subtract(self, n: int) -> AritmeticLogicT:
-        """Subtract a number to the state
+    def addition(self, delimiter=None) -> AritmeticLogicT:
+        """Adds a list of numbers. If an item in the string is not a number it is excluded from the list.
+
+        Args:
+            delimiter (str, optional): Delimiter. Defaults to None.
+
+        Returns:
+            Chepy: The Chepy object.
+        """
+        data = self._convert_to_str()
+        print('🟢 ', data)
+        if not delimiter:
+            delimiter = detect_delimiter(data)
+        # only work on numbers
+        nums = []
+        for n in data.split(delimiter):
+            try:
+                nums.append(int(n))
+            except:  # noqa: E722
+                continue
+
+        self.state = functools_reduce(lambda x, y: x + y, nums)
+        return self
+
+    @ChepyDecorators.call_stack
+    def sub(self, n: int) -> AritmeticLogicT:
+        """SUB the input with the given key
 
         Args:
             n (int): Number to subtract with
@@ -104,6 +131,30 @@ def subtract(self, n: int) -> AritmeticLogicT:
         self.state = hold
         return self
 
+    @ChepyDecorators.call_stack
+    def subtract(self, delimiter=None) -> AritmeticLogicT:
+        """Subtracts a list of numbers. If an item in the string is not a number it is excluded from the list.
+
+        Args:
+            delimiter (str, optional): Delimiter. Defaults to None.
+
+        Returns:
+            Chepy: The Chepy object.
+        """
+        data = self._convert_to_str()
+        if not delimiter:
+            delimiter = detect_delimiter(data)
+        # only work on numbers
+        nums = []
+        for n in data.split(delimiter):
+            try:
+                nums.append(int(n))
+            except:  # noqa: E722
+                continue
+
+        self.state = functools_reduce(lambda x, y: x - y, nums)
+        return self
+
     @ChepyDecorators.call_stack
     def multiply(self, n: int) -> AritmeticLogicT:
         """Multiply a number to the state

chepy/modules/aritmeticlogic.pyi

@@ -8,7 +8,9 @@ class AritmeticLogic(ChepyCore):
     state: Any = ...
     def str_bit_shift_right(self: AritmeticLogicT, amount: int) -> AritmeticLogicT: ...
     def add(self: AritmeticLogicT, n: Union[int, str]) -> AritmeticLogicT: ...
-    def subtract(self: AritmeticLogicT, n: Union[int, str]) -> AritmeticLogicT: ...
+    def addition(self: AritmeticLogicT, delimiter: Union[str, None]=None) -> AritmeticLogicT: ...
+    def sub(self: AritmeticLogicT, n: Union[int, str]) -> AritmeticLogicT: ...
+    def subtract(self: AritmeticLogicT, delimiter: Union[str, None]=None) -> AritmeticLogicT: ...
     def multiply(self: AritmeticLogicT, n: int) -> AritmeticLogicT: ...
     def divide(self: AritmeticLogicT, n: int) -> AritmeticLogicT: ...
     def power(self: AritmeticLogicT, n: int) -> AritmeticLogicT: ...

chepy/modules/dataformat.py

@@ -386,7 +386,7 @@ def from_base91(self) -> DataFormatT:  # pragma: no cover
         n = 0
         out = bytearray()
         for strletter in encoded_str:
-            if not strletter in decode_table:
+            if strletter not in decode_table:
                 continue
             c = decode_table[strletter]
             if v < 0:
@@ -960,7 +960,7 @@ def to_binary(
         """
         hold = []
         join_by = self._str_to_bytes(join_by)
-        out = list(format(s, "08b").encode() for s in list(self._convert_to_bytes()))
+        # out = list(format(s, "08b").encode() for s in list(self._convert_to_bytes()))
         for s in list(self._convert_to_bytes()):
             hold.append(str(bin(s)[2:].zfill(byte_length)).encode())
         self.state = join_by.join(hold)
@@ -1434,7 +1434,6 @@ def bruteforce_from_base_xx(self) -> DataFormatT:
             "base16": base64.b16decode,
             "base32": base64.b32decode,
             "base64": base64.b64decode,
-            "base85": base64.b85decode,
             "base58": base58.b58decode,
         }
         data = self._convert_to_bytes()

chepy/modules/utils.py

@@ -118,6 +118,7 @@ def remove_nullbytes(self) -> UtilsT:
     def regex_search(
         self,
         pattern: str,
+        is_bytes: bool = False,
         ignore_case: bool = False,
         multiline: bool = False,
         dotall: bool = False,
@@ -128,6 +129,7 @@ def regex_search(
 
         Args:
             pattern (str): Required. The regex pattern to search by
+            is_bytes (bool, optional): Treat the pattern and state as bytes. Defaults to False.
             ignore_case (bool, optional): Set case insensitive flag. Defaults to False.
             multiline (bool, optional): ^/$ match start/end. Defaults to False.
             dotall (bool, optional): `.` matches newline. Defaults to False.
@@ -152,7 +154,10 @@ def regex_search(
             flags += re.UNICODE
         if extended:
             flags += re.X
-        self.state = re.findall(pattern, self._convert_to_str(), flags=flags)
+        if is_bytes:
+            self.state = re.findall(self._to_bytes(pattern), self._convert_to_bytes(), flags=flags)
+        else:
+            self.state = re.findall(pattern, self._convert_to_str(), flags=flags)
         return self
 
     @ChepyDecorators.call_stack

chepy/modules/utils.pyi

@@ -10,7 +10,7 @@ class Utils(ChepyCore):
     def count_occurances(self: UtilsT, regex: str, case_sensitive: bool=...) -> UtilsT: ...
     def remove_whitespace(self: UtilsT, spaces: bool=..., carriage_return: bool=..., line_feeds: bool=..., tabs: bool=..., form_feeds: bool=...) -> UtilsT: ...
     def remove_nullbytes(self: UtilsT) -> UtilsT: ...
-    def regex_search(self: UtilsT, pattern: str, ignore_case: bool=..., multiline: bool=..., dotall: bool=..., unicode: bool=..., extended: bool=...) -> UtilsT: ...
+    def regex_search(self: UtilsT, pattern: str, is_bytes: bool=False, ignore_case: bool=False, multiline: bool=False, dotall: bool=False, unicode: bool=False, extended: bool=False) -> UtilsT: ...
     def split_by_char(self: UtilsT, delimiter: str=...) -> UtilsT: ...
     def split_by_regex(self: UtilsT, pattern: str=..., trim: Any=...) -> UtilsT: ...
     def split_by_n(self: UtilsT, n: int) -> UtilsT: ...

docs/examples.md

@@ -8,7 +8,7 @@ We are given a file called encoding which contains the following string.
 =0GDAqREMS0EEWHGOEHJWWRAZqGDHAyHDSxDWyHImRHEEcIGOAyZMqHDPyHFScIDUSyDASREMS0EYS0AWEyJMyRARyHDQWGGUSxJAqREEy1EIWHFOAIDXqHHP1HDRyIDUSyHQySFPImEGW0AOAHHWqHIPy0GHcIDSAyJAS0DEEmEGWHGOEHJOqHHP1HIOMwDVIREFySFESHEMcIGOAxZMqHDP5HFRSIDSIyHASREMS0EEWHGOExEEOSJn5HGSMwDSSxJBqREEEmEHWHFAExZOIHDX1HERyIDUSyDASREMyxD0DxJUE0DFOIDPyHFRAGDSEyJAS0DlR1EOWHFYIIJOqHHP1HDRyIDUgHD3ZSEP50E0DHFOASAAqHDF1HFRSIGUEyDWS0DPM0EEWHGOEHJOqHHFAHJLAyDVIyD3R0DEI1EKWHFEEyJOIHIn1HDQSIDVWyDASREMS0EEWHGISxAnyxHP9HJVSIDSqyDBS0HM10EOW0GUEHHOIxIX1HDRyIDUSyDASRETSSHFWyGUIxAPIHDX10ERSIJUEyDWqRElRHEOWIGQEHJOqHHP1HDRyIFPEQGFq0EQWSHOWHFYExZOIRIF5HDQWGHUSxDW1HEMS0EEWHGOEHJOq0FOqGIHcIHCEQEWS0HO50EOcIGUEHHEqRJPyHDGWxDUSyDASREMS0EEW1DMuSJP9RIQqGDQSIJWqyDWSRImRHEHcxGOAHHSuHHP1HDRyIDUSyDAIID24RHUAxIMuHHOI0Dl4HDQAGHUSxDBgREESHEKWHGOEHJOqHHP1HDRMHHDE0FmHRF2VHEOcIGWEHHEy0IPyHEHAGDSSxJASREMS0EEWHGOEHJWWRAmZGFLcxHDSxDW1HEmRHEIcyGOAyJIqHDPyHDRyIDUSyDASREMS0E
 ```
 
-#### Script
+# Script
 We can script the solution using the following python script:
 ```python
 from chepy import Chepy
@@ -28,7 +28,7 @@ StormCTF{Spot3:DcEC6181F48e3B9D3dF77Dd827BF34e0}
 
 #### Recipe 3: From charCode
 
-```md
+```py
 from chepy import Chepy
 
 c = Chepy(data)
@@ -41,7 +41,7 @@ print(c)
 ```
 
 #### Recipe 5: Google ei timestamp
-```md
+```py
 from chepy import Chepy
 
 data = 'X9guXK1UxvTxBYLJiYAD'
@@ -57,10 +57,29 @@ c.from_unix_timestamp()
 print(c.o)
 ```
 
+## Advanced usage 1
+Partial Chepy implementation of the following advanced usage documented at https://embee-research.ghost.io/advanced-cyberchef-operations-netsupport/
+
+```py
+from chepy import Chepy
+
+c = Chepy('/tmp/malware/netsupport_loader.vbs.txt').load_file()
+
+c.register('\w+ = (\d+)')
+
+c.search('\(([\d,]{1000,})\)')
+c.get_by_index(0)
+c.find_replace('\d{3}', '\g<0> '+c.get_register('$R0'))
+c.split_by_char(',')
+c.for_each([(c.subtract,), (c.from_decimal,)], '')
+
+print(c)
+```
+
 # CTF
 
 #### HTB Business 2022 - Perseverence
-```md
+```py
 import base64
 from chepy import Chepy