🗓 Oct 30, 2023 10:24:47 PM

🐙 plugins updated
✨ to/fom_utf21
🧪 tests added/updated
✨ new helpers

chepy/modules/dataformat.py

@@ -11,7 +11,7 @@
 import itertools
 from random import randint
 from .internal.constants import Encoding
-from .internal.helpers import detect_delimiter, Rotate
+from .internal.helpers import detect_delimiter, Rotate, Uint1Array
 
 yaml = lazy_import.lazy_module("yaml")
 import regex as re
@@ -1982,3 +1982,47 @@ def flatten(self) -> DataFormatT:
             return self
         except:
             return self
+
+    @ChepyDecorators.call_stack
+    def to_utf21(self) -> DataFormatT:
+        """Convert to UTF-21
+
+        Returns:
+            Chepy: The Chepy object.
+        """
+        data = self._convert_to_bytes()
+        bits = Uint1Array(len(data) * 21)
+
+        bit_index = 0
+        for codepoint in data:
+            for i in range(20, -1, -1):
+                bits.set(bit_index, (codepoint & (1 << i)) >> i)
+                bit_index += 1
+
+        self.state = bytes(bits.buffer)
+        return self
+
+    @ChepyDecorators.call_stack
+    def from_utf21(self) -> DataFormatT:
+        """Convert from UTF-21
+
+        Returns:
+            Chepy: The Chepy object.
+        """
+        buffer = self._convert_to_bytes()
+        bits = Uint1Array(buffer)
+        codepoint_length = (len(buffer) * 8) // 21
+        codepoints = []
+
+        for codepoint_index in range(codepoint_length):
+            start_bit_index = codepoint_index * 21
+            codepoint = 0
+
+            for i in range(21):
+                bit = bits.get(start_bit_index + i)
+                codepoint += bit << (20 - i)
+
+            codepoints.append(codepoint)
+
+        self.state = bytes(codepoints)
+        return self

chepy/modules/dataformat.pyi

@@ -106,3 +106,5 @@ class DataFormat(ChepyCore):
     def from_base62(self: DataFormatT, alphabet: str="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") -> DataFormatT: ...
     def cut(self: DataFormatT, start: int, end: int) -> DataFormatT: ...
     def flatten(self: DataFormatT) -> DataFormatT: ...
+    def to_utf21(self: DataFormatT) -> DataFormatT: ...
+    def from_utf21(self: DataFormatT) -> DataFormatT: ...

chepy/modules/internal/helpers.py

@@ -1,6 +1,36 @@
 from typing import List, Union
 
 
+class Uint1Array:
+    # reference https://git.sr.ht/~evanhahn/UTF-21.js
+    def __init__(self, bit_length_or_buffer):
+        if isinstance(bit_length_or_buffer, int):
+            bit_length = bit_length_or_buffer
+            self.bit_length = bit_length
+            self.bytes = bytearray((bit_length + 7) // 8)
+        else:
+            buffer = bit_length_or_buffer
+            self.bit_length = len(buffer) * 8
+            self.bytes = bytearray(buffer)
+
+    @property
+    def buffer(self):
+        return bytes(self.bytes)
+
+    def get(self, index):
+        byte_index = index // 8
+        bit_index = index % 8
+        byte = self.bytes[byte_index]
+        return (byte >> (7 - bit_index)) & 1
+
+    def set(self, index, value):
+        byte_index = index // 8
+        bit_index = index % 8
+        old_byte = self.bytes[byte_index]
+        new_byte = old_byte | (value << (7 - bit_index))
+        self.bytes[byte_index] = new_byte
+
+
 def detect_delimiter(
     data: Union[str, bytes],
     delimiters: List[Union[str, bytes]] = [

tests/test_dataformat.py

@@ -638,3 +638,89 @@ def test_cut():
 def test_flatten():
     assert Chepy(["a", 1, True]).flatten().o == ["a", 1, True]
     assert Chepy([["a"], [1], [True]]).flatten().o == ["a", 1, True]
+
+
+def test_utf21():
+    flag = [
+        0,
+        2,
+        168,
+        0,
+        17,
+        0,
+        0,
+        134,
+        0,
+        5,
+        64,
+        0,
+        35,
+        0,
+        1,
+        236,
+        0,
+        6,
+        224,
+        0,
+        119,
+        0,
+        1,
+        152,
+        0,
+        27,
+        128,
+        0,
+        232,
+        0,
+        7,
+        144,
+        0,
+        47,
+        128,
+        0,
+        192,
+        0,
+        13,
+        192,
+        0,
+        51,
+        0,
+        1,
+        248,
+        0,
+        23,
+        192,
+        0,
+        178,
+        0,
+        3,
+        0,
+        0,
+        58,
+        128,
+        1,
+        124,
+        0,
+        6,
+        160,
+        0,
+        55,
+        0,
+        3,
+        168,
+        0,
+        28,
+        0,
+        0,
+        98,
+        0,
+        6,
+        64,
+        0,
+        16,
+        128,
+        1,
+        244,
+    ]
+    assert Chepy(bytes(flag)).from_utf21().o == b"UDCTF{7w3nty_0n3?_Y0u_57up1d!}"
+    assert Chepy("UDCTF{7w3nty_0n3?_Y0u_57up1d!}").to_utf21().o == bytes(flag)

tests_plugins/test_ml.py

@@ -17,10 +17,19 @@ def test_ml_detect():
     )
 
 
-# def test_ml_magic():
-#     assert (
-#         Chepy("Ca6wMuk9H4Y3rYb8uMQordMrH6JbvsaWx2Ua7dNQvF1tujWPvi2AEijEKsD6Mpe7Ld37T1y")
-#         .ml_magic()
-#         .o[-1]["out"]
-#         == b"hello world"
-#     )
+def test_ml_magic():
+    assert (
+        Chepy("Ca6wMuk9H4Y3rYb8uMQordMrH6JbvsaWx2Ua7dNQvF1tujWPvi2AEijEKsD6Mpe7Ld37T1y")
+        .ml_magic()
+        .o[-1]
+        == b"hello world"
+    )
+
+    assert (
+        Chepy(
+            "KZWTCNCVGFEXSUTOJJHFM3LIKVMWWZDPKZLFS522I44VOYSHPB5FSVLDGVLGWMKXKNWGYWSNI5UFAVSHPBQWGMKOONREM4COKJWHATKXK52GCWKWLJLVE3SGLBREQQTTLFMHAV2NNRNHCU3KKJMGEVS2JFKWY2DXMFDEU42XNM4VMYSYKIZVMMC2MFJDC4CFKVWXATSSIZNDMV2WKJHWIMLMK5JW4SSUMJDEUV2ZK52EOTJROBCVE3DQNRLFIRS2K5VWI43BI5LHEV3MNBLVEM3IOZKWURTLKIYXARTBIZNGSUSWOBMFOVSSI5SDERSHMEZWIWDCLBJFSVTKIEYVEMKZPFGVIUTIKIYVUMC2KVLGWVRSJJDWGRSSK5QWWWT2KZVEM22XK5DEQYKFGVHGERLQMFLDCWSTKMZEK52NJBUFOYJSKJKFM23EGRKWYWSXKVMGIUCVKQYDS==="
+        )
+        .ml_magic(10)
+        .o[-1]
+        == b"InfoSeCon2023{1af5856c70878f8566085bc13849ef4d}"
+    )