From aae8022cb90cd3567a622db85a222374f7ff0816 Mon Sep 17 00:00:00 2001 From: securisec Date: Tue, 2 Jul 2024 23:59:57 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=93=20Jul=202,=202024=2011:59:14?= =?UTF-8?q?=E2=80=AFPM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💚 build steps added/updated ✨ aws_account_id_from_access_key ✨ expand_alpha_range --- .github/workflows/tests_multi_os.yml | 4 +-- Makefile | 4 +-- chepy/modules/extractors.py | 17 ++++++++++ chepy/modules/extractors.pyi | 1 + chepy/modules/utils.py | 50 ++++++++++++++++++++++++++++ chepy/modules/utils.pyi | 1 + tests/test_extractors.py | 14 +++++--- tests/test_utils.py | 43 ++++++++++++++++++------ 8 files changed, 114 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests_multi_os.yml b/.github/workflows/tests_multi_os.yml index e6f192e..f7bf3c7 100644 --- a/.github/workflows/tests_multi_os.yml +++ b/.github/workflows/tests_multi_os.yml @@ -56,11 +56,11 @@ jobs: - name: Install test requirements run: | - pip install sphinx recommonmark pytest==8.1.1 pytest-cov pyperclip + pip install sphinx recommonmark pytest==8.1.1 pytest-cov==5.0.0 pyperclip - name: Test with pytest run: | - pytest -v --disable-pytest-warnings --cov-report=xml --cov=chepy --cov-config=.coveragerc tests/ + COVERAGE_CORE=sysmon pytest -v --disable-pytest-warnings --cov-report=xml --cov=chepy --cov-config=.coveragerc tests/ coverage report -m - name: Test plugins osx diff --git a/Makefile b/Makefile index 46f216c..cf034ad 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ test: - python -m pytest --noconftest -v --disable-pytest-warnings --cov-report=xml --cov=chepy --cov-config=.coveragerc tests/ + COVERAGE_CORE=sysmon python -m pytest --noconftest -v --disable-pytest-warnings --cov-report=xml --cov=chepy --cov-config=.coveragerc tests/ test-all: test - python -m pytest --noconftest -v --disable-pytest-warnings tests_plugins/ + COVERAGE_CORE=sysmon python -m pytest --noconftest -v --disable-pytest-warnings tests_plugins/ # git log --format=%B 4.0.0..5.0.0 | sed '/^\s*$/d' | sort | uniq \ No newline at end of file diff --git a/chepy/modules/extractors.py b/chepy/modules/extractors.py index 084e163..27082f5 100644 --- a/chepy/modules/extractors.py +++ b/chepy/modules/extractors.py @@ -1,5 +1,6 @@ import math from binascii import unhexlify +import base64 from typing import TypeVar, Union, List from urllib.parse import urlparse as _pyurlparse import lazy_import @@ -684,3 +685,19 @@ def extract_html_comments(self): filter(lambda x: x != "", self._parsel_obj().xpath("//comment()").getall()) ) return self + + @ChepyDecorators.call_stack + def aws_account_id_from_access_key(self): + """Extract AWS account id from access key + + Returns: + Chepy: The Chepy object. + """ + trimmed_AWSKeyID = self._convert_to_str()[4:] + x = base64.b32decode(trimmed_AWSKeyID) + y = x[0:6] + z = int.from_bytes(y, byteorder='big', signed=False) + mask = int.from_bytes(unhexlify(b'7fffffffff80'), byteorder='big', signed=False) + + self.state = (z & mask)>>7 + return self diff --git a/chepy/modules/extractors.pyi b/chepy/modules/extractors.pyi index 2c03796..7256e4f 100644 --- a/chepy/modules/extractors.pyi +++ b/chepy/modules/extractors.pyi @@ -44,3 +44,4 @@ class Extractors(ChepyCore): def css_selector(self: ExtractorsT, query: str) -> ExtractorsT: ... def extract_html_comments(self: ExtractorsT) -> ExtractorsT: ... def extract_html_tags(self: ExtractorsT, tag: List[str]) -> ExtractorsT: ... + def aws_account_id_from_access_key(self: ExtractorsT) -> ExtractorsT: ... diff --git a/chepy/modules/utils.py b/chepy/modules/utils.py index ce3edd1..ed881b3 100644 --- a/chepy/modules/utils.py +++ b/chepy/modules/utils.py @@ -819,3 +819,53 @@ def pick(self, *values: Any): else: # pragma: no cover raise TypeError("Input should be a list, dictionary, string, or bytes") return self + + @ChepyDecorators.call_stack + def expand_alpha_range(self, join_by: Union[str, None] = None): + """Get all alphanumberic or hex chars for the specified range + + Args: + join_by (str, optional): Join by. Defaults to Union[str, None]. + + Returns: + Chepy: The Chepy object. + """ + alph_str = self._convert_to_str() + hold = [] + + def expand_range(start, end): + return [str(x) for x in range(int(start), int(end) + 1)] + + def expand_char_range(start, end): + return [chr(x) for x in range(ord(start), ord(end) + 1)] + + hold = [] + i = 0 + length = len(alph_str) + + while i < length: + # Match numerical ranges like 10-20 + num_match = re.match(r"(\d+)-(\d+)", alph_str[i:]) + if num_match: + start, end = num_match.groups() + hold.extend(expand_range(start, end)) + i += len(start) + len(end) + 1 # move past the number range + elif i < length - 2 and alph_str[i + 1] == "-" and alph_str[i] != "\\": + # Handle character ranges like a-z + start = alph_str[i] + end = alph_str[i + 2] + hold.extend(expand_char_range(start, end)) + i += 2 + elif ( + i < length - 2 and alph_str[i] == "\\" and alph_str[i + 1] == "-" + ): # pragma: no cover + hold.append("-") + i += 1 + else: + hold.append(alph_str[i]) + i += 1 + + if join_by is not None: + hold = join_by.join(hold) + self.state = hold + return self diff --git a/chepy/modules/utils.pyi b/chepy/modules/utils.pyi index c00ec4c..c1ab35d 100644 --- a/chepy/modules/utils.pyi +++ b/chepy/modules/utils.pyi @@ -41,3 +41,4 @@ class Utils(ChepyCore): def drop_bytes(self: UtilsT, start: int, length: int) -> UtilsT: ... def without(self: UtilsT, *values: Any) -> UtilsT: ... def pick(self: UtilsT, *values: Any) -> UtilsT: ... + def expand_alpha_range(self, join_by: Union[str, None]=None) -> UtilsT: ... diff --git a/tests/test_extractors.py b/tests/test_extractors.py index 2d10f39..0c6a9bf 100644 --- a/tests/test_extractors.py +++ b/tests/test_extractors.py @@ -239,9 +239,7 @@ def test_find_continuous_patterns(): def test_zero_with_chars_tags(): assert ( - Chepy("this 󠁮󠁩󠁣is 󠁣󠁻󠀰just 󠁲󠁟󠀱a 󠀵󠁟󠀱simple 󠀷󠁽text file") - .extract_zero_width_chars_tags() - .o + Chepy("this 󠁮󠁩󠁣is 󠁣󠁻󠀰just 󠁲󠁟󠀱a 󠀵󠁟󠀱simple 󠀷󠁽text file").extract_zero_width_chars_tags().o == b"nicc{0r_15_17}" ) @@ -252,7 +250,7 @@ def test_decode_zero_width(): "e2808be2808be2808be2808befbbbfe280ace2808b68656c6c6fe2808be2808be2808be2808befbbbfe2808be2808ce2808be2808be2808be2808be280acefbbbfefbbbfe2808be2808be2808be2808befbbbfe2808defbbbfe2808be2808be2808be2808befbbbfe2808be2808ce2808be2808be2808be2808befbbbfe280ace2808c" ) .from_hex() - .decode_zero_width("\u200B\u200c\u200d\u202c\ufeff") + .decode_zero_width("\u200b\u200c\u200d\u202c\ufeff") .o["hidden"] == "secret" ) @@ -271,7 +269,7 @@ def test_decode_zero_width(): ) .from_hex() .decode_zero_width( - "\u200B\u200C\u200D\u200E\u202A\u202C\u202D\u2062\u2063\ufeff" + "\u200b\u200c\u200d\u200e\u202a\u202c\u202d\u2062\u2063\ufeff" ) .o["hidden"] == "secret" @@ -296,3 +294,9 @@ def test_html_tags(): def test_html_comments(): c = Chepy("tests/files/test.html").load_file().extract_html_comments() assert len(c.o) == 3 + + +def test_aws_access_key_from_account_id(): + assert ( + Chepy("ASIAQNZGKIQY56JQ7WML").aws_account_id_from_access_key().o == 29608264753 + ) diff --git a/tests/test_utils.py b/tests/test_utils.py index b94b29b..64f030c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -246,17 +246,38 @@ def test_shuffle(): def test_drop_bytes(): assert Chepy("hello").drop_bytes(2, 2).o == b"heo" + def test_without_pick(): - data1 = 'hello' - data2 = [1,2,'a', 'b'] - data3 = {'a':1, 2: 3} + data1 = "hello" + data2 = [1, 2, "a", "b"] + data3 = {"a": 1, 2: 3} # test without - assert Chepy(data1).without('ll').o == b'heo' - assert Chepy(data1).without('l', b'l').o == b'heo' - assert Chepy(data2).without(1, 'a').o == [2,'b'] - assert Chepy(data3).without('a').o == {2: 3} + assert Chepy(data1).without("ll").o == b"heo" + assert Chepy(data1).without("l", b"l").o == b"heo" + assert Chepy(data2).without(1, "a").o == [2, "b"] + assert Chepy(data3).without("a").o == {2: 3} # test pick - assert Chepy(data1).pick('ll').o == b'll' - assert Chepy(data1).pick('l', b'l').o == b'll' - assert Chepy(data2).pick(1, 'a').o == [1,'a'] - assert Chepy(data3).pick('a').o == {'a': 1} \ No newline at end of file + assert Chepy(data1).pick("ll").o == b"ll" + assert Chepy(data1).pick("l", b"l").o == b"ll" + assert Chepy(data2).pick(1, "a").o == [1, "a"] + assert Chepy(data3).pick("a").o == {"a": 1} + + +def test_alpha_range(): + assert Chepy("a-e").expand_alpha_range().o == ["a", "b", "c", "d", "e"] + assert Chepy("a-cA-C0-2").expand_alpha_range().o == [ + "a", + "b", + "c", + "A", + "B", + "C", + "0", + "1", + "2", + ] + assert ( + Chepy("a-cA-C0-2").expand_alpha_range("").o + == "".join(["a", "b", "c", "A", "B", "C", "0", "1", "2"]).encode() + ) + assert Chepy(" a-c:").expand_alpha_range().o == [" ", "a", "b", "c", ":"]