From 118302ed4b9996c31b020ea3fa036075ff19d4b1 Mon Sep 17 00:00:00 2001 From: Henrique Date: Sun, 15 Dec 2024 21:21:36 -0300 Subject: [PATCH] Changed the use of requests to httpx --- poetry.lock | 97 ++++++++++++++++++++++++++++-- pyproject.toml | 8 +-- requirements.txt | 2 +- streamsnapper/downloader.py | 8 +-- streamsnapper/platforms/youtube.py | 21 +++---- 5 files changed, 111 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7a515b8..6c1ae51 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,27 @@ # This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +[[package]] +name = "anyio" +version = "4.7.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = true +python-versions = ">=3.9" +files = [ + {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"}, + {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +trio = ["trio (>=0.26.1)"] + [[package]] name = "certifi" version = "2024.12.14" @@ -150,6 +172,62 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = true +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.7" +description = "A minimal low-level HTTP client." +optional = true +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = true +python-versions = ">=3.8" +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "idna" version = "3.10" @@ -410,6 +488,17 @@ files = [ requests = "*" typing-extensions = "*" +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = true +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + [[package]] name = "tomli" version = "2.2.1" @@ -501,12 +590,12 @@ static-analysis = ["autopep8 (>=2.0,<3.0)", "ruff (>=0.8.0,<0.9.0)"] test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"] [extras] -all = ["requests", "rich", "scrapetube", "yt-dlp"] -downloader = ["requests", "rich"] +all = ["httpx", "rich", "scrapetube", "yt-dlp"] +downloader = ["httpx", "rich"] merger = [] -youtube = ["requests", "rich", "scrapetube", "yt-dlp"] +youtube = ["httpx", "rich", "scrapetube", "yt-dlp"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "41a6e7db54eeb76c2e480ac0a8fbc2eb1eef212163db4841363d88dad832a2f1" +content-hash = "4bd87ef278e5bafd2426f9aefdb62f531ab802279c81ccccb389558539f81201" diff --git a/pyproject.toml b/pyproject.toml index c2f4b79..f6ef7b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.9" -requests = { version = "2.32.3", optional = true } +httpx = { version = "0.28.1", optional = true } rich = { version = "13.9.4", optional = true } scrapetube = { version = "2.5.1", optional = true } yt-dlp = { version = "2024.12.13", optional = true } @@ -40,10 +40,10 @@ orjson = "*" pytest = "*" [tool.poetry.extras] -downloader = ["requests", "rich"] +downloader = ["httpx", "rich"] merger = [] -youtube = ["requests", "rich", "scrapetube", "yt-dlp"] -all = ["requests", "rich", "scrapetube", "yt-dlp"] +youtube = ["httpx", "rich", "scrapetube", "yt-dlp"] +all = ["httpx", "rich", "scrapetube", "yt-dlp"] [tool.ruff] include = ["streamsnapper/*.py", "streamsnapper/platforms/*.py", "tests/*.py"] diff --git a/requirements.txt b/requirements.txt index 005217d..33c14b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests == 2.32.3 +httpx == 0.28.1 rich == 13.9.4 scrapetube == 2.5.1 yt-dlp == 2024.12.13 \ No newline at end of file diff --git a/streamsnapper/downloader.py b/streamsnapper/downloader.py index 0bcdf8c..c7f4c35 100644 --- a/streamsnapper/downloader.py +++ b/streamsnapper/downloader.py @@ -8,7 +8,7 @@ from urllib.parse import unquote, urlparse # Third-party imports -from requests import get, head, exceptions as requests_exceptions +from httpx import get, head, HTTPError from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TimeRemainingColumn, TransferSpeedColumn # Local imports @@ -114,8 +114,8 @@ def _get_file_info(self, url: str) -> Tuple[int, str, str]: """ try: - r = head(url, headers=self.headers, timeout=self._timeout, allow_redirects=True) - except requests_exceptions.RequestException as e: + r = head(url, headers=self.headers, timeout=self._timeout, follow_redirects=True) + except HTTPError as e: raise RequestError(f'An error occurred while getting file info: {str(e)}') from e content_length = int(r.headers.get('content-length', 0)) @@ -198,7 +198,7 @@ def _download_chunk(self, url: str, start: int, end: int, progress: Progress, ta progress.update(task_id, advance=len(chunk)) return chunk - except requests_exceptions.RequestException as e: + except HTTPError as e: raise DownloadError(f'An error occurred while downloading chunk: {str(e)}') from e def download(self, url: str, output_path: Union[str, PathLike] = Path.cwd()) -> None: diff --git a/streamsnapper/platforms/youtube.py b/streamsnapper/platforms/youtube.py index cedfcf5..99e5afd 100644 --- a/streamsnapper/platforms/youtube.py +++ b/streamsnapper/platforms/youtube.py @@ -10,7 +10,7 @@ from urllib.parse import unquote # Third-party imports -from requests import get, head +from httpx import get, head from scrapetube import ( get_search as scrape_youtube_search, get_playlist as scrape_youtube_playlist, @@ -157,7 +157,7 @@ def analyze_info(self, check_thumbnails: bool = False, retrieve_dislike_count: b }, ) - if r.status_code == 200: + if r.is_success: try: dislike_count = get_value(r.json(), 'dislikes', convert_to=int) except JSONDecodeError: @@ -201,16 +201,13 @@ def analyze_info(self, check_thumbnails: bool = False, retrieve_dislike_count: b if check_thumbnails: while general_info['thumbnails']: - if ( - head( - general_info['thumbnails'][0], - headers={ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36' - }, - allow_redirects=False, - ).status_code - == 200 - ): + if head( + general_info['thumbnails'][0], + headers={ + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36' + }, + follow_redirects=False, + ).is_success: break else: general_info['thumbnails'].pop(0)