Skip to content

Commit

Permalink
Changed the use of requests to httpx
Browse files Browse the repository at this point in the history
  • Loading branch information
henrique-coder committed Dec 16, 2024
1 parent a02b66a commit 118302e
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 25 deletions.
97 changes: 93 additions & 4 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ classifiers = [

[tool.poetry.dependencies]
python = "^3.9"
requests = { version = "2.32.3", optional = true }
httpx = { version = "0.28.1", optional = true }
rich = { version = "13.9.4", optional = true }
scrapetube = { version = "2.5.1", optional = true }
yt-dlp = { version = "2024.12.13", optional = true }
Expand All @@ -40,10 +40,10 @@ orjson = "*"
pytest = "*"

[tool.poetry.extras]
downloader = ["requests", "rich"]
downloader = ["httpx", "rich"]
merger = []
youtube = ["requests", "rich", "scrapetube", "yt-dlp"]
all = ["requests", "rich", "scrapetube", "yt-dlp"]
youtube = ["httpx", "rich", "scrapetube", "yt-dlp"]
all = ["httpx", "rich", "scrapetube", "yt-dlp"]

[tool.ruff]
include = ["streamsnapper/*.py", "streamsnapper/platforms/*.py", "tests/*.py"]
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
requests == 2.32.3
httpx == 0.28.1
rich == 13.9.4
scrapetube == 2.5.1
yt-dlp == 2024.12.13
8 changes: 4 additions & 4 deletions streamsnapper/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from urllib.parse import unquote, urlparse

# Third-party imports
from requests import get, head, exceptions as requests_exceptions
from httpx import get, head, HTTPError
from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TimeRemainingColumn, TransferSpeedColumn

# Local imports
Expand Down Expand Up @@ -114,8 +114,8 @@ def _get_file_info(self, url: str) -> Tuple[int, str, str]:
"""

try:
r = head(url, headers=self.headers, timeout=self._timeout, allow_redirects=True)
except requests_exceptions.RequestException as e:
r = head(url, headers=self.headers, timeout=self._timeout, follow_redirects=True)
except HTTPError as e:
raise RequestError(f'An error occurred while getting file info: {str(e)}') from e

content_length = int(r.headers.get('content-length', 0))
Expand Down Expand Up @@ -198,7 +198,7 @@ def _download_chunk(self, url: str, start: int, end: int, progress: Progress, ta
progress.update(task_id, advance=len(chunk))

return chunk
except requests_exceptions.RequestException as e:
except HTTPError as e:
raise DownloadError(f'An error occurred while downloading chunk: {str(e)}') from e

def download(self, url: str, output_path: Union[str, PathLike] = Path.cwd()) -> None:
Expand Down
21 changes: 9 additions & 12 deletions streamsnapper/platforms/youtube.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from urllib.parse import unquote

# Third-party imports
from requests import get, head
from httpx import get, head
from scrapetube import (
get_search as scrape_youtube_search,
get_playlist as scrape_youtube_playlist,
Expand Down Expand Up @@ -157,7 +157,7 @@ def analyze_info(self, check_thumbnails: bool = False, retrieve_dislike_count: b
},
)

if r.status_code == 200:
if r.is_success:
try:
dislike_count = get_value(r.json(), 'dislikes', convert_to=int)
except JSONDecodeError:
Expand Down Expand Up @@ -201,16 +201,13 @@ def analyze_info(self, check_thumbnails: bool = False, retrieve_dislike_count: b

if check_thumbnails:
while general_info['thumbnails']:
if (
head(
general_info['thumbnails'][0],
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
},
allow_redirects=False,
).status_code
== 200
):
if head(
general_info['thumbnails'][0],
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
},
follow_redirects=False,
).is_success:
break
else:
general_info['thumbnails'].pop(0)
Expand Down

0 comments on commit 118302e

Please sign in to comment.