From 04d6e892f46e83d42c04daa18da72776f9f14029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=20B=C3=BCsgen?= Date: Sun, 24 Nov 2024 00:25:42 +0100 Subject: [PATCH] feat: initial implementation for manual api calls --- .pre-commit-config.yaml | 20 +-- poetry.lock | 30 ++--- pyproject.toml | 3 +- rockbox_listenbrainz_scrobbler/api_model.py | 122 +++++++++++++++++++ rockbox_listenbrainz_scrobbler/model.py | 4 +- rockbox_listenbrainz_scrobbler/scrobbling.py | 61 +++++++--- 6 files changed, 194 insertions(+), 46 deletions(-) create mode 100644 rockbox_listenbrainz_scrobbler/api_model.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e5424fe..38d52a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ default_stages: - - commit + - pre-commit repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/myint/autoflake - rev: v2.3.0 + rev: v2.3.1 hooks: - id: autoflake args: @@ -16,11 +16,11 @@ repos: - --remove-all-unused-imports - --expand-star-imports - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.13.0 hooks: - id: mypy additional_dependencies: - - numpy + - types-requests - repo: local hooks: - id: pytest-unit-tests @@ -32,7 +32,7 @@ repos: types: - python stages: - - push + - pre-push pass_filenames: false - id: pytest-integration-tests name: pytest integration tests @@ -43,10 +43,10 @@ repos: types: - python stages: - - push + - pre-push pass_filenames: false - repo: https://github.com/commitizen-tools/commitizen - rev: v3.15.0 + rev: v3.31.0 hooks: - id: commitizen stages: @@ -71,12 +71,12 @@ repos: - --profile - black - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.2.2 + rev: v0.8.0 hooks: # Run the linter. - id: ruff diff --git a/poetry.lock b/poetry.lock index e5a36e5..9d4b298 100644 --- a/poetry.lock +++ b/poetry.lock @@ -723,20 +723,6 @@ tomlkit = ">=0.10.1" spelling = ["pyenchant (>=3.2,<4.0)"] testutils = ["gitpython (>3)"] -[[package]] -name = "pylistenbrainz" -version = "0.5.1" -description = "A simple ListenBrainz client library for Python" -optional = false -python-versions = ">=3.5" -files = [ - {file = "pylistenbrainz-0.5.1-py3-none-any.whl", hash = "sha256:5d56c281898f0ff3a75ade720a09fbfe56832e277fd0c2b5cc0949848c46dfc2"}, - {file = "pylistenbrainz-0.5.1.tar.gz", hash = "sha256:20f4649bad2e5d5e949ad0bf9238c4e632a8df0a0b2b2220bd055506fb95ab48"}, -] - -[package.dependencies] -requests = ">=2.23.0" - [[package]] name = "pyside6" version = "6.8.0.2" @@ -941,6 +927,20 @@ rich = ">=10.11.0" shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" +[[package]] +name = "types-requests" +version = "2.32.0.20241016" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, + {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, +] + +[package.dependencies] +urllib3 = ">=2" + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1104,4 +1104,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.13,<3.14" -content-hash = "2fc0d3dac0ea9772ed580f47bb137deab95f61b17912bb22179121e7b71afb64" +content-hash = "66421ba7269d329b086a97b52da08ba7ba5be1641d7cf7e56efd4f58cee6de1f" diff --git a/pyproject.toml b/pyproject.toml index d406bd0..79a3f54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,10 @@ readme = "README.md" [tool.poetry.dependencies] python = ">=3.13,<3.14" -pylistenbrainz = "^0.5.1" typer = "^0.13.0" pydantic = "^2.8.2" pyside6 = "^6.7.2" +requests = "^2.32.3" [tool.poetry.scripts] rockbox-listenbrainz = "rockbox_listenbrainz_scrobbler.cli:main" @@ -21,6 +21,7 @@ pylint = "^3.2.6" black = "^24.8.0" pre-commit = "^4.0.0" nuitka = "^2.4.7" +types-requests = "^2.32.0.20241016" [build-system] requires = ["poetry-core"] diff --git a/rockbox_listenbrainz_scrobbler/api_model.py b/rockbox_listenbrainz_scrobbler/api_model.py new file mode 100644 index 0000000..5637ede --- /dev/null +++ b/rockbox_listenbrainz_scrobbler/api_model.py @@ -0,0 +1,122 @@ +from collections.abc import Callable +from enum import Enum +from typing import List, Optional, Set + +from pydantic import BaseModel, model_validator +from typing_extensions import Annotated + +from rockbox_listenbrainz_scrobbler import __version__ +from rockbox_listenbrainz_scrobbler.model import ScrobblerEntry + +# Relevant listenbrainz Constants + +# The maximum number of listens in a request. +MAX_LISTENS_PER_REQUEST = 1000 + +# Maximum overall listen size in bytes, to prevent egregious spamming. +MAX_LISTEN_SIZE = 10240 + +# The maximum size of a payload in bytes. The same as MAX_LISTEN_SIZE * MAX_LISTENS_PER_REQUEST. +MAX_LISTEN_PAYLOAD_SIZE = MAX_LISTENS_PER_REQUEST * MAX_LISTEN_SIZE + +# The maximum number of tags per listen. +MAX_TAGS_PER_LISTEN = 50 + +# The maximum length of a tag +MAX_TAG_SIZE = 64 + +# The minimum acceptable value for listened_at field +LISTEN_MINIMUM_TS = 1033430400 + + +def validate_max_size[T: BaseModel](max_size) -> Callable[[T], T]: + def validator(model: T): + json_data = model.model_dump_json().encode(encoding="utf8") + if len(json_data) > max_size: + raise ValueError( + f"The payload is too large. Size: {len(json_data)} Bytes. Allowed: {max_size}" + ) + return model + + return validator + + +PayloadValidator = Annotated[ + "ListenPayload", validate_max_size(MAX_LISTEN_PAYLOAD_SIZE) +] + + +class ListenType(str, Enum): + SINGLE = "single" + PLAYING_NOW = "playing_now" + IMPORT = "import" + + +class AdditionalInfo(BaseModel): + artist_mbids: Optional[List[str]] = None + release_group_mbid: Optional[str] = None + release_mbid: Optional[str] = None + recording_mbid: Optional[str] = None + track_mbid: Optional[str] = None + work_mbids: Optional[List[str]] = None + tracknumber: Optional[int] = None + isrc: Optional[str] = None + spotify_id: Optional[str] = None + tags: Optional[Set[str]] = None + media_player: Optional[str] = "Rockbox" + media_player_version: Optional[str] = None + submission_client: Optional[str] = "Rockbox Scrobbler" + submission_client_version: Optional[str] = __version__ + music_service: Optional[str] = None + music_service_name: Optional[str] = None + origin_url: Optional[str] = None + duration_ms: Optional[int] = None + duration: Optional[int] = None + + +class TrackMetadata(BaseModel): + artist_name: str + track_name: str + release_name: Optional[str] = None + additional_info: Optional[AdditionalInfo] = None + + +class ListenPayload(BaseModel): + track_metadata: TrackMetadata + listened_at: Optional[int] = None + + @classmethod + def from_rockbox_listen(cls, listen: ScrobblerEntry) -> "ListenPayload": + return cls( + listened_at=listen.timestamp, + track_metadata=TrackMetadata( + artist_name=listen.artist, + release_name=listen.album, + track_name=listen.title, + additional_info=AdditionalInfo( + tracknumber=listen.tracknum, + duration=listen.length, + track_mbid=listen.musicbrainz_trackid, + ), + ), + ) + + +class SubmitListens(BaseModel): + listen_type: ListenType + payload: List[PayloadValidator] + + @classmethod + def from_scrobbler_entries(cls, entries: List[ScrobblerEntry]) -> "SubmitListens": + listen_type = ListenType.SINGLE if len(entries) == 1 else ListenType.IMPORT + payloads = [ListenPayload.from_rockbox_listen(entry) for entry in entries] + return cls(listen_type=listen_type, payload=payloads) + + @model_validator(mode="after") + def validate_payload_size(self): + total_size = sum(len(p.model_dump_json()) for p in self.payload) + if total_size > MAX_LISTEN_PAYLOAD_SIZE: + raise ValueError( + f"The payload is too large. Size: {total_size} Bytes. Allowed: {MAX_LISTEN_PAYLOAD_SIZE}" + ) + return self diff --git a/rockbox_listenbrainz_scrobbler/model.py b/rockbox_listenbrainz_scrobbler/model.py index 58f692f..5fa084b 100644 --- a/rockbox_listenbrainz_scrobbler/model.py +++ b/rockbox_listenbrainz_scrobbler/model.py @@ -4,8 +4,8 @@ class SongRatingEnum(str, Enum): - LISTENED: str = "L" - SKIPPED: str = "S" + LISTENED = "L" + SKIPPED = "S" class ScrobblerEntry(BaseModel): diff --git a/rockbox_listenbrainz_scrobbler/scrobbling.py b/rockbox_listenbrainz_scrobbler/scrobbling.py index 80faf12..ed616aa 100644 --- a/rockbox_listenbrainz_scrobbler/scrobbling.py +++ b/rockbox_listenbrainz_scrobbler/scrobbling.py @@ -1,10 +1,19 @@ import csv +import logging +import time from abc import ABC, abstractmethod +from itertools import batched from pathlib import Path from typing import Iterable, List +from urllib.parse import urljoin -from pylistenbrainz import Listen, ListenBrainz +import requests +from pydantic import ValidationError +from rockbox_listenbrainz_scrobbler.api_model import ( + MAX_LISTENS_PER_REQUEST, + SubmitListens, +) from rockbox_listenbrainz_scrobbler.model import ScrobblerEntry, SongRatingEnum @@ -17,7 +26,7 @@ def scrobble(self, entry: ScrobblerEntry) -> None: raise NotImplementedError() @abstractmethod - def scrobble_multiple(self, entry: Iterable[ScrobblerEntry]) -> None: + def scrobble_multiple(self, entries: Iterable[ScrobblerEntry]) -> None: """ Scrobble multiple songs to your Endpoint """ @@ -25,29 +34,45 @@ def scrobble_multiple(self, entry: Iterable[ScrobblerEntry]) -> None: class ListenBrainzScrobbler(AbstractScrobbler): - def __init__(self, auth_token: str) -> None: + def __init__( + self, auth_token: str, base_url="https://api.listenbrainz.org" + ) -> None: super().__init__() - self.client = ListenBrainz() - self.client.set_auth_token(auth_token) + self.base_url = base_url + self.auth_token = auth_token + self.headers = {"Authorization": "Token {0}".format(self.auth_token)} def scrobble(self, entry: ScrobblerEntry) -> None: return self.scrobble_multiple([entry]) - def scrobble_multiple(self, entry: Iterable[ScrobblerEntry]) -> None: - listens = [ - Listen( - track_name=listen.title, - artist_name=listen.artist, - release_name=listen.album, - listened_at=listen.timestamp, - recording_mbid=listen.musicbrainz_trackid, - listening_from=listen.listening_from, - ) - for listen in entry - ] + def scrobble_multiple( + self, + entries: Iterable[ScrobblerEntry], + batchsize: int = MAX_LISTENS_PER_REQUEST, + ) -> None: + for batch in batched(entries, n=batchsize): + current_batch = list(batch) - self.client.submit_multiple_listens(listens) + try: + api_request = SubmitListens.from_scrobbler_entries(current_batch) + result = requests.post( + urljoin(self.base_url, "/1/submit-listens"), + data=api_request.model_dump(), + headers=self.headers, + ) + + remaining_calls = int(result.headers["X-RateLimit-Remaining"]) + ratelimit_reset = int(result.headers["X-RateLimit-Reset-In"]) + if remaining_calls == 0: + time.sleep(ratelimit_reset) + + except ValidationError as err: + logging.error( + f"Payload too large. Trying smaller batchsize...\nReason: {err}" + ) + if batchsize >= 2: + self.scrobble_multiple(current_batch, int(len(current_batch) / 2)) def read_rockbox_log(