From b56a4708ccec0f66bb3e6ff2dc38faa68c700e3c Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 7 Jan 2022 15:28:25 -0500 Subject: [PATCH 01/24] move ReplayEvents to dataclasses --- osrparse/replay.py | 28 ++++++++++------ osrparse/utils.py | 80 +++++++++++----------------------------------- 2 files changed, 36 insertions(+), 72 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index 4940e36..2c38e5d 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -5,7 +5,8 @@ from io import TextIOWrapper from osrparse.utils import (Mod, GameMode, ReplayEvent, ReplayEventOsu, - ReplayEventCatch, ReplayEventMania, ReplayEventTaiko) + ReplayEventCatch, ReplayEventMania, ReplayEventTaiko, Key, KeyMania, + KeyTaiko) from osrparse.dump import dump_replay class Replay: @@ -123,15 +124,22 @@ def _parse_play_data(self, replay_data): datastring = lzma.decompress(replay_data[self.offset:offset_end], format=lzma.FORMAT_AUTO).decode('ascii')[:-1] events = [eventstring.split('|') for eventstring in datastring.split(',')] - if self.game_mode is GameMode.STD: - self.play_data = [ReplayEventOsu(int(event[0]), float(event[1]), float(event[2]), int(event[3])) for event in events] - if self.game_mode is GameMode.TAIKO: - self.play_data = [ReplayEventTaiko(int(event[0]), float(event[1]), int(event[3])) for event in events] - if self.game_mode is GameMode.CTB: - self.play_data = [ReplayEventCatch(int(event[0]), float(event[1]), int(event[3])) for event in events] - if self.game_mode is GameMode.MANIA: - self.play_data = [ReplayEventMania(int(event[0]), int(event[1])) for event in events] - + self.play_data = [] + for event in events: + time_delta = int(event[0]) + x = event[1] + y = event[2] + keys = int(event[3]) + + if self.game_mode is GameMode.STD: + event = ReplayEventOsu(time_delta, float(x), float(y), Key(keys)) + if self.game_mode is GameMode.TAIKO: + event = ReplayEventTaiko(time_delta, int(x), KeyTaiko(keys)) + if self.game_mode is GameMode.CTB: + event = ReplayEventCatch(time_delta, float(x), int(keys) == 1) + if self.game_mode is GameMode.MANIA: + event = ReplayEventMania(time_delta, KeyMania(keys)) + self.play_data.append(event) self.offset = offset_end if self.game_version >= self.LAST_FRAME_SEED_VERSION and self.play_data: diff --git a/osrparse/utils.py b/osrparse/utils.py index 25aa36a..dfe3aa3 100644 --- a/osrparse/utils.py +++ b/osrparse/utils.py @@ -1,5 +1,5 @@ from enum import Enum, IntFlag -import abc +from dataclasses import dataclass class GameMode(Enum): STD = 0 @@ -78,72 +78,28 @@ class KeyMania(IntFlag): # the reference I used for non-std replay events below: # https://github.com/kszlim/osu-replay-parser/pull/27#issuecomment-845679072. -class ReplayEvent(abc.ABC): - def __init__(self, time_delta: int): - self.time_delta = time_delta - - @abc.abstractmethod - def _members(self): - pass - - def __eq__(self, other): - if not isinstance(other, ReplayEvent): - return False - return all(m1 == m2 for m1, m2 in zip(self._members(), other._members())) - - def __hash__(self): - return hash(self._members()) +@dataclass +class ReplayEvent: + time_delta: int +@dataclass class ReplayEventOsu(ReplayEvent): - def __init__(self, time_delta: int, x: float, y: float, - keys: int): - super().__init__(time_delta) - self.x = x - self.y = y - self.keys = Key(keys) - - def __str__(self): - return (f"{self.time_delta} ({self.x}, {self.y}) " - f"{self.keys}") - - def _members(self): - return (self.time_delta, self.x, self.y, self.keys) + x: float + y: float + keys: Key +@dataclass class ReplayEventTaiko(ReplayEvent): - def __init__(self, time_delta: int, x: int, keys: int): - super().__init__(time_delta) - # we have no idea what this is supposed to represent. It's always one - # of 0, 320, or 640, depending on ``keys``. Leaving untouched for now. - self.x = x - self.keys = KeyTaiko(keys) - - def __str__(self): - return f"{self.time_delta} {self.x} {self.keys}" - - def _members(self): - return (self.time_delta, self.x, self.keys) + # we have no idea what this is supposed to represent. It's always one of 0, + # 320, or 640, depending on `keys`. Leaving untouched for now. + x: int + keys: KeyTaiko +@dataclass class ReplayEventCatch(ReplayEvent): - def __init__(self, time_delta: int, x: int, keys: int): - super().__init__(time_delta) - self.x = x - self.dashing = keys == 1 - - def __str__(self): - return f"{self.time_delta} {self.x} {self.dashing}" - - def _members(self): - return (self.time_delta, self.x, self.dashing) + x: float + dashing: bool +@dataclass class ReplayEventMania(ReplayEvent): - def __init__(self, time_delta: int, x: int): - super().__init__(time_delta) - # no, this isn't a typo. osu! really stores keys pressed inside ``x`` - # for mania. - self.keys = KeyMania(x) - - def __str__(self): - return f"{self.time_delta} {self.keys}" - - def _members(self): - return (self.time_delta, self.keys) + keys: KeyMania From d3227a5f4074a23bd81ef00fea0b153d073690d0 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 7 Jan 2022 19:04:45 -0500 Subject: [PATCH 02/24] completely restructure code --- README.md | 91 +++++---- osrparse/__init__.py | 8 +- osrparse/dump.py | 45 +++-- osrparse/parse.py | 51 ----- osrparse/replay.py | 436 ++++++++++++++++++++++++++++-------------- tests/test_dumping.py | 16 +- tests/test_replay.py | 56 +++--- 7 files changed, 413 insertions(+), 290 deletions(-) delete mode 100644 osrparse/parse.py diff --git a/README.md b/README.md index aace01b..c38bf30 100644 --- a/README.md +++ b/README.md @@ -17,36 +17,60 @@ pip install osrparse ### Parsing -To parse a replay from a filepath: +To parse a replay: ```python -from osrparse import parse_replay_file +from osrparse import Replay +replay = Replay.from_path("path/to/osr.osr") -# returns a Replay object -replay = parse_replay_file("path/to/osr.osr") +# or from an opened file object +with open("path/to/osr.osr") as f: + replay = Replay.from_file() + +# or from a string +with open("path/to/osr.osr") as f: + replay_string = f.read() +replay = Replay.from_string(replay_string) ``` -To parse a replay from an lzma string (such as the one returned from the `/get_replay` osu! api endpoint): +To parse only the `replay_data` portion of a `Replay`, such as the data returned from the api `/get_replay` endpoint: ```python -from osrparse import parse_replay - -# returns a Replay object that only has a `play_data` attribute -replay = parse_replay(lzma_string, pure_lzma=True) +from osrparse import parse_replay_data +import base64 +import lzma + +lzma_string = retrieve_from_api() +replay_data = parse_replay_data(lzma_string) +assert isinstance(replay_data[0], ReplayEvent) + +# or parse an already decoded lzma string +lzma_string = retrieve_from_api() +lzma_string = base64.b64decode(lzma_string) +replay_data = parse_replay_data(lzma_string, decoded=True) + +# or parse an already decoded and decompressed lzma string +lzma_string = retrieve_from_api() +lzma_string = base64.b64decode(lzma_string) +lzma_string = lzma.decompress(lzma_string).decode("ascii") +replay_data = parse_replay_data(lzma_string, decompressed=True) ``` -Note that if you use the `/get_replay` endpoint to retrieve a replay, you must decode the response before passing it to osrparse, as the response is encoded in base 64 by default. +The response returned from `/get_replay` is base 64 encoded, which is why we provide automatic decoding in `parse_replay_data`. If you are retrieving this data from a different source where the replay data is already decoded, pass `decoded=True`. -### Dumping +### Writing -Existing `Replay` objects can be "dumped" back to a `.osr` file: +Existing `Replay` objects can be written back to `.osr` files: ```python +replay.write_path("path/to/osr.osr") -replay.dump("path/to/osr.osr") # or to an opened file object with open("path/to/osr.osr") as f: - replay.dump(f) + replay.write_file(f) + +# or to a string +dumped = replay.dump() ``` You can also edit osr files by parsing a replay, editing an attribute, and dumping it back to its file: @@ -62,23 +86,26 @@ replay.dump(""path/to/osr.osr") `Replay` objects have the following attibutes: ```python -self.game_mode # GameMode enum -self.game_version # int -self.beatmap_hash # str -self.player_name # str -self.replay_hash # str -self.number_300s # int -self.number_100s # int -self.number_50s # int -self.gekis # int -self.katus # int -self.misses # int -self.score # int -self.max_combo # int -self.is_perfect_combo # bool -self.mod_combination # Mod enum -self.life_bar_graph # str, currently unparsed -self.timestamp # datetime.datetime object +self.mode # GameMode +self.game_version # int +self.beatmap_hash # str +self.username # str +self.replay_hash # str +self.count_300 # int +self.count_100 # int +self.count_50 # int +self.count_geki # int +self.count_katu # int +self.count_miss # int +self.score # int +self.max_combo # int +self.perfect # bool +self.mods # Mod +self.life_bar_graph # str or None +self.timestamp # datetime +self.replay_data # List[ReplayEvent] +self.replay_id # int +self.rng_seed # int or None # list of either ReplayEventOsu, ReplayEventTaiko, ReplayEventCatch, # or ReplayEventMania objects, depending on self.game_mode self.play_data @@ -113,7 +140,7 @@ self.dashing # bool, whether the player was dashing or not ```python self.time_delta # int, time since previous event in milliseconds -self.keys # KeyMania enum +self.keys # KeyMania enum, keys pressed ``` The `Key` enums used in the above `ReplayEvent`s are defined as follows: diff --git a/osrparse/__init__.py b/osrparse/__init__.py index 7109dfa..8c25faa 100644 --- a/osrparse/__init__.py +++ b/osrparse/__init__.py @@ -1,12 +1,10 @@ from osrparse.utils import (GameMode, Mod, Key, ReplayEvent, ReplayEventOsu, ReplayEventTaiko, ReplayEventMania, ReplayEventCatch, KeyTaiko, KeyMania) -from osrparse.parse import parse_replay_file, parse_replay -from osrparse.replay import Replay +from osrparse.replay import Replay, parse_replay_data __version__ = "5.0.0" -__all__ = ["GameMode", "Mod", "parse_replay_file", "parse_replay", - "Replay", "ReplayEvent", "Key", +__all__ = ["GameMode", "Mod", "Replay", "ReplayEvent", "Key", "ReplayEventOsu", "ReplayEventTaiko", "ReplayEventMania", - "ReplayEventCatch", "KeyTaiko", "KeyMania"] + "ReplayEventCatch", "KeyTaiko", "KeyMania", "parse_replay_data"] diff --git a/osrparse/dump.py b/osrparse/dump.py index 4d9b9c3..d9f1c38 100644 --- a/osrparse/dump.py +++ b/osrparse/dump.py @@ -47,18 +47,27 @@ def dump_timestamp(replay): def dump_replay_data(replay): replay_data = "" - for event in replay.play_data: + for event in replay.replay_data: + t = event.time_delta if isinstance(event, ReplayEventOsu): - replay_data += f"{event.time_delta}|{event.x}|{event.y}|{event.keys.value}," + replay_data += f"{t}|{event.x}|{event.y}|{event.keys.value}," elif isinstance(event, ReplayEventTaiko): - replay_data += f"{event.time_delta}|{event.x}|0|{event.keys.value}," + replay_data += f"{t}|{event.x}|0|{event.keys.value}," elif isinstance(event, ReplayEventCatch): - replay_data += f"{event.time_delta}|{event.x}|0|{int(event.dashing)}," + replay_data += f"{t}|{event.x}|0|{int(event.dashing)}," elif isinstance(event, ReplayEventMania): - replay_data += f"{event.time_delta}|{event.keys}|0|0," - - filters = [{"id": lzma.FILTER_LZMA1, "dict_size": 1 << 21, "mode": lzma.MODE_FAST}] - compressed = lzma.compress(replay_data.encode("ascii"), format=lzma.FORMAT_ALONE, filters=filters) + replay_data += f"{t}|{event.keys.value}|0|0," + + filters = [ + { + "id": lzma.FILTER_LZMA1, + "dict_size": 1 << 21, + "mode": lzma.MODE_FAST + } + ] + replay_data = replay_data.encode("ascii") + compressed = lzma.compress(replay_data, format=lzma.FORMAT_ALONE, + filters=filters) return pack_int(len(compressed)) + compressed @@ -66,25 +75,25 @@ def dump_replay_data(replay): def dump_replay(replay): data = b"" - data += pack_byte(replay.game_mode.value) + data += pack_byte(replay.mode.value) data += pack_int(replay.game_version) data += pack_string(replay.beatmap_hash) - data += pack_string(replay.player_name) + data += pack_string(replay.username) data += pack_string(replay.replay_hash) - data += pack_short(replay.number_300s) - data += pack_short(replay.number_100s) - data += pack_short(replay.number_50s) - data += pack_short(replay.gekis) - data += pack_short(replay.katus) - data += pack_short(replay.misses) + data += pack_short(replay.count_300) + data += pack_short(replay.count_100) + data += pack_short(replay.count_50) + data += pack_short(replay.count_geki) + data += pack_short(replay.count_katu) + data += pack_short(replay.count_miss) data += pack_int(replay.score) data += pack_short(replay.max_combo) - data += pack_byte(replay.is_perfect_combo) + data += pack_byte(replay.perfect) - data += pack_int(replay.mod_combination.value) + data += pack_int(replay.mods.value) data += pack_string(replay.life_bar_graph) data += dump_timestamp(replay) diff --git a/osrparse/parse.py b/osrparse/parse.py deleted file mode 100644 index 7285a32..0000000 --- a/osrparse/parse.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -from typing import Union - -from osrparse.replay import Replay - -def parse_replay(replay_data: str, pure_lzma: bool = False, decompressed_lzma: bool = False) -> Replay: - """ - Parses a Replay from the given replay data. - - Args: - String replay_data: The replay data from either parsing an osr file or from the api get_replay endpoint. - Boolean pure_lzma: Whether replay_data conatins the entirety of an osr file, or only the lzma compressed - data containing the cursor movements and keyboard presses of the player. - If replay data was loaded from an osr, this value should be False, as an osr contains - more information than just the lzma, such as username and game version (see - https://osu.ppy.sh/help/wiki/osu!_File_Formats/Osr_(file_format)). If replay data - was retrieved from the api, this value should be True, as the api only - returns the lzma data (see https://github.com/ppy/osu-api/wiki#apiget_replay) - Boolean decompressed_lzma: Whether replay_data is compressed lzma, or decompressed - (and decoded to ascii) lzma. For example, the following calls are equivalent: - ``` - >>> osrparse.parse_replay(lzma_string, pure_lzma=True) - ``` - and - ``` - >>> lzma_string = lzma.decompress(lzma_string).decode("ascii") - >>> osrparse.parse_replay(lzma_string, pure_lzma=True, decompressed_lzma=True) - ``` - This parameter only has an affect if ``pure_lzma`` is ``True``. - Returns: - A Replay object with the fields specific in the Replay's init method. If pure_lzma is False, all fields will - be filled (nonnull). If pure_lzma is True, only the play_data will be filled. - """ - - return Replay(replay_data, pure_lzma, decompressed_lzma) - -def parse_replay_file(replay_path: Union[os.PathLike, str], pure_lzma: bool = False) -> Replay: - """ - Parses a Replay from the file at the given path. - - Args: - [String or Path]: A pathlike object representing the absolute path to the file to parse data from. - Boolean pure_lzma: False if the file contains data equivalent to an osr file (or is itself an osr file), - and True if the file contains only lzma data. See parse_replay documentation for - more information on the difference between these two and how each affect the - fields in the final Replay object. - """ - - with open(replay_path, 'rb') as f: - data = f.read() - return parse_replay(data, pure_lzma) diff --git a/osrparse/replay.py b/osrparse/replay.py index 2c38e5d..1c305c6 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -1,80 +1,33 @@ +# CHANGELOG: +# * number_{300, 100, 50}s -> count_{300, 100, 50} +# * {geki, katu, misses} -> {count_geki, count_katu, count_miss} +# * is_perfect_combo -> perfect +# * game_mode -> mode +# * player_name -> username +# * play_data -> replay_data +# * added rng_seed + import lzma import struct from datetime import datetime, timezone, timedelta -from typing import List -from io import TextIOWrapper +from typing import List, Optional +import base64 from osrparse.utils import (Mod, GameMode, ReplayEvent, ReplayEventOsu, ReplayEventCatch, ReplayEventMania, ReplayEventTaiko, Key, KeyMania, KeyTaiko) from osrparse.dump import dump_replay -class Replay: - # first version with rng seed value added as the last frame in the lzma data - LAST_FRAME_SEED_VERSION = 20130319 - _BYTE = 1 - _SHORT = 2 - _INT = 4 - _LONG = 8 - - def __init__(self, replay_data: List[ReplayEvent], pure_lzma: bool, decompressed_lzma: bool): +class _Unpacker: + """ + Helper class for dealing with the ``.osr`` format. Not intended to be used + by consumers. + """ + def __init__(self, replay_data): + self.replay_data = replay_data self.offset = 0 - self.game_mode = None - self.game_version = None - self.beatmap_hash = None - self.player_name = None - self.replay_hash = None - self.number_300s = None - self.number_100s = None - self.number_50s = None - self.gekis = None - self.katus = None - self.misses = None - self.score = None - self.max_combo = None - self.is_perfect_combo = None - self.mod_combination = None - self.life_bar_graph = None - self.timestamp = None - self.play_data = None - self.replay_id = None - self.replay_length = None - self._parse_replay_and_initialize_fields(replay_data, pure_lzma, decompressed_lzma) - - def _parse_replay_and_initialize_fields(self, replay_data, pure_lzma, decompressed_lzma): - if pure_lzma: - self.data_from_lmza(replay_data, decompressed_lzma) - return - self._parse_game_mode_and_version(replay_data) - self._parse_beatmap_hash(replay_data) - self._parse_player_name(replay_data) - self._parse_replay_hash(replay_data) - self._parse_score_stats(replay_data) - self._parse_life_bar_graph(replay_data) - self._parse_timestamp_and_replay_length(replay_data) - self._parse_play_data(replay_data) - self._parse_replay_id(replay_data) - - def _parse_game_mode_and_version(self, replay_data): - format_specifier = "= self.LAST_FRAME_SEED_VERSION and self.play_data: - if self.play_data[-1].time_delta != -12345: - pass - # I've disabled this warning temporarily as it turns out that - # many replays (perhaps all replays in non-std gamemodes?) don't - # have an RNG seed value even after the expected version, so - # this was more of an annoying false positive than anything. - - # print("The RNG seed value was expected in the last frame, but was not found. " - # f"\nGame Version: {self.game_version}, version threshold: " - # f"{self.LAST_FRAME_SEED_VERSION}, replay hash: {self.replay_hash}") - else: - del self.play_data[-1] - - def data_from_lmza(self, lzma_string, decompressed_lzma): - if decompressed_lzma: - # replay data is already decompressed and decoded. - # Remove last character (comma) so splitting works, same below - datastring = lzma_string[:-1] - else: - datastring = lzma.decompress(lzma_string, format=lzma.FORMAT_AUTO).decode('ascii')[:-1] - events = [eventstring.split('|') for eventstring in datastring.split(',')] - self.play_data = [ReplayEventOsu(int(event[0]), float(event[1]), float(event[2]), int(event[3])) for event in events] + return play_data - if self.play_data[-1].time_delta == -12345: - del self.play_data[-1] - - def _parse_replay_id(self, replay_data): - format_specifier = " List[ReplayEvent]: + """ + Parses the replay data portion of a replay from a string. This method is + siutable for use with the replay data returned by api v1's ``/get_replay`` + endpoint, for instance. + + Parameters + ---------- + data_string: str or bytes + The replay data to parse. + decoded: bool + Whether ``data_string`` has already been decoded from a b64 + representation. Api v1 returns a base 64 encoded string, for instance. + decompressed: bool + Whether ``data_string`` has already been both decompressed from lzma, + and decoded to ascii. + |br| + For instance, the following two calls are equivalent: + ``` + >>> parse_replay_data(lzma_string, decoded=True) + >>> ... + >>> lzma_string = lzma.decompress(lzma_string).decode("ascii") + >>> parse_replay_data(lzma_string, decompressed=True) + ``` + |br| + If ``decompressed`` is ``True``, ``decoded`` is automatically set to + ``True`` as well (ie, if ``decompressed`` is ``True``, we will assume + ``data_string`` is not base 64 encoded). + mode: GameMode + What mode to parse the replay data as. + """ + # assume the data is already decoded if it's been decompressed + if not decoded and not decompressed: + data_string = base64.b64decode(data_string) + if not decompressed: + data_string = lzma.decompress(data_string, format=lzma.FORMAT_AUTO) + data_string = data_string.decode("ascii") + return _Unpacker.parse_replay_data(data_string, mode) diff --git a/tests/test_dumping.py b/tests/test_dumping.py index 7ecaf53..fd9caf0 100644 --- a/tests/test_dumping.py +++ b/tests/test_dumping.py @@ -2,7 +2,7 @@ from unittest import TestCase from tempfile import TemporaryDirectory -from osrparse import parse_replay_file +from osrparse import Replay RES = Path(__file__).parent / "resources" @@ -11,21 +11,21 @@ class TestDumping(TestCase): @classmethod def setUpClass(cls): - cls.replay = parse_replay_file(RES / "replay.osr") + cls.replay = Replay.from_path(RES / "replay.osr") def test_dumping(self): with TemporaryDirectory() as tempdir: r2_path = Path(tempdir) / "dumped.osr" - self.replay.dump(r2_path) - r2 = parse_replay_file(r2_path) + self.replay.write_path(r2_path) + r2 = Replay.from_path(r2_path) # `replay_length` is intentionally not tested for equality here, as the # length of the compressed replay data may change after dumping due to # varying lzma settings. - attrs = ["game_mode", "game_version", "beatmap_hash", "player_name", - "replay_hash", "number_300s", "number_100s", "number_50s", "gekis", - "katus", "misses", "score", "max_combo", "is_perfect_combo", - "mod_combination", "life_bar_graph", "timestamp", "play_data", + attrs = ["mode", "game_version", "beatmap_hash", "username", + "replay_hash", "count_300", "count_100", "count_50", "count_geki", + "count_katu", "count_miss", "score", "max_combo", "perfect", + "mods", "life_bar_graph", "timestamp", "replay_data", "replay_id"] for attr in attrs: self.assertEqual(getattr(self.replay, attr), getattr(r2, attr), diff --git a/tests/test_replay.py b/tests/test_replay.py index 75d3962..e293f18 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -1,8 +1,8 @@ from pathlib import Path from unittest import TestCase from datetime import datetime, timezone -from osrparse import (parse_replay, parse_replay_file, ReplayEventOsu, GameMode, - Mod, ReplayEventTaiko, ReplayEventCatch, ReplayEventMania) +from osrparse import (ReplayEventOsu, GameMode, Mod, ReplayEventTaiko, + ReplayEventCatch, ReplayEventMania, Replay) RES = Path(__file__).parent / "resources" @@ -14,13 +14,13 @@ def setUpClass(cls): replay1_path = RES / "replay.osr" with open(replay1_path, "rb") as f: data = f.read() - cls._replays = [parse_replay(data, pure_lzma=False), parse_replay_file(replay1_path)] - cls._combination_replay = parse_replay_file(RES / "replay2.osr") - cls._old_replayid_replay = parse_replay_file(RES / "replay_old_replayid.osr") + cls._replays = [Replay.from_string(data), Replay.from_path(replay1_path)] + cls._combination_replay = Replay.from_path(RES / "replay2.osr") + cls._old_replayid_replay = Replay.from_path(RES / "replay_old_replayid.osr") def test_replay_mode(self): for replay in self._replays: - self.assertEqual(replay.game_mode, GameMode.STD, "Game mode is incorrect") + self.assertEqual(replay.mode, GameMode.STD, "Game mode is incorrect") def test_game_version(self): for replay in self._replays: @@ -32,16 +32,16 @@ def test_beatmap_hash(self): def test_player_name(self): for replay in self._replays: - self.assertEqual(replay.player_name, "Cookiezi", "Player name is incorrect") + self.assertEqual(replay.username, "Cookiezi", "Player name is incorrect") def test_number_hits(self): for replay in self._replays: - self.assertEqual(replay.number_300s, 1982, "Number of 300s is wrong") - self.assertEqual(replay.number_100s, 1, "Number of 100s is wrong") - self.assertEqual(replay.number_50s, 0, "Number of 50s is wrong") - self.assertEqual(replay.gekis, 250, "Number of gekis is wrong") - self.assertEqual(replay.katus, 1, "Number of katus is wrong") - self.assertEqual(replay.misses, 0, "Number of misses is wrong") + self.assertEqual(replay.count_300, 1982, "Number of 300s is wrong") + self.assertEqual(replay.count_100, 1, "Number of 100s is wrong") + self.assertEqual(replay.count_50, 0, "Number of 50s is wrong") + self.assertEqual(replay.count_geki, 250, "Number of gekis is wrong") + self.assertEqual(replay.count_katu, 1, "Number of katus is wrong") + self.assertEqual(replay.count_miss, 0, "Number of misses is wrong") def test_max_combo(self): for replay in self._replays: @@ -49,14 +49,14 @@ def test_max_combo(self): def test_is_perfect_combo(self): for replay in self._replays: - self.assertEqual(replay.is_perfect_combo, True, "is_perfect_combo is wrong") + self.assertEqual(replay.perfect, True, "is_perfect_combo is wrong") def test_nomod(self): for replay in self._replays: - self.assertEqual(replay.mod_combination, Mod.NoMod, "Mod combination is wrong") + self.assertEqual(replay.mods, Mod.NoMod, "Mod combination is wrong") def test_mod_combination(self): - self.assertEqual(self._combination_replay.mod_combination, Mod.Hidden | Mod.HardRock, "Mod combination is wrong") + self.assertEqual(self._combination_replay.mods, Mod.Hidden | Mod.HardRock, "Mod combination is wrong") def test_timestamp(self): for replay in self._replays: @@ -64,8 +64,8 @@ def test_timestamp(self): def test_play_data(self): for replay in self._replays: - self.assertIsInstance(replay.play_data[0], ReplayEventOsu, "Replay data is wrong") - self.assertEqual(len(replay.play_data), 17500, "Replay data is wrong") + self.assertIsInstance(replay.replay_data[0], ReplayEventOsu, "Replay data is wrong") + self.assertEqual(len(replay.replay_data), 17500, "Replay data is wrong") def test_replay_id(self): for replay in self._replays: @@ -78,31 +78,31 @@ class TestTaikoReplay(TestCase): @classmethod def setUpClass(cls): - cls.replay = parse_replay_file(RES / "taiko.osr") + cls.replay = Replay.from_path(RES / "taiko.osr") def test_play_data(self): - play_data = self.replay.play_data - self.assertIsInstance(play_data[0], ReplayEventTaiko, "Replay data is wrong") - self.assertEqual(len(play_data), 17475, "Replay data is wrong") + replay_data = self.replay.replay_data + self.assertIsInstance(replay_data[0], ReplayEventTaiko, "Replay data is wrong") + self.assertEqual(len(replay_data), 17475, "Replay data is wrong") class TestCatchReplay(TestCase): @classmethod def setUpClass(cls): - cls.replay = parse_replay_file(RES / "ctb.osr") + cls.replay = Replay.from_path(RES / "ctb.osr") def test_play_data(self): - play_data = self.replay.play_data - self.assertIsInstance(play_data[0], ReplayEventCatch, "Replay data is wrong") - self.assertEqual(len(play_data), 10439, "Replay data is wrong") + replay_data = self.replay.replay_data + self.assertIsInstance(replay_data[0], ReplayEventCatch, "Replay data is wrong") + self.assertEqual(len(replay_data), 10439, "Replay data is wrong") class TestManiaReplay(TestCase): @classmethod def setUpClass(cls): - cls.replay = parse_replay_file(RES / "mania.osr") + cls.replay = Replay.from_path(RES / "mania.osr") def test_play_data(self): - play_data = self.replay.play_data + play_data = self.replay.replay_data self.assertIsInstance(play_data[0], ReplayEventMania, "Replay data is wrong") self.assertEqual(len(play_data), 17432, "Replay data is wrong") From afb8331210ed83586c2a0077434de90f532136c5 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 7 Jan 2022 19:09:05 -0500 Subject: [PATCH 03/24] remove temporary changelog --- osrparse/replay.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index 1c305c6..609acaa 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -1,12 +1,3 @@ -# CHANGELOG: -# * number_{300, 100, 50}s -> count_{300, 100, 50} -# * {geki, katu, misses} -> {count_geki, count_katu, count_miss} -# * is_perfect_combo -> perfect -# * game_mode -> mode -# * player_name -> username -# * play_data -> replay_data -# * added rng_seed - import lzma import struct from datetime import datetime, timezone, timedelta From dd2661978957b394f231a20c5c1010c6f8058cd5 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 7 Jan 2022 21:08:48 -0500 Subject: [PATCH 04/24] bump version to 6.0.0 --- osrparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osrparse/__init__.py b/osrparse/__init__.py index 8c25faa..6881b95 100644 --- a/osrparse/__init__.py +++ b/osrparse/__init__.py @@ -2,7 +2,7 @@ ReplayEventTaiko, ReplayEventMania, ReplayEventCatch, KeyTaiko, KeyMania) from osrparse.replay import Replay, parse_replay_data -__version__ = "5.0.0" +__version__ = "6.0.0" __all__ = ["GameMode", "Mod", "Replay", "ReplayEvent", "Key", From 687917be6e688aa2eb82c4464a9cb5623c1f2491 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 7 Jan 2022 21:29:32 -0500 Subject: [PATCH 05/24] fix examples --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c38bf30..501215a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ replay = Replay.from_path("path/to/osr.osr") # or from an opened file object with open("path/to/osr.osr") as f: - replay = Replay.from_file() + replay = Replay.from_file(f) # or from a string with open("path/to/osr.osr") as f: @@ -76,9 +76,9 @@ dumped = replay.dump() You can also edit osr files by parsing a replay, editing an attribute, and dumping it back to its file: ```python -replay = parse_replay_file("path/to/osr.osr") -replay.player_name = "fake username" -replay.dump(""path/to/osr.osr") +replay = Replay.from_path("path/to/osr.osr") +replay.username = "fake username" +replay.write_path("path/to/osr.osr") ``` ### Attributes From d62c44746a36934692e635962a7859dc2a42018a Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 7 Jan 2022 21:31:35 -0500 Subject: [PATCH 06/24] fix naming --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 501215a..f97be34 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ self.replay_id # int self.rng_seed # int or None # list of either ReplayEventOsu, ReplayEventTaiko, ReplayEventCatch, # or ReplayEventMania objects, depending on self.game_mode -self.play_data +self.replay_data ``` `ReplayEventOsu` objects have the following attributes: From 777a1a8a5d593183dd99cd7bb66c9860d1f7707b Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sun, 9 Jan 2022 13:11:37 -0500 Subject: [PATCH 07/24] expose replay writing dict_size and mode --- osrparse/dump.py | 10 +++++----- osrparse/replay.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/osrparse/dump.py b/osrparse/dump.py index d9f1c38..26fbf90 100644 --- a/osrparse/dump.py +++ b/osrparse/dump.py @@ -45,7 +45,7 @@ def dump_timestamp(replay): return pack_long(ticks) -def dump_replay_data(replay): +def dump_replay_data(replay, *, dict_size=None, mode=None): replay_data = "" for event in replay.replay_data: t = event.time_delta @@ -61,8 +61,8 @@ def dump_replay_data(replay): filters = [ { "id": lzma.FILTER_LZMA1, - "dict_size": 1 << 21, - "mode": lzma.MODE_FAST + "dict_size": dict_size or 1 << 21, + "mode": mode or lzma.MODE_FAST } ] replay_data = replay_data.encode("ascii") @@ -72,7 +72,7 @@ def dump_replay_data(replay): return pack_int(len(compressed)) + compressed -def dump_replay(replay): +def dump_replay(replay, *, dict_size=None, mode=None): data = b"" data += pack_byte(replay.mode.value) @@ -97,7 +97,7 @@ def dump_replay(replay): data += pack_string(replay.life_bar_graph) data += dump_timestamp(replay) - data += dump_replay_data(replay) + data += dump_replay_data(replay, dict_size=dict_size, mode=mode) data += pack_long(replay.replay_id) return data diff --git a/osrparse/replay.py b/osrparse/replay.py index 609acaa..aca08d8 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -248,7 +248,7 @@ def from_string(data): """ return _Unpacker(data).unpack() - def write_path(self, path): + def write_path(self, path, *, dict_size=None, mode=None): """ Writes the replay to the given ``path``. @@ -264,9 +264,9 @@ def write_path(self, path): an attribute, then writing the replay back to its file. """ with open(path, "wb") as f: - self.write_file(f) + self.write_file(f, dict_size=dict_size, mode=mode) - def write_file(self, file): + def write_file(self, file, *, dict_size=None, mode=None): """ Writes the replay to an open file object. @@ -275,10 +275,10 @@ def write_file(self, file): file: file-like The file object to write to. """ - dumped = self.dump() + dumped = self.dump(dict_size=dict_size, mode=mode) file.write(dumped) - def dump(self): + def dump(self, *, dict_size=None, mode=None): """ Returns the text representing this ``Replay``, in ``.osr`` format. The text returned by this method is suitable for writing to a file as a @@ -289,7 +289,7 @@ def dump(self): str The text representing this ``Replay``, in ``.osr`` format. """ - return dump_replay(self) + return dump_replay(self, dict_size=dict_size, mode=mode) def parse_replay_data(data_string, *, decoded=False, decompressed=False, From f7601acce623e3e4204edbb857cf3c4ffc640037 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sun, 9 Jan 2022 17:27:21 -0500 Subject: [PATCH 08/24] renaming dumping to packing, move to class --- osrparse/dump.py | 103 ----------------------------------------- osrparse/replay.py | 111 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 108 deletions(-) delete mode 100644 osrparse/dump.py diff --git a/osrparse/dump.py b/osrparse/dump.py deleted file mode 100644 index 26fbf90..0000000 --- a/osrparse/dump.py +++ /dev/null @@ -1,103 +0,0 @@ -import lzma -import struct - -from osrparse.utils import (ReplayEventOsu, ReplayEventTaiko, ReplayEventCatch, - ReplayEventMania) - -def pack_byte(data: int): - return struct.pack("> 7 - - if (i == 0 and byte & 0x40 == 0) or (i == -1 and byte & 0x40 != 0): - r.append(byte) - return b"".join(map(pack_byte, r)) - - r.append(0x80 | byte) - -def pack_string(data: str): - if data: - return pack_byte(11) + pack_ULEB128(data) + data.encode("utf-8") - return pack_byte(11) + pack_byte(0) - -def dump_timestamp(replay): - # windows ticks starts at year 0001, in contrast to unix time (1970). - # 62135596800 is the number of seconds between these two years and is added - # to account for this difference. - # The factor of 10000000 converts seconds to ticks. - ticks = (62135596800 + replay.timestamp.timestamp()) * 10000000 - ticks = int(ticks) - return pack_long(ticks) - - -def dump_replay_data(replay, *, dict_size=None, mode=None): - replay_data = "" - for event in replay.replay_data: - t = event.time_delta - if isinstance(event, ReplayEventOsu): - replay_data += f"{t}|{event.x}|{event.y}|{event.keys.value}," - elif isinstance(event, ReplayEventTaiko): - replay_data += f"{t}|{event.x}|0|{event.keys.value}," - elif isinstance(event, ReplayEventCatch): - replay_data += f"{t}|{event.x}|0|{int(event.dashing)}," - elif isinstance(event, ReplayEventMania): - replay_data += f"{t}|{event.keys.value}|0|0," - - filters = [ - { - "id": lzma.FILTER_LZMA1, - "dict_size": dict_size or 1 << 21, - "mode": mode or lzma.MODE_FAST - } - ] - replay_data = replay_data.encode("ascii") - compressed = lzma.compress(replay_data, format=lzma.FORMAT_ALONE, - filters=filters) - - return pack_int(len(compressed)) + compressed - - -def dump_replay(replay, *, dict_size=None, mode=None): - data = b"" - - data += pack_byte(replay.mode.value) - data += pack_int(replay.game_version) - data += pack_string(replay.beatmap_hash) - - data += pack_string(replay.username) - data += pack_string(replay.replay_hash) - - data += pack_short(replay.count_300) - data += pack_short(replay.count_100) - data += pack_short(replay.count_50) - data += pack_short(replay.count_geki) - data += pack_short(replay.count_katu) - data += pack_short(replay.count_miss) - - data += pack_int(replay.score) - data += pack_short(replay.max_combo) - data += pack_byte(replay.perfect) - - data += pack_int(replay.mods.value) - data += pack_string(replay.life_bar_graph) - data += dump_timestamp(replay) - - data += dump_replay_data(replay, dict_size=dict_size, mode=mode) - data += pack_long(replay.replay_id) - - return data diff --git a/osrparse/replay.py b/osrparse/replay.py index aca08d8..f4ef6d8 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -7,7 +7,7 @@ from osrparse.utils import (Mod, GameMode, ReplayEvent, ReplayEventOsu, ReplayEventCatch, ReplayEventMania, ReplayEventTaiko, Key, KeyMania, KeyTaiko) -from osrparse.dump import dump_replay + class _Unpacker: """ @@ -144,6 +144,107 @@ def unpack(self): timestamp, play_data, replay_id, rng_seed) +class _Packer: + + def __init__(self, replay, *, dict_size=None, mode=None): + self.replay = replay + self.dict_size = dict_size or 1 << 21 + self.mode = mode or lzma.MODE_FAST + + def pack_byte(self, data): + return struct.pack("> 7 + + if (i == 0 and byte & 0x40 == 0) or (i == -1 and byte & 0x40 != 0): + r.append(byte) + return b"".join(map(self.pack_byte, r)) + + r.append(0x80 | byte) + + def pack_string(self, data): + if data: + return (self.pack_byte(11) + self.pack_ULEB128(data) + + data.encode("utf-8")) + return self.pack_byte(11) + self.pack_byte(0) + + def pack_timestamp(self): + # windows ticks starts at year 0001, in contrast to unix time (1970). + # 62135596800 is the number of seconds between these two years and is + # added to account for this difference. + # The factor of 10000000 converts seconds to ticks. + ticks = (62135596800 + self.replay.timestamp.timestamp()) * 10000000 + ticks = int(ticks) + return self.pack_long(ticks) + + def pack_replay_data(self): + replay_data = "" + for event in self.replay.replay_data: + t = event.time_delta + if isinstance(event, ReplayEventOsu): + replay_data += f"{t}|{event.x}|{event.y}|{event.keys.value}," + elif isinstance(event, ReplayEventTaiko): + replay_data += f"{t}|{event.x}|0|{event.keys.value}," + elif isinstance(event, ReplayEventCatch): + replay_data += f"{t}|{event.x}|0|{int(event.dashing)}," + elif isinstance(event, ReplayEventMania): + replay_data += f"{t}|{event.keys.value}|0|0," + + filters = [ + { + "id": lzma.FILTER_LZMA1, + "dict_size": self.dict_size, + "mode": self.mode + } + ] + replay_data = replay_data.encode("ascii") + compressed = lzma.compress(replay_data, format=lzma.FORMAT_ALONE, + filters=filters) + + return self.pack_int(len(compressed)) + compressed + + + def pack(self): + r = self.replay + data = b"" + + data += self.pack_byte(r.mode.value) + data += self.pack_int(r.game_version) + data += self.pack_string(r.beatmap_hash) + data += self.pack_string(r.username) + data += self.pack_string(r.replay_hash) + data += self.pack_short(r.count_300) + data += self.pack_short(r.count_100) + data += self.pack_short(r.count_50) + data += self.pack_short(r.count_geki) + data += self.pack_short(r.count_katu) + data += self.pack_short(r.count_miss) + data += self.pack_int(r.score) + data += self.pack_short(r.max_combo) + data += self.pack_byte(r.perfect) + data += self.pack_int(r.mods.value) + data += self.pack_string(r.life_bar_graph) + data += self.pack_timestamp() + data += self.pack_replay_data() + data += self.pack_long(r.replay_id) + + return data + class Replay: """ @@ -275,10 +376,10 @@ def write_file(self, file, *, dict_size=None, mode=None): file: file-like The file object to write to. """ - dumped = self.dump(dict_size=dict_size, mode=mode) - file.write(dumped) + packed = self.pack(dict_size=dict_size, mode=mode) + file.write(packed) - def dump(self, *, dict_size=None, mode=None): + def pack(self, *, dict_size=None, mode=None): """ Returns the text representing this ``Replay``, in ``.osr`` format. The text returned by this method is suitable for writing to a file as a @@ -289,7 +390,7 @@ def dump(self, *, dict_size=None, mode=None): str The text representing this ``Replay``, in ``.osr`` format. """ - return dump_replay(self, dict_size=dict_size, mode=mode) + return _Packer(self, dict_size=dict_size, mode=mode).pack() def parse_replay_data(data_string, *, decoded=False, decompressed=False, From 790a5e7a6772f59fe69feb99c7d14da43b79e7ab Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sun, 9 Jan 2022 17:27:56 -0500 Subject: [PATCH 09/24] update readme example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f97be34..c1f82c0 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ with open("path/to/osr.osr") as f: replay.write_file(f) # or to a string -dumped = replay.dump() +packed = replay.pack() ``` You can also edit osr files by parsing a replay, editing an attribute, and dumping it back to its file: From bef905a652ad6c5893a8f3385b12f8c81849c092 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sun, 9 Jan 2022 17:49:19 -0500 Subject: [PATCH 10/24] correctly set rng seed --- osrparse/replay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index f4ef6d8..c3c34fc 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -135,7 +135,7 @@ def unpack(self): rng_seed = None if play_data[-1].time_delta == -12345: - rng_seed = play_data[-1] + rng_seed = play_data[-1].keys.value del play_data[-1] return Replay(mode, game_version, beatmap_hash, username, From 776bd67529476256da190dd5ef9feb82b0e94b90 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sun, 9 Jan 2022 17:49:26 -0500 Subject: [PATCH 11/24] pack rng seed --- osrparse/replay.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/osrparse/replay.py b/osrparse/replay.py index c3c34fc..e65d03f 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -205,6 +205,9 @@ def pack_replay_data(self): elif isinstance(event, ReplayEventMania): replay_data += f"{t}|{event.keys.value}|0|0," + if self.replay.rng_seed: + replay_data += f"-12345|0|0|{self.replay.rng_seed}," + filters = [ { "id": lzma.FILTER_LZMA1, From df8c65aa877330eb594903d9e4d9e2d2d0bf142c Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sun, 9 Jan 2022 17:49:32 -0500 Subject: [PATCH 12/24] convert replay to dataclass --- osrparse/replay.py | 64 ++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index e65d03f..4729650 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -3,6 +3,7 @@ from datetime import datetime, timezone, timedelta from typing import List, Optional import base64 +from dataclasses import dataclass from osrparse.utils import (Mod, GameMode, ReplayEvent, ReplayEventOsu, ReplayEventCatch, ReplayEventMania, ReplayEventTaiko, Key, KeyMania, @@ -249,54 +250,33 @@ def pack(self): return data +@dataclass class Replay: """ A replay found in a ``.osr`` file, or following the osr format. To create a replay, you likely want ``Replay.from_path``, ``Replay.from_file``, or ``Replay.from_string``. """ - def __init__(self, - mode: GameMode, - game_version: int, - beatmap_hash: str, - username: str, - replay_hash: str, - count_300: int, - count_100: int, - count_50: int, - count_geki: int, - count_katu: int, - count_miss: int, - score: int, - max_combo: int, - perfect: bool, - mods: Mod, - life_bar_graph: Optional[str], - timestamp: datetime, - replay_data: List[ReplayEvent], - replay_id: int, - rng_seed: Optional[int] - ): - self.mode = mode - self.game_version = game_version - self.beatmap_hash = beatmap_hash - self.username = username - self.replay_hash = replay_hash - self.count_300 = count_300 - self.count_100 = count_100 - self.count_50 = count_50 - self.count_geki = count_geki - self.count_katu = count_katu - self.count_miss = count_miss - self.score = score - self.max_combo = max_combo - self.perfect = perfect - self.mods = mods - self.life_bar_graph = life_bar_graph - self.timestamp = timestamp - self.replay_data = replay_data - self.replay_id = replay_id - self.rng_seed = rng_seed + mode: GameMode + game_version: int + beatmap_hash: str + username: str + replay_hash: str + count_300: int + count_100: int + count_50: int + count_geki: int + count_katu: int + count_miss: int + score: int + max_combo: int + perfect: bool + mods: Mod + life_bar_graph: Optional[str] + timestamp: datetime + replay_data: List[ReplayEvent] + replay_id: int + rng_seed: Optional[int] @staticmethod def from_path(path): From a792c646d37e304ea6289e99fbcbd411df3b16d2 Mon Sep 17 00:00:00 2001 From: bemxio Date: Mon, 10 Jan 2022 18:27:30 +0100 Subject: [PATCH 13/24] made a packer and an unpacker for a life bar graph --- osrparse/replay.py | 86 +++++++++++++++++++++++++++++++++------------- osrparse/utils.py | 5 +++ 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index 4729650..79810ec 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -7,8 +7,7 @@ from osrparse.utils import (Mod, GameMode, ReplayEvent, ReplayEventOsu, ReplayEventCatch, ReplayEventMania, ReplayEventTaiko, Key, KeyMania, - KeyTaiko) - + KeyTaiko, LifeBarState) class _Unpacker: """ @@ -113,6 +112,29 @@ def unpack_replay_id(self): replay_id = self.unpack_once("l") return replay_id + def unpack_life_bar(self): + data = self.unpack_string() + graph = [] + + if not data: + return [] + + data = [x.split(",") for x in data.split("|")] + + # it seems like many replays have a graph starting with time, without the life state + # and then at the end of the graph, the life state without a time is there + # just in case, i am checking for them and adding None to values that are lacking + + for index, state in enumerate(data): + if index == 0: + graph.append(LifeBarState(int(state[0]), None)) + elif index == len(data) - 1: + graph.append(LifeBarState(None, int(state[0]))) + else: + graph.append(LifeBarState(int(state[1]), float(state[0]))) + + return graph + def unpack(self): mode = GameMode(self.unpack_once("b")) game_version = self.unpack_once("i") @@ -129,7 +151,7 @@ def unpack(self): max_combo = self.unpack_once("h") perfect = self.unpack_once("?") mods = Mod(self.unpack_once("i")) - life_bar_graph = self.unpack_string() + life_bar_graph = self.unpack_life_bar() timestamp = self.unpack_timestamp() play_data = self.unpack_play_data(mode) replay_id = self.unpack_replay_id() @@ -144,9 +166,7 @@ def unpack(self): count_miss, score, max_combo, perfect, mods, life_bar_graph, timestamp, play_data, replay_id, rng_seed) - class _Packer: - def __init__(self, replay, *, dict_size=None, mode=None): self.replay = replay self.dict_size = dict_size or 1 << 21 @@ -184,30 +204,50 @@ def pack_string(self, data): data.encode("utf-8")) return self.pack_byte(11) + self.pack_byte(0) - def pack_timestamp(self): + def pack_timestamp(self, date): # windows ticks starts at year 0001, in contrast to unix time (1970). # 62135596800 is the number of seconds between these two years and is # added to account for this difference. # The factor of 10000000 converts seconds to ticks. - ticks = (62135596800 + self.replay.timestamp.timestamp()) * 10000000 + + ticks = (62135596800 + date.timestamp()) * 10000000 ticks = int(ticks) return self.pack_long(ticks) - def pack_replay_data(self): - replay_data = "" - for event in self.replay.replay_data: + def pack_life_bar(self, graph): + text = "" + for state in graph: + if state.life is None: + text += f"{state.time}|" + continue + + if int(state.life) == state.life: # checking if time is actually a fraction (osu! wants an integer for 0 or 1) + life = int(state.life) + else: + life = state.life + + if state.time is None: + text += f"{life}," + else: + text += f"{life},{state.time}|" + + return self.pack_string(text) + + def pack_replay_data(self, replay_data): + data = "" + for event in replay_data: t = event.time_delta if isinstance(event, ReplayEventOsu): - replay_data += f"{t}|{event.x}|{event.y}|{event.keys.value}," + data += f"{t}|{event.x}|{event.y}|{event.keys.value}," elif isinstance(event, ReplayEventTaiko): - replay_data += f"{t}|{event.x}|0|{event.keys.value}," + data += f"{t}|{event.x}|0|{event.keys.value}," elif isinstance(event, ReplayEventCatch): - replay_data += f"{t}|{event.x}|0|{int(event.dashing)}," + data += f"{t}|{event.x}|0|{int(event.dashing)}," elif isinstance(event, ReplayEventMania): - replay_data += f"{t}|{event.keys.value}|0|0," + data += f"{t}|{event.keys.value}|0|0," if self.replay.rng_seed: - replay_data += f"-12345|0|0|{self.replay.rng_seed}," + data += f"-12345|0|0|{self.replay.rng_seed}," filters = [ { @@ -216,13 +256,13 @@ def pack_replay_data(self): "mode": self.mode } ] - replay_data = replay_data.encode("ascii") - compressed = lzma.compress(replay_data, format=lzma.FORMAT_ALONE, + + data = data.encode("ascii") + compressed = lzma.compress(data, format=lzma.FORMAT_ALONE, filters=filters) return self.pack_int(len(compressed)) + compressed - def pack(self): r = self.replay data = b"" @@ -242,14 +282,13 @@ def pack(self): data += self.pack_short(r.max_combo) data += self.pack_byte(r.perfect) data += self.pack_int(r.mods.value) - data += self.pack_string(r.life_bar_graph) - data += self.pack_timestamp() - data += self.pack_replay_data() + data += self.pack_life_bar(r.life_bar_graph) + data += self.pack_timestamp(r.timestamp) + data += self.pack_replay_data(r.replay_data) data += self.pack_long(r.replay_id) return data - @dataclass class Replay: """ @@ -272,7 +311,7 @@ class Replay: max_combo: int perfect: bool mods: Mod - life_bar_graph: Optional[str] + life_bar_graph: Optional[List[LifeBarState]] timestamp: datetime replay_data: List[ReplayEvent] replay_id: int @@ -375,7 +414,6 @@ def pack(self, *, dict_size=None, mode=None): """ return _Packer(self, dict_size=dict_size, mode=mode).pack() - def parse_replay_data(data_string, *, decoded=False, decompressed=False, mode=GameMode.STD) -> List[ReplayEvent]: """ diff --git a/osrparse/utils.py b/osrparse/utils.py index dfe3aa3..8986445 100644 --- a/osrparse/utils.py +++ b/osrparse/utils.py @@ -103,3 +103,8 @@ class ReplayEventCatch(ReplayEvent): @dataclass class ReplayEventMania(ReplayEvent): keys: KeyMania + +@dataclass +class LifeBarState: + time: int + life: float \ No newline at end of file From 5a7bc0b2da35a4e879a2d0975141d34da1c80167 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Jan 2022 14:03:37 -0500 Subject: [PATCH 14/24] linting fix --- osrparse/replay.py | 37 ++++++++++++++++++++++--------------- osrparse/utils.py | 2 +- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index 79810ec..6c9d861 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -9,6 +9,7 @@ ReplayEventCatch, ReplayEventMania, ReplayEventTaiko, Key, KeyMania, KeyTaiko, LifeBarState) + class _Unpacker: """ Helper class for dealing with the ``.osr`` format. Not intended to be used @@ -118,13 +119,14 @@ def unpack_life_bar(self): if not data: return [] - + data = [x.split(",") for x in data.split("|")] - - # it seems like many replays have a graph starting with time, without the life state - # and then at the end of the graph, the life state without a time is there - # just in case, i am checking for them and adding None to values that are lacking - + + # it seems like many replays have a graph starting with time, without + # the life state and then at the end of the graph, the life state + # without a time is there just in case, i am checking for them and + # adding None to values that are lacking + for index, state in enumerate(data): if index == 0: graph.append(LifeBarState(int(state[0]), None)) @@ -132,9 +134,9 @@ def unpack_life_bar(self): graph.append(LifeBarState(None, int(state[0]))) else: graph.append(LifeBarState(int(state[1]), float(state[0]))) - + return graph - + def unpack(self): mode = GameMode(self.unpack_once("b")) game_version = self.unpack_once("i") @@ -166,6 +168,7 @@ def unpack(self): count_miss, score, max_combo, perfect, mods, life_bar_graph, timestamp, play_data, replay_id, rng_seed) + class _Packer: def __init__(self, replay, *, dict_size=None, mode=None): self.replay = replay @@ -209,7 +212,7 @@ def pack_timestamp(self, date): # 62135596800 is the number of seconds between these two years and is # added to account for this difference. # The factor of 10000000 converts seconds to ticks. - + ticks = (62135596800 + date.timestamp()) * 10000000 ticks = int(ticks) return self.pack_long(ticks) @@ -220,19 +223,21 @@ def pack_life_bar(self, graph): if state.life is None: text += f"{state.time}|" continue - - if int(state.life) == state.life: # checking if time is actually a fraction (osu! wants an integer for 0 or 1) + + # checking if time is actually a fraction (osu! wants an integer for + # 0 or 1) + if int(state.life) == state.life: life = int(state.life) else: life = state.life - + if state.time is None: text += f"{life}," else: text += f"{life},{state.time}|" - + return self.pack_string(text) - + def pack_replay_data(self, replay_data): data = "" for event in replay_data: @@ -256,7 +261,7 @@ def pack_replay_data(self, replay_data): "mode": self.mode } ] - + data = data.encode("ascii") compressed = lzma.compress(data, format=lzma.FORMAT_ALONE, filters=filters) @@ -289,6 +294,7 @@ def pack(self): return data + @dataclass class Replay: """ @@ -414,6 +420,7 @@ def pack(self, *, dict_size=None, mode=None): """ return _Packer(self, dict_size=dict_size, mode=mode).pack() + def parse_replay_data(data_string, *, decoded=False, decompressed=False, mode=GameMode.STD) -> List[ReplayEvent]: """ diff --git a/osrparse/utils.py b/osrparse/utils.py index 8986445..b577002 100644 --- a/osrparse/utils.py +++ b/osrparse/utils.py @@ -107,4 +107,4 @@ class ReplayEventMania(ReplayEvent): @dataclass class LifeBarState: time: int - life: float \ No newline at end of file + life: float From 7f5a8db45f00ea9cc3a51290c43ad1c7394d99bb Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Jan 2022 14:17:17 -0500 Subject: [PATCH 15/24] fix life bar parsing --- osrparse/replay.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index 6c9d861..a93a96e 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -114,28 +114,16 @@ def unpack_replay_id(self): return replay_id def unpack_life_bar(self): - data = self.unpack_string() - graph = [] + life_bar = self.unpack_string() - if not data: + if not life_bar: return [] - data = [x.split(",") for x in data.split("|")] - - # it seems like many replays have a graph starting with time, without - # the life state and then at the end of the graph, the life state - # without a time is there just in case, i am checking for them and - # adding None to values that are lacking - - for index, state in enumerate(data): - if index == 0: - graph.append(LifeBarState(int(state[0]), None)) - elif index == len(data) - 1: - graph.append(LifeBarState(None, int(state[0]))) - else: - graph.append(LifeBarState(int(state[1]), float(state[0]))) + # remove trailing comma to make splitting easier + life_bar = life_bar[:-1] + states = [state.split("|") for state in life_bar.split(",")] - return graph + return [LifeBarState(int(s[0]), float(s[1])) for s in states] def unpack(self): mode = GameMode(self.unpack_once("b")) From fc85e96e398ba6f9d96923560695c5ff91b4c58f Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Jan 2022 14:35:24 -0500 Subject: [PATCH 16/24] fix life bar packing, return none instead of empty list --- osrparse/replay.py | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index a93a96e..b9c37fb 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -115,9 +115,8 @@ def unpack_replay_id(self): def unpack_life_bar(self): life_bar = self.unpack_string() - if not life_bar: - return [] + return None # remove trailing comma to make splitting easier life_bar = life_bar[:-1] @@ -195,40 +194,34 @@ def pack_string(self, data): data.encode("utf-8")) return self.pack_byte(11) + self.pack_byte(0) - def pack_timestamp(self, date): + def pack_timestamp(self): # windows ticks starts at year 0001, in contrast to unix time (1970). # 62135596800 is the number of seconds between these two years and is # added to account for this difference. # The factor of 10000000 converts seconds to ticks. - ticks = (62135596800 + date.timestamp()) * 10000000 + ticks = (62135596800 + self.replay.timestamp.timestamp()) * 10000000 ticks = int(ticks) return self.pack_long(ticks) - def pack_life_bar(self, graph): + def pack_life_bar(self): text = "" - for state in graph: - if state.life is None: - text += f"{state.time}|" - continue - - # checking if time is actually a fraction (osu! wants an integer for - # 0 or 1) - if int(state.life) == state.life: + if self.replay.life_bar_graph is None: + return self.pack_string(text) + + for state in self.replay.life_bar_graph: + life = state.life + # store 0 or 1 instead of 0.0 or 1.0 + if int(life) == life: life = int(state.life) - else: - life = state.life - if state.time is None: - text += f"{life}," - else: - text += f"{life},{state.time}|" + text += f"{state.time}|{life}," return self.pack_string(text) - def pack_replay_data(self, replay_data): + def pack_replay_data(self): data = "" - for event in replay_data: + for event in self.replay.replay_data: t = event.time_delta if isinstance(event, ReplayEventOsu): data += f"{t}|{event.x}|{event.y}|{event.keys.value}," @@ -275,9 +268,9 @@ def pack(self): data += self.pack_short(r.max_combo) data += self.pack_byte(r.perfect) data += self.pack_int(r.mods.value) - data += self.pack_life_bar(r.life_bar_graph) - data += self.pack_timestamp(r.timestamp) - data += self.pack_replay_data(r.replay_data) + data += self.pack_life_bar() + data += self.pack_timestamp() + data += self.pack_replay_data() data += self.pack_long(r.replay_id) return data From 6721f48403fab856c6fb4ebdbad22ad9f496078e Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Jan 2022 14:36:48 -0500 Subject: [PATCH 17/24] text -> data --- osrparse/replay.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index b9c37fb..bd2bd46 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -205,9 +205,9 @@ def pack_timestamp(self): return self.pack_long(ticks) def pack_life_bar(self): - text = "" + data = "" if self.replay.life_bar_graph is None: - return self.pack_string(text) + return self.pack_string(data) for state in self.replay.life_bar_graph: life = state.life @@ -215,9 +215,9 @@ def pack_life_bar(self): if int(life) == life: life = int(state.life) - text += f"{state.time}|{life}," + data += f"{state.time}|{life}," - return self.pack_string(text) + return self.pack_string(data) def pack_replay_data(self): data = "" From 53920e0c6f76bee54c991f5c26b726ddbcf276df Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Jan 2022 14:39:01 -0500 Subject: [PATCH 18/24] parse rng seed before converting to ReplayEvent classes --- osrparse/replay.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/osrparse/replay.py b/osrparse/replay.py index 4729650..f293de4 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -66,9 +66,9 @@ def unpack_play_data(self, mode): data = self.replay_data[self.offset:offset_end] data = lzma.decompress(data, format=lzma.FORMAT_AUTO) data = data.decode("ascii") - replay_data = self.parse_replay_data(data, mode) + (replay_data, rng_seed) = self.parse_replay_data(data, mode) self.offset = offset_end - return replay_data + return (replay_data, rng_seed) @staticmethod def parse_replay_data(replay_data_str, mode): @@ -76,6 +76,7 @@ def parse_replay_data(replay_data_str, mode): replay_data_str = replay_data_str[:-1] events = [event.split('|') for event in replay_data_str.split(',')] + rng_seed = None play_data = [] for event in events: time_delta = int(event[0]) @@ -83,6 +84,10 @@ def parse_replay_data(replay_data_str, mode): y = event[2] keys = int(event[3]) + if time_delta == -12345 and event == events[-1]: + rng_seed = keys + continue + if mode is GameMode.STD: keys = Key(keys) event = ReplayEventOsu(time_delta, float(x), float(y), keys) @@ -92,9 +97,10 @@ def parse_replay_data(replay_data_str, mode): event = ReplayEventCatch(time_delta, float(x), int(keys) == 1) if mode is GameMode.MANIA: event = ReplayEventMania(time_delta, KeyMania(keys)) + play_data.append(event) - return play_data + return (play_data, rng_seed) def unpack_replay_id(self): # old replays had replay_id stored as a short (4 bytes) instead of a @@ -131,18 +137,13 @@ def unpack(self): mods = Mod(self.unpack_once("i")) life_bar_graph = self.unpack_string() timestamp = self.unpack_timestamp() - play_data = self.unpack_play_data(mode) + (replay_data, rng_seed) = self.unpack_play_data(mode) replay_id = self.unpack_replay_id() - rng_seed = None - if play_data[-1].time_delta == -12345: - rng_seed = play_data[-1].keys.value - del play_data[-1] - return Replay(mode, game_version, beatmap_hash, username, replay_hash, count_300, count_100, count_50, count_geki, count_katu, count_miss, score, max_combo, perfect, mods, life_bar_graph, - timestamp, play_data, replay_id, rng_seed) + timestamp, replay_data, replay_id, rng_seed) class _Packer: @@ -414,4 +415,5 @@ def parse_replay_data(data_string, *, decoded=False, decompressed=False, if not decompressed: data_string = lzma.decompress(data_string, format=lzma.FORMAT_AUTO) data_string = data_string.decode("ascii") - return _Unpacker.parse_replay_data(data_string, mode) + (replay_data, _seed) = _Unpacker.parse_replay_data(data_string, mode) + return replay_data From 5e8b65588c21a34e381444b6c647a7d2214c7505 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Wed, 12 Jan 2022 12:44:13 -0500 Subject: [PATCH 19/24] add docs to utils --- osrparse/utils.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/osrparse/utils.py b/osrparse/utils.py index b577002..2926940 100644 --- a/osrparse/utils.py +++ b/osrparse/utils.py @@ -2,12 +2,18 @@ from dataclasses import dataclass class GameMode(Enum): + """ + An osu! game mode. + """ STD = 0 TAIKO = 1 CTB = 2 MANIA = 3 class Mod(IntFlag): + """ + An osu! mod, or combination of mods. + """ NoMod = 0 NoFail = 1 << 0 Easy = 1 << 1 @@ -42,6 +48,10 @@ class Mod(IntFlag): Mirror = 1 << 30 class Key(IntFlag): + """ + A key that can be pressed during osu!standard gameplay - mouse 1 and 2, key + 1 and 2, and smoke. + """ M1 = 1 << 0 M2 = 1 << 1 K1 = 1 << 2 @@ -49,12 +59,18 @@ class Key(IntFlag): SMOKE = 1 << 4 class KeyTaiko(IntFlag): + """ + A key that can be pressed during osu!taiko gameplay. + """ LEFT_DON = 1 << 0 LEFT_KAT = 1 << 1 RIGHT_DON = 1 << 2 RIGHT_KAT = 1 << 3 class KeyMania(IntFlag): + """ + A key that can be pressed during osu!mania gameplay + """ K1 = 1 << 0 K2 = 1 << 1 K3 = 1 << 2 @@ -80,16 +96,25 @@ class KeyMania(IntFlag): @dataclass class ReplayEvent: + """ + Base class for an event (ie a frame) in a replay. + """ time_delta: int @dataclass class ReplayEventOsu(ReplayEvent): + """ + A single frame in an osu!standard replay. + """ x: float y: float keys: Key @dataclass class ReplayEventTaiko(ReplayEvent): + """ + A single frame in an osu!taiko replay. + """ # we have no idea what this is supposed to represent. It's always one of 0, # 320, or 640, depending on `keys`. Leaving untouched for now. x: int @@ -97,14 +122,23 @@ class ReplayEventTaiko(ReplayEvent): @dataclass class ReplayEventCatch(ReplayEvent): + """ + A single frame in an osu!catch replay. + """ x: float dashing: bool @dataclass class ReplayEventMania(ReplayEvent): + """ + A single frame in an osu!mania replay. + """ keys: KeyMania @dataclass class LifeBarState: + """ + A state of the lifebar shown on the results screen. + """ time: int life: float From 38179b1776ec60d6387582196a242e1181c6891b Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Wed, 12 Jan 2022 13:27:45 -0500 Subject: [PATCH 20/24] write full / proper documentation --- Makefile | 20 ++++++ README.md | 145 +++------------------------------------ docs/appendix.rst | 12 ++++ docs/conf.py | 66 ++++++++++++++++++ docs/index.rst | 50 ++++++++++++++ docs/parsing-replays.rst | 52 ++++++++++++++ docs/writing-replays.rst | 34 +++++++++ 7 files changed, 243 insertions(+), 136 deletions(-) create mode 100644 Makefile create mode 100644 docs/appendix.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/parsing-replays.rst create mode 100644 docs/writing-replays.rst diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b97de95 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = docs +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/README.md b/README.md index c1f82c0..d52db00 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,13 @@ pip install osrparse ## Documentation -### Parsing +Please see the full documentation for a comprehensive guide: . A quickstart follows below, for the impatient, but you should read the full documentation regardless. -To parse a replay: +### Quickstart ```python -from osrparse import Replay +from osrparse import Replay, parse_replay_data +# parse from a path replay = Replay.from_path("path/to/osr.osr") # or from an opened file object @@ -31,38 +32,13 @@ with open("path/to/osr.osr") as f: with open("path/to/osr.osr") as f: replay_string = f.read() replay = Replay.from_string(replay_string) -``` - -To parse only the `replay_data` portion of a `Replay`, such as the data returned from the api `/get_replay` endpoint: - -```python -from osrparse import parse_replay_data -import base64 -import lzma +# parse the replay data from api v1's /get_replay endpoint lzma_string = retrieve_from_api() replay_data = parse_replay_data(lzma_string) -assert isinstance(replay_data[0], ReplayEvent) - -# or parse an already decoded lzma string -lzma_string = retrieve_from_api() -lzma_string = base64.b64decode(lzma_string) -replay_data = parse_replay_data(lzma_string, decoded=True) - -# or parse an already decoded and decompressed lzma string -lzma_string = retrieve_from_api() -lzma_string = base64.b64decode(lzma_string) -lzma_string = lzma.decompress(lzma_string).decode("ascii") -replay_data = parse_replay_data(lzma_string, decompressed=True) -``` - -The response returned from `/get_replay` is base 64 encoded, which is why we provide automatic decoding in `parse_replay_data`. If you are retrieving this data from a different source where the replay data is already decoded, pass `decoded=True`. - -### Writing +# replay_data is a list of ReplayEvents -Existing `Replay` objects can be written back to `.osr` files: - -```python +# write a replay back to a path replay.write_path("path/to/osr.osr") # or to an opened file object @@ -71,111 +47,8 @@ with open("path/to/osr.osr") as f: # or to a string packed = replay.pack() -``` -You can also edit osr files by parsing a replay, editing an attribute, and dumping it back to its file: - -```python -replay = Replay.from_path("path/to/osr.osr") +# edited attributes are saved replay.username = "fake username" -replay.write_path("path/to/osr.osr") -``` - -### Attributes - -`Replay` objects have the following attibutes: - -```python -self.mode # GameMode -self.game_version # int -self.beatmap_hash # str -self.username # str -self.replay_hash # str -self.count_300 # int -self.count_100 # int -self.count_50 # int -self.count_geki # int -self.count_katu # int -self.count_miss # int -self.score # int -self.max_combo # int -self.perfect # bool -self.mods # Mod -self.life_bar_graph # str or None -self.timestamp # datetime -self.replay_data # List[ReplayEvent] -self.replay_id # int -self.rng_seed # int or None -# list of either ReplayEventOsu, ReplayEventTaiko, ReplayEventCatch, -# or ReplayEventMania objects, depending on self.game_mode -self.replay_data -``` - -`ReplayEventOsu` objects have the following attributes: - -```python -self.time_delta # int, time since previous event in milliseconds -self.x # float, x axis location -self.y # float, y axis location -self.keys # Key enum, keys pressed -``` - -`ReplayEventTaiko` objects have the following attributes: - -```python -self.time_delta # int, time since previous event in milliseconds -self.x # float, x axis location -self.keys # KeyTaiko enum, keys pressed -``` - -`ReplayEventCatch` objects have the following attributes: - -```python -self.time_delta # int, time since previous event in milliseconds -self.x # float, x axis location -self.dashing # bool, whether the player was dashing or not -``` - -`ReplayEventMania` objects have the following attributes: - -```python -self.time_delta # int, time since previous event in milliseconds -self.keys # KeyMania enum, keys pressed -``` - -The `Key` enums used in the above `ReplayEvent`s are defined as follows: - -```python -class Key(IntFlag): - M1 = 1 << 0 - M2 = 1 << 1 - K1 = 1 << 2 - K2 = 1 << 3 - SMOKE = 1 << 4 - -class KeyTaiko(IntFlag): - LEFT_DON = 1 << 0 - LEFT_KAT = 1 << 1 - RIGHT_DON = 1 << 2 - RIGHT_KAT = 1 << 3 - -class KeyMania(IntFlag): - K1 = 1 << 0 - K2 = 1 << 1 - K3 = 1 << 2 - K4 = 1 << 3 - K5 = 1 << 4 - K6 = 1 << 5 - K7 = 1 << 6 - K8 = 1 << 7 - K9 = 1 << 8 - K10 = 1 << 9 - K11 = 1 << 10 - K12 = 1 << 11 - K13 = 1 << 12 - K14 = 1 << 13 - K15 = 1 << 14 - K16 = 1 << 15 - K17 = 1 << 16 - K18 = 1 << 17 +replay.write_path("path/to/new_osr.osr") ``` diff --git a/docs/appendix.rst b/docs/appendix.rst new file mode 100644 index 0000000..3ba71f2 --- /dev/null +++ b/docs/appendix.rst @@ -0,0 +1,12 @@ +Appendix +======== + +Replay +------ +.. automodule:: osrparse.replay + :members: + +Utils +----- +.. automodule:: osrparse.utils + :members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..54d8fbd --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,66 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +from osrparse import __version__ + +project = "osrparse" +copyright = "2022, Kevin Lim, Liam DeVoe" +author = "Kevin Lim, Liam DeVoe" +release = "v" + __version__ +version = "v" + __version__ +master_doc = 'index' + +# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_show_copyright +html_show_copyright = False +# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_show_sphinx +html_show_sphinx = False + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx.ext.todo" +] + +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} +# https://stackoverflow.com/a/37210251 +autodoc_member_order = "bysource" + +html_theme = "furo" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ["_static"] + +# references that we want to use easily in any file +rst_prolog = """ +.. |Replay| replace:: :class:`~osrparse.replay.Replay` +.. |from_path| replace:: :func:`Replay.from_path() ` +.. |from_file| replace:: :func:`Replay.from_file() ` +.. |from_string| replace:: :func:`Replay.from_string() ` +.. |write_path| replace:: :func:`Replay.write_path() ` +.. |write_file| replace:: :func:`Replay.write_file() ` +.. |pack| replace:: :func:`Replay.pack() ` +.. |parse_replay_data| replace:: :func:`parse_replay_data() ` + +.. |br| raw:: html + +
+""" + +# linebreak workaround documented here +# https://stackoverflow.com/a/9664844/12164878 diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..207bd90 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,50 @@ +osrparse +========== + +osrparse is a parse for the ``.osr`` format described `here `__. + +osrparse is maintained by: + +* `tybug `__ +* `kszlim `__ + +Installation +------------ + +osrparse can be installed from pip: + +.. code-block:: console + + $ pip install osrparse + +Links +----- + +| Github: https://github.com/kszlim/osu-replay-parser +| Documentation: https://kevin-lim.ca/osu-replay-parser/ + + +.. + couple notes about these toctrees - the first toctree is so our sidebar has + a link back to the index page. the ``self`` keyword comes with its share of + issues (https://github.com/sphinx-doc/sphinx/issues/2103), but none that matter + that much to us. It's better than using ``index`` which works but generates + many warnings when building. + + Hidden toctrees appear on the sidebar but not as text on the table of contents + displayed on this page. + +Contents +-------- + +.. toctree:: + :hidden: + + self + +.. toctree:: + :maxdepth: 2 + + parsing-replays + writing-replays + appendix diff --git a/docs/parsing-replays.rst b/docs/parsing-replays.rst new file mode 100644 index 0000000..e2aa3c0 --- /dev/null +++ b/docs/parsing-replays.rst @@ -0,0 +1,52 @@ +Parsing Replays +=============== + +Creating a Replay +----------------- + +Depending on the type of data you have, a |Replay| can be created multiple ways, using either one of |from_path|, |from_file|, or |from_string|: + +.. code-block:: python + + from osrparse import Replay + # from a path + replay = Replay.from_path("path/to/osr.osr") + + # or from an opened file object + with open("path/to/osr.osr") as f: + replay = Replay.from_file(f) + + # or from a string + with open("path/to/osr.osr") as f: + replay_string = f.read() + replay = Replay.from_string(replay_string) + +Most likely, you will be using |from_path| to create a |Replay|. + +Parsing Just Replay Data +------------------------ + +Unfortunately, the `/get_replay `__ endpoint of `osu!api v1 `__ does not return the full contents of a replay, but only the replay data potion. This means that you cannot create a full replay from the response of this endpoint. + +For this, we provide |parse_replay_data|, a function that takes the response of this endpoint and returns List[:class:`~osrparse.utils.ReplayEvent`] (ie, the parsed replay data): + +.. code-block:: python + + from osrparse import parse_replay_data + import base64 + import lzma + + lzma_string = retrieve_from_api() + replay_data = parse_replay_data(lzma_string) + assert isinstance(replay_data[0], ReplayEvent) + + # or parse an already decoded lzma string + lzma_string = retrieve_from_api() + lzma_string = base64.b64decode(lzma_string) + replay_data = parse_replay_data(lzma_string, decoded=True) + + # or parse an already decoded and decompressed lzma string + lzma_string = retrieve_from_api() + lzma_string = base64.b64decode(lzma_string) + lzma_string = lzma.decompress(lzma_string).decode("ascii") + replay_data = parse_replay_data(lzma_string, decompressed=True) diff --git a/docs/writing-replays.rst b/docs/writing-replays.rst new file mode 100644 index 0000000..d160160 --- /dev/null +++ b/docs/writing-replays.rst @@ -0,0 +1,34 @@ +Writing Replays +=============== + +Writing a Replay +---------------- + +Just as replays can be parsed from a path, file, or string, they can also be written back to a path, file, or string, with |write_path|, |write_file|, and |pack| respectively: + + +.. code-block:: python + + replay.write_path("path/to/new_osr.osr") + + # or to an opened file object + with open("path/to/new_osr.osr") as f: + replay.write_file(f) + + # or to a string + packed = replay.pack() + +Editing a Replay +---------------- + +The writing facilities of osrparse can be used to parse a replay, edit some or all of its attributes, and write it back to its file. The result is an edited replay. + +For instance, to change the username of a replay: + +.. code-block:: python + + from osrparse import Replay + + replay = Replay.from_path("path/to/osr.osr") + replay.username = "fake username" + replay.write_path("path/to/osr.osr") From 24aa7629c949d6193ccfd8fe7175b98008534c0e Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Wed, 12 Jan 2022 13:28:19 -0500 Subject: [PATCH 21/24] reword intro --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d52db00..ef6d274 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ pip install osrparse ## Documentation -Please see the full documentation for a comprehensive guide: . A quickstart follows below, for the impatient, but you should read the full documentation regardless. +Please see the full documentation for a comprehensive guide: . A quickstart follows below, for the impatient, but you should read the full documentation if you are at all confused. ### Quickstart From 7c23a44b7437b3c44ea65d66729eeacacf0138ee Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Wed, 12 Jan 2022 13:28:35 -0500 Subject: [PATCH 22/24] fix grammar --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef6d274..685b581 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ pip install osrparse ## Documentation -Please see the full documentation for a comprehensive guide: . A quickstart follows below, for the impatient, but you should read the full documentation if you are at all confused. +Please see the full documentation for a comprehensive guide: . A quickstart follows below for the impatient, but you should read the full documentation if you are at all confused. ### Quickstart From 1cb138aef55d58fb2a524d552c05c3f0af777d0b Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Wed, 12 Jan 2022 13:43:00 -0500 Subject: [PATCH 23/24] document dataclass members --- README.md | 7 ++++++ osrparse/replay.py | 43 +++++++++++++++++++++++++++++++++++++ osrparse/utils.py | 53 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 685b581..12c85c5 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,13 @@ with open("path/to/osr.osr") as f: replay_string = f.read() replay = Replay.from_string(replay_string) +# a replay has various attributes +r = replay +print(r.mode, r.game_version, r.beatmap_hash, r.username, + r.r_hash, r.count_300, r.count_100, r.count_50, r.count_geki, + r.count_miss, r.score, r.max_combo, r.perfect, r.mods, + r.life_bar_graph, r.timestamp, r.r_data, r.r_id, r.rng_seed) + # parse the replay data from api v1's /get_replay endpoint lzma_string = retrieve_from_api() replay_data = parse_replay_data(lzma_string) diff --git a/osrparse/replay.py b/osrparse/replay.py index b245f93..5852d13 100644 --- a/osrparse/replay.py +++ b/osrparse/replay.py @@ -283,6 +283,49 @@ class Replay: A replay found in a ``.osr`` file, or following the osr format. To create a replay, you likely want ``Replay.from_path``, ``Replay.from_file``, or ``Replay.from_string``. + + Attributes + ---------- + mode: GameMode + The game mode this replay was played on. + game_version: int + The game version this replay was played on. + beatmap_hash: str + The hash of the beatmap this replay was played on. + username: str + The user that played this replay. + replay_hash: + The hash of this replay. + count_300: int + The number of 300 judgments in this replay. + count_100: int + The number of 100 judgments in this replay. + count_50: int + The number of 50 judgments in this replay. + count_geki: int + The number of geki judgments in this replay. + count_katu: int + The number of katu judgments in this replay. + count_miss: int + The number of misses in this replay. + score: int + The score of this replay. + max_combo: int + The maximum combo attained in this replay. + perfect: bool + Whether this replay was perfect or not. + mods: Mod + The mods this replay was played with. + life_bar_graph: Optional[List[LifeBarState]] + The life bar of this replay over time. + replay_data: List[ReplayEvent] + The replay data of the replay, including cursor position and keys + pressed. + replay_id: int + The replay id of this replay, or 0 if not submitted. + rng_seed: Optional[int] + The rng seed of this replay, or ``None`` if not present (typically not + present on older replays). """ mode: GameMode game_version: int diff --git a/osrparse/utils.py b/osrparse/utils.py index 2926940..6f0c42e 100644 --- a/osrparse/utils.py +++ b/osrparse/utils.py @@ -98,6 +98,11 @@ class KeyMania(IntFlag): class ReplayEvent: """ Base class for an event (ie a frame) in a replay. + + Attributes + ---------- + time_delta: int + The time since the previous event (ie frame). """ time_delta: int @@ -105,6 +110,17 @@ class ReplayEvent: class ReplayEventOsu(ReplayEvent): """ A single frame in an osu!standard replay. + + Attributes + ---------- + time_delta: int + The time since the previous event (ie frame). + x: float + The x position of the cursor. + y: float + The y position of the cursor. + keys: Key + The keys pressed. """ x: float y: float @@ -114,6 +130,16 @@ class ReplayEventOsu(ReplayEvent): class ReplayEventTaiko(ReplayEvent): """ A single frame in an osu!taiko replay. + + Attributes + ---------- + time_delta: int + The time since the previous event (ie frame). + x: int + Unknown what this represents. Always one of 0, 320, or 640, depending on + ``keys``. + keys: KeyTaiko + The keys pressed. """ # we have no idea what this is supposed to represent. It's always one of 0, # 320, or 640, depending on `keys`. Leaving untouched for now. @@ -124,6 +150,15 @@ class ReplayEventTaiko(ReplayEvent): class ReplayEventCatch(ReplayEvent): """ A single frame in an osu!catch replay. + + Attributes + ---------- + time_delta: int + The time since the previous event (ie frame). + x: float + The x position of the player. + dashing: bool + Whether we are dashing or not. """ x: float dashing: bool @@ -132,13 +167,29 @@ class ReplayEventCatch(ReplayEvent): class ReplayEventMania(ReplayEvent): """ A single frame in an osu!mania replay. + + Attributes + ---------- + time_delta: int + The time since the previous event (ie frame). + keys: KeyMania + The keys pressed. """ keys: KeyMania @dataclass class LifeBarState: """ - A state of the lifebar shown on the results screen. + A state of the lifebar shown on the results screen, at a particular point in + time. + + Attributes + ---------- + time: int + The time, in ms, this life bar state corresponds to in the replay. + The time since the previous event (ie frame). + life: float + The amount of life at this life bar state. """ time: int life: float From 70dee37439cf6f63076d20a91d0240d3ccae27be Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Wed, 12 Jan 2022 13:44:42 -0500 Subject: [PATCH 24/24] reword osu wiki links --- README.md | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 12c85c5..67f4208 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # osrparse, a python parser for osu! replays -This is a parser for osu! replay files (.osr) as described by . +This is a parser for the ``.osr`` format for osu! replay files, as described by [the wiki](https://osu.ppy.sh/wiki/en/Client/File_formats/Osr_%28file_format%29). ## Installation diff --git a/docs/index.rst b/docs/index.rst index 207bd90..74b4c1f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ osrparse ========== -osrparse is a parse for the ``.osr`` format described `here `__. +osrparse is a parser for the ``.osr`` format, as described `on the osu! wiki `__. osrparse is maintained by: