msdparser 2.0.0b3 integration

Pipfile

@@ -12,8 +12,12 @@ twine = "*"
 mypy = "*"
 
 [packages]
-msdparser = ">=1.0.0"
+msdparser = "==2.0.0b3"
 pyfakefs = "*"
 
 [requires]
 python_version = "3.7"
+allow_prereleases = true
+
+[pipenv]
+allow_prereleases = true

simfile/__init__.py

@@ -9,14 +9,14 @@
 import builtins
 from contextlib import contextmanager
 from io import StringIO
+from itertools import tee
 from typing import Iterator, List, Optional, TextIO, Tuple, Union
 
 from msdparser import parse_msd
 
 from .ssc import SSCSimfile
 from .sm import SMSimfile
 from .types import Simfile
-from ._private.tee_file import tee_file
 
 
 __version__ = '2.0.0'
@@ -30,23 +30,34 @@
 
 
 def _detect_ssc(
-    peek_file: Union[TextIO, Iterator[str]],
+    file: Union[TextIO, Iterator[str]],
     strict: bool = True
-) -> bool:
-    if isinstance(peek_file, TextIO) and type(peek_file.name) is str:
-        _, _, suffix = peek_file.name.rpartition('.')
-        if suffix == '.ssc':
-            return True
-        elif suffix == '.sm':
-            return False
+) -> Tuple[Union[TextIO, Iterator[str]], bool]:
+    if isinstance(file, TextIO):
+        if type(file.name) is str:
+            _, _, suffix = file.name.rpartition('.')
+            if suffix == '.ssc':
+                return (file, True)
+            elif suffix == '.sm':
+                return (file, False)
+        parser = parse_msd(file=file, ignore_stray_text=not strict)
+    else:
+        file, peek_file = [StringIO(''.join(f)) for f in tee(file)]
+        parser = parse_msd(
+            string=''.join(peek_file),
+            ignore_stray_text=not strict,
+        )
 
     # Check if the first property is an SSC version
-    parser = parse_msd(file=peek_file, ignore_stray_text=not strict)
-    first_key = ''
-    for first_key, _ in parser:
-        break
+    try:
+        first_param = next(parser)
+    except StopIteration:
+        return (file, False)
+
+    if isinstance(file, TextIO):
+        file.seek(0)
 
-    return first_key.upper() == 'VERSION'
+    return (file, first_param.key is not None and first_param.key.upper() == 'VERSION')
 
 
 def load(file: Union[TextIO, Iterator[str]], strict: bool = True) -> Simfile:
@@ -59,8 +70,7 @@ def load(file: Union[TextIO, Iterator[str]], strict: bool = True) -> Simfile:
     the file is treated as an SSC simfile; otherwise, it's treated as
     an SM simfile.
     """
-    peek_file, file = tee_file(file)
-    is_ssc = _detect_ssc(peek_file)
+    file, is_ssc = _detect_ssc(file)
     if is_ssc:
         return SSCSimfile(file=file, strict=strict)
     else:
@@ -145,7 +155,7 @@ def open_with_detected_encoding(
             continue
 
     # If all encodings failed, raise the exception chain
-    raise exception or UnicodeDecodeError
+    raise exception or UnicodeError
 
 
 class CancelMutation(BaseException):

simfile/base.py

@@ -7,9 +7,10 @@
 """
 from abc import ABCMeta, abstractclassmethod, abstractmethod
 from collections import OrderedDict
+from io import StringIO
 from typing import Iterator, Optional, TextIO, Tuple, Union
 
-from msdparser import parse_msd
+from msdparser import parse_msd, MSDParameter
 
 from ._private.generic import E, ListWithRepr
 from ._private.property import item_property
@@ -19,7 +20,7 @@
 __all__ = ['BaseChart', 'BaseCharts', 'BaseSimfile']
 
 
-MSD_ITERATOR = Iterator[Tuple[str, str]]
+MSD_ITERATOR = Iterator[MSDParameter]
 
 
 class BaseChart(OrderedDict, Serializable, metaclass=ABCMeta):
@@ -102,6 +103,8 @@ class BaseSimfile(OrderedDict, Serializable, metaclass=ABCMeta):
     finds any stray text between parameters. This behavior can be
     overridden by setting `strict` to False in the constructor.
     """
+    MULTI_VALUE_PROPERTIES = ('ATTACKS', 'DISPLAYBPM')
+
     title = item_property('TITLE')
     subtitle = item_property('SUBTITLE')
     artist = item_property('ARTIST')
@@ -135,9 +138,18 @@ def __init__(self, *,
                  file: Optional[Union[TextIO, Iterator[str]]] = None,
                  string: Optional[str] = None,
                  strict: bool = True):
+        # msdparser no longer supports Iterator[str] as a file-like object
+        # but simfile does for backwards compatibility
+        file_for_msdparser = None
+        if file:
+            if isinstance(file, TextIO):
+                file_for_msdparser = file
+            else:
+                file_for_msdparser = StringIO(''.join(file))
+
         if file is not None or string is not None:
             self._parse(parse_msd(
-                file=file,
+                file=file_for_msdparser,
                 string=string,
                 ignore_stray_text=not strict,
             ))
@@ -157,7 +169,11 @@ def blank(cls):
 
     def serialize(self, file: TextIO):
         for (key, value) in self.items():
-            file.write(f'#{key}:{value};\n')
+            if key in BaseSimfile.MULTI_VALUE_PROPERTIES:
+                param = MSDParameter((key, *value.split(':')))
+            else:
+                param = MSDParameter((key, value))
+            file.write(f'{param}\n')
         file.write('\n')
         self.charts.serialize(file)
 

simfile/sm.py

@@ -1,8 +1,9 @@
 """
 Simfile & chart classes for SM files.
 """
+from msdparser import MSDParameter
 from simfile._private.property import item_property
-from typing import Iterator, List, Optional, Tuple, Type
+from typing import Iterator, List, Optional, Sequence, Type
 
 from .base import BaseChart, BaseCharts, BaseSimfile, MSD_ITERATOR
 from ._private.dedent import dedent_and_trim
@@ -48,22 +49,44 @@ def blank(cls: Type['SMChart']) -> 'SMChart':
     @classmethod
     def from_str(cls: Type['SMChart'], string: str) -> 'SMChart':
         """
-        Parse the MSD value component of a NOTES property.
+        Parse the serialized MSD value components of a NOTES property.
 
         The string should contain six colon-separated components,
-        corresponding to each of the base known properties documented
-        in :class:`.BaseChart`. Any additional components will be
-        stored in :data:`extradata`.
+        corresponding to each of the base known properties documented in
+        :class:`.BaseChart`. Any additional components will be stored in
+        :data:`extradata`.
 
         Raises :code:`ValueError` if the string contains fewer than six
         components.
+
+        .. deprecated:: 2.1
+            This is now a less efficient version of :func:`from_msd`, which
+            interoperates better with ``msdparser`` version 2.0.
         """
         instance = cls()
         instance._from_str(string)
         return instance
 
+    @classmethod
+    def from_msd(cls: Type['SMChart'], values: Sequence[str]) -> 'SMChart':
+        """
+        Parse the MSD value components of a NOTES property.
+
+        The list should contain six strings, corresponding to each of the
+        base known properties documented in :class:`.BaseChart`. Any
+        additional components will be stored in :data:`extradata`.
+
+        Raises :code:`ValueError` if the list contains fewer than six
+        components.
+        """
+        instance = cls()
+        instance._from_msd(values)
+        return instance
+
     def _from_str(self, string: str) -> None:
-        values = string.split(':')
+        self._from_msd(string.split(':'))
+
+    def _from_msd(self, values: Sequence[str]) -> None:
         if len(values) < len(SM_CHART_PROPERTIES):
             raise ValueError(
                 f'expected at least {len(SM_CHART_PROPERTIES)} '
@@ -74,27 +97,27 @@ def _from_str(self, string: str) -> None:
             self[property] = value.strip()
 
         if len(values) > len(SM_CHART_PROPERTIES):
-            self.extradata = values[len(SM_CHART_PROPERTIES):]
+            self.extradata = list(values[len(SM_CHART_PROPERTIES):])
 
-    def _parse(self, parser: Iterator[Tuple[str, str]]):
-        property, value = next(parser)
-        if property.upper() != 'NOTES':
+    def _parse(self, parser: Iterator[MSDParameter]):
+        param = next(parser)
+        if param.key.upper() != 'NOTES':
             raise ValueError(f'expected a NOTES property, got {property}')
 
-        self._from_str(value)
+        self._from_msd(param.components[1:])
 
     def serialize(self, file):
-        file.write(
-            f'#NOTES:\n'
-            f'     {self.stepstype}:\n'
-            f'     {self.description}:\n'
-            f'     {self.difficulty}:\n'
-            f'     {self.meter}:\n'
-            f'     {self.radarvalues}:\n'
-            f'{self.notes}\n'
-            f'{":" + ":".join(self.extradata) if self.extradata else ""}'
-            f';'
-        )
+        param = MSDParameter((
+            'NOTES',
+            f'\n     {self.stepstype}',
+            f'\n     {self.description}',
+            f'\n     {self.difficulty}',
+            f'\n     {self.meter}',
+            f'\n     {self.radarvalues}',
+            f'\n{self.notes}\n',
+            *(self.extradata or []),
+        ))
+        file.write(str(param))
 
     def __eq__(self, other):
         return (type(self) is type(other) and
@@ -156,12 +179,14 @@ class SMSimfile(BaseSimfile):
 
     def _parse(self, parser: MSD_ITERATOR):
         self._charts = SMCharts()
-        for (key, value) in parser:
-            key = key.upper()
+        for param in parser:
+            key = param.key.upper()
             if key == 'NOTES':
-                self._charts.append(SMChart.from_str(value))
+                self._charts.append(SMChart.from_msd(param.components[1:]))
+            elif key in BaseSimfile.MULTI_VALUE_PROPERTIES:
+                self[key] = ':'.join(param.components[1:])
             else:                
-                self[key] = value
+                self[key] = param.value
 
     @classmethod
     def blank(cls: Type['SMSimfile']) -> 'SMSimfile':

simfile/ssc.py

@@ -3,7 +3,7 @@
 """
 from typing import Optional, Type
 
-from msdparser import parse_msd
+from msdparser import parse_msd, MSDParameter
 
 from .base import BaseChart, BaseCharts, BaseSimfile, MSD_ITERATOR
 from ._private.dedent import dedent_and_trim
@@ -95,25 +95,27 @@ def blank(cls: Type['SSCChart']) -> 'SSCChart':
     def _parse(self, parser: MSD_ITERATOR) -> None:
         iterator = iter(parser)
 
-        first_key, _ = next(iterator)
-        if first_key.upper() != 'NOTEDATA':
+        param = next(iterator)
+        if param.key.upper() != 'NOTEDATA':
             raise ValueError('expected NOTEDATA property first')
 
-        for key, value in iterator:
-            self[key] = value
-            if value is self.notes:
+        for param in iterator:
+            self[param.key] = param.value
+            if param.value is self.notes:
                 break
 
     def serialize(self, file):
-        file.write('#NOTEDATA:;\n')
+        file.write(f"{MSDParameter(('NOTEDATA', ''))}\n")
         notes_key = 'NOTES'
         for (key, value) in self.items():
             # notes must always be the last property in a chart
             if value is self.notes:
                 notes_key = key
                 continue
-            file.write(f'#{key}:{value};\n')
-        file.write(f'#{notes_key}:{self[notes_key]};\n\n')
+            param = MSDParameter((key, value))
+            file.write(f'{param}\n')
+        notes_param = MSDParameter((notes_key, self[notes_key]))
+        file.write(f'{notes_param}\n\n')
 
 
 class SSCCharts(BaseCharts[SSCChart]):
@@ -197,11 +199,15 @@ def blank(cls: Type['SSCSimfile']) -> 'SSCSimfile':
             #ATTACKS:;
         """))
 
-    def _parse(self, parser):
+    def _parse(self, parser: MSD_ITERATOR):
         self._charts = SSCCharts()
         partial_chart: Optional[SSCChart] = None
-        for key, value in parser:
-            key = key.upper()            
+        for param in parser:
+            key = param.key.upper()
+            if key not in BaseSimfile.MULTI_VALUE_PROPERTIES:
+                value: Optional[str] = ':'.join(param.components[1:])
+            else:
+                value = param.value
             if key == 'NOTEDATA':
                 if partial_chart is not None:
                     self._charts.append(partial_chart)

simfile/tests/test_sm.py

@@ -54,6 +54,13 @@ def test_serialize(self):
         unit = SMChart.from_str(testing_chart())
 
         self.assertEqual(f'#NOTES:{testing_chart()};', str(unit))
+
+    def test_serialize_with_escapes(self):
+        unit = SMChart.from_str(testing_chart())
+        unit.description = 'A;B//C\\D:E'
+        expected_substring = 'A\\;B\\//C\\\\D\\:E:\n'
+
+        self.assertIn(expected_substring, str(unit))
 
     def test_eq(self):
         variants = testing_charts()