From 2f2c713f09574cc3ac5af8e6f3d97cc629501b7b Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Sun, 8 Oct 2017 08:08:29 -0700 Subject: [PATCH 01/65] Convert requirements from exact to minimum Fixes #40 --- requirements.txt | 6 +++--- setup.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index a2c44fc..31fad6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -ply==3.10 -pyRFC3339==0.2 -pytz==2016.10 +ply>=3.10 +pyRFC3339>=0.2 +pytz>=2016.10 diff --git a/setup.py b/setup.py index 800a4be..f76f311 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,9 @@ long_description=open('README.md').read(), url="https://github.com/swaroopch/edn_format", install_requires=[ - "pytz==2016.10", - "pyRFC3339==0.2", - "ply==3.10", + "pytz>=2016.10", + "pyRFC3339>=0.2", + "ply>=3.10", ], license="Apache 2.0", packages=['edn_format'], From ec3239fb9078cf866690223d2d76cafe9aa0c6bc Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Sun, 8 Oct 2017 08:09:20 -0700 Subject: [PATCH 02/65] Bump version to 0.5.13 --- README.md | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d44ee6a..ecd5cf9 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ usable: To release a new version: -1. Bump up the version number in `setup.py`, e.g. `0.5.12` -2. Create a git tag: `git tag -a v0.5.12 -m 'Version 0.5.12'` +1. Bump up the version number in `setup.py`, e.g. `0.5.13` +2. Create a git tag: `git tag -a v0.5.13 -m 'Version 0.5.13'` 3. Push git tag: `git push origin master --tags` 4. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. 5. Run `python setup.py sdist upload` diff --git a/setup.py b/setup.py index f76f311..77e9864 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup setup(name="edn_format", - version="0.5.12", + version="0.5.13", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From c670ff278c7fc42eeac599cb7afca745ad229a34 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Thu, 2 Aug 2018 11:17:49 +0200 Subject: [PATCH 03/65] LICENSE: bump year --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 2337007..45e555d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright 2012-2017, Swaroop C H +Copyright 2012-2018, Swaroop C H Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 6d1a0ba6426479320d1257ca08f0211eea8c1254 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 4 Aug 2018 19:29:39 +0200 Subject: [PATCH 04/65] Fix ambiguous variable name (#44) --- edn_format/edn_lex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index 051ef0d..b79e900 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -258,7 +258,7 @@ def lex(text=None): kwargs = {} if __debug__: kwargs = dict(debug=True, debuglog=logging.getLogger(__name__)) - l = ply.lex.lex(reflags=re.UNICODE, **kwargs) + lex = ply.lex.lex(reflags=re.UNICODE, **kwargs) if text is not None: - l.input(text) - return l + lex.input(text) + return lex From aab186e4bc3baf96b93d2aae7ab1f4282000949d Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 4 Aug 2018 20:26:39 +0200 Subject: [PATCH 05/65] Add a Changelog --- CHANGELOG.md | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2047af8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,80 @@ +# `edn_format` Changelog + +## v0.5.13 (2017/10/08) + +* Convert requirements from exact to minimum + +## v0.5.12 (2017/02/14) + +* Unbreak tests on Python 2.x +* PEP8 style fixes + +## v0.5.11 (2017/02/14) + +* Add a `sort_keys` optional keyword argument to `dumps` +* Correctly parse floats with an exponent + +## v0.5.10 (2017/02/13) + +* Support string-\>keyword keys in `dumps` +* Add Travis config +* Bump dependencies + +## v0.5.9 (2015/07/09) + +* Add support to dump `#inst` with microseconds + +## v0.5.8 (2015/06/19) + +* Fix Python 2/3 support + +## v0.5.7 (2015/06/08) + +* Fix Python 3 `unichr`/`chr` incompatibility +* Changed Python version detection from exactly equal to 3 to greater than or + equal to 3 + +## v0.5.6 (2015/05/31) + +* Make UTF-8 the default expectation of text type + +## v0.5.5 (2015/05/25) + +* Fixed string parsing and escaping +* Unicode now consistently used internally +* New method `dumpu` added that returns unicode +* `dumps` method now takes optional encoding arguments: `output_encoding` + (specifies encoding of output string) and `string_encoding` (specifies + encoding of non-unicode strings in object to be serialized), both default to + `'utf-8'` +* `loads` now takes optional argument `input_encoding`, defaults to `'utf-8'` +* New method `loadu` added that assumes unicode input +* Bump dependencies + +## v0.5.4 (2015/02/26) + +* Fix parsing of booleans before comma + +## v0.5.3 (2014/05/03) + +## v0.5 (2014/01/25) + +* Fix `BaseEdnType.__hash__` +* Fix equality +* Handle escape sequences in strings +* Allow backslashed double-quotes in strings +* Add `<`, `>`, `@` + +## v0.4 (2014/01/24) + +* Parse empty collections +* Raise an exception on syntax error +* Improves patterns for `SYMBOL`, `TAG`, and `KEYWORD` to more closely match + the [EDN specifications][spec] +* Improve Python 3 support + +[spec]: https://github.com/edn-format/edn + +## v0.3.6 (2014/01/15) + +First published release. From bf69bb347ae27a1976a02a4848e94fde91ab48dc Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sun, 5 Aug 2018 11:41:14 +0200 Subject: [PATCH 06/65] Fix parsing of exact-precision floats with a negative exposant (#47) --- edn_format/edn_lex.py | 3 ++- tests.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index b79e900..a455525 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -206,7 +206,8 @@ def t_FLOAT(t): raise SyntaxError('Invalid float : {}'.format(t.value)) e_value = int(matches.group(1)) if t.value.endswith('M'): - t.value = decimal.Decimal(t.value[:-1]) * pow(1, e_value) + ctx = decimal.getcontext() + t.value = decimal.Decimal(t.value[:-1]) * ctx.power(1, e_value) else: t.value = float(t.value) * pow(1, e_value) return t diff --git a/tests.py b/tests.py index 6cbc3ca..1109522 100644 --- a/tests.py +++ b/tests.py @@ -209,6 +209,7 @@ def test_round_trip_same(self): "32.23", "32.23M", "-32.23M", + "-3.903495E-73M", "3.23e-10", "3e+20", "3E+20M", From ec2c10ed3cce3beed1cf193c92362856f693343c Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sun, 5 Aug 2018 11:42:02 +0200 Subject: [PATCH 07/65] Accept missing printable ASCII chars (#45) --- edn_format/edn_lex.py | 3 ++- tests.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index a455525..a84a60c 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -170,7 +170,8 @@ def t_WHITESPACE(t): def t_CHAR(t): - r"(\\\w)" + # from "!" to "~" = all printable ASCII chars except the space + r"(\\[!-~])" t.value = t.value[1] return t diff --git a/tests.py b/tests.py index 1109522..ad4d7e2 100644 --- a/tests.py +++ b/tests.py @@ -252,6 +252,13 @@ def __str__(self): step3 = dumps(step2) self.assertEqual(step1, step3) + def test_chars(self): + # 33-126 = printable ASCII range, except space (code 32) + for i in range(33, 127): + ch = chr(i) + edn_data = "\\{}".format(ch) + self.assertEqual(ch, loads(edn_data), edn_data) + def test_keyword_keys(self): unchanged = ( None, From ea068bd120231e55a94a0eb1f5ad238d5cb0891f Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Thu, 16 Aug 2018 23:17:38 +0200 Subject: [PATCH 08/65] Add a sort_sets optional keyword argument to dump (#42) --- edn_format/edn_dump.py | 12 +++++++++--- tests.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/edn_format/edn_dump.py b/edn_format/edn_dump.py index c2ef2db..a52d6c8 100644 --- a/edn_format/edn_dump.py +++ b/edn_format/edn_dump.py @@ -61,12 +61,14 @@ def seq(obj, **kwargs): def udump(obj, string_encoding=DEFAULT_INPUT_ENCODING, keyword_keys=False, - sort_keys=False): + sort_keys=False, + sort_sets=False): kwargs = { "string_encoding": string_encoding, "keyword_keys": keyword_keys, "sort_keys": sort_keys, + "sort_sets": sort_sets, } if obj is None: @@ -91,6 +93,8 @@ def udump(obj, elif isinstance(obj, list): return '[{}]'.format(seq(obj, **kwargs)) elif isinstance(obj, set) or isinstance(obj, frozenset): + if sort_sets: + obj = sorted(obj) return '#{{{}}}'.format(seq(obj, **kwargs)) elif isinstance(obj, dict) or isinstance(obj, ImmutableDict): pairs = obj.items() @@ -117,11 +121,13 @@ def dump(obj, string_encoding=DEFAULT_INPUT_ENCODING, output_encoding=DEFAULT_OUTPUT_ENCODING, keyword_keys=False, - sort_keys=False): + sort_keys=False, + sort_sets=False): outcome = udump(obj, string_encoding=string_encoding, keyword_keys=keyword_keys, - sort_keys=sort_keys) + sort_keys=sort_keys, + sort_sets=sort_sets) if __PY3: return outcome return outcome.encode(output_encoding) diff --git a/tests.py b/tests.py index ad4d7e2..993b13c 100644 --- a/tests.py +++ b/tests.py @@ -4,6 +4,7 @@ from collections import OrderedDict from uuid import uuid4 +import random import datetime import unittest @@ -130,6 +131,7 @@ def check_roundtrip(self, data_input, **kw): def test_dump(self): self.check_roundtrip({1, 2, 3}) + self.check_roundtrip({1, 2, 3}, sort_sets=True) self.check_roundtrip( {Keyword("a"): 1, "foo": Keyword("gone"), @@ -310,6 +312,25 @@ def test_sort_keys(self): {"a": 1, "d": 1, "b": 1, "c": 1}, sort_keys=True, keyword_keys=True) + def test_sort_sets(self): + def misordered_set_sequence(): + """" + Return a tuple that, if put in a set then iterated over, doesn't + yield elements in the original order + """ + while True: + seq = tuple(random.sample(range(10000), random.randint(2, 8))) + s = set(seq) + if tuple(s) != seq: + return seq + + for _ in range(10): + seq = misordered_set_sequence() + self.check_roundtrip(set(seq)) + self.check_dumps("#{{{}}}".format(" ".join(str(i) for i in sorted(seq))), + set(seq), + sort_sets=True) + class EdnInstanceTest(unittest.TestCase): def test_hashing(self): From b71fd3ba3f877b14125c1cefcef957c4bde6c7c7 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 10 May 2017 19:35:23 -0300 Subject: [PATCH 09/65] fix latin --- edn_format/edn_dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edn_format/edn_dump.py b/edn_format/edn_dump.py index a52d6c8..0027553 100644 --- a/edn_format/edn_dump.py +++ b/edn_format/edn_dump.py @@ -81,7 +81,7 @@ def udump(obj, return '{}M'.format(obj) elif isinstance(obj, (Keyword, Symbol)): return unicode(obj) - # CAVEAT EMPTOR! In Python 3 'basestring' is alised to 'str' above. + # CAVEAT LECTOR! In Python 3 'basestring' is alised to 'str' above. # Furthermore, in Python 2 bytes is an instance of 'str'/'basestring' while # in Python 3 it is not. elif isinstance(obj, bytes): From 7cb0c3f56c7b44eb70cf1f6fc71ecb94d565b982 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 10 May 2017 19:34:06 -0300 Subject: [PATCH 10/65] add failing unit test --- tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 993b13c..fcb0f96 100644 --- a/tests.py +++ b/tests.py @@ -11,7 +11,7 @@ import pytz from edn_format import edn_lex, edn_parse, \ - loads, dumps, Keyword, Symbol, TaggedElement, ImmutableDict, add_tag + loads, dumps, Keyword, Symbol, TaggedElement, ImmutableDict, ImmutableList, add_tag class ConsoleTest(unittest.TestCase): @@ -125,6 +125,7 @@ def test_parser(self): self.check_parse('\\', r'"\\"') self.check_parse(["abc", "123"], '["abc", "123"]') self.check_parse({"key": "value"}, '{"key" "value"}') + self.check_parse(frozenset({ImmutableList([u"ab", u"cd"]), ImmutableList([u"ef"])}), '#{["ab", "cd"], ["ef"]}') def check_roundtrip(self, data_input, **kw): self.assertEqual(data_input, loads(dumps(data_input, **kw))) From faf098ec98cf560338c977bdb3ddec2e21018f8d Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 10 May 2017 19:34:48 -0300 Subject: [PATCH 11/65] add ImmutableList --- edn_format/__init__.py | 2 ++ edn_format/immutable_list.py | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 edn_format/immutable_list.py diff --git a/edn_format/__init__.py b/edn_format/__init__.py index 2d2d26a..2ca0b33 100644 --- a/edn_format/__init__.py +++ b/edn_format/__init__.py @@ -6,8 +6,10 @@ from .edn_parse import add_tag, remove_tag, TaggedElement from .edn_dump import dump as dumps from .immutable_dict import ImmutableDict +from .immutable_list import ImmutableList __all__ = ( + 'ImmutableList', 'ImmutableDict', 'Keyword', 'Symbol', diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py new file mode 100644 index 0000000..9004a16 --- /dev/null +++ b/edn_format/immutable_list.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections + +class ImmutableList(collections.MutableSequence, collections.Hashable): + def __init__(self, wrapped_list, copy=True): + self._list = list(wrapped_list) if copy else wrapped_list + self._hash = None + + def __repr__(self): + return self._list.__repr__() + + def __eq__(self, other): + if isinstance(other, ImmutableList): + return self._list == other._list + else: + return self._list == other + + def _call_wrapped_list_method(self, method, *args): + new_list = list(self._list) + getattr(new_list, method)(*args) + return ImmutableList(new_list, copy=False) + + # collection.MutableSequence methods https://docs.python.org/2/library/collections.html#collections-abstract-base-classes + + def __getitem__(self, index): + return self._list[index] + + def __delitem__(self, *args): + return self._call_wrapped_list_method("__delitem__", *args) + + def __setitem__(self, *args): + return self._call_wrapped_list_method("__setitem__", *args) + + def __len__(self): + return len(self._list) + + # collection.Hashable methods https://docs.python.org/2/library/collections.html#collections-abstract-base-classes + + def __hash__(self): + if self._hash is None: + self._hash = hash(tuple(self._list)) + return self._hash + + # Other list methods https://docs.python.org/2/tutorial/datastructures.html#more-on-lists + + def insert(self, *args): + return self._call_wrapped_list_method("insert", *args) + + def sort(self, *args): + return self._call_wrapped_list_method("sort", *args) From 55793ce933dfac3b12f886e425fee653579e1805 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 10 May 2017 19:35:05 -0300 Subject: [PATCH 12/65] use ImmutableList on code --- edn_format/edn_dump.py | 3 +++ edn_format/edn_parse.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/edn_format/edn_dump.py b/edn_format/edn_dump.py index 0027553..56c8a22 100644 --- a/edn_format/edn_dump.py +++ b/edn_format/edn_dump.py @@ -11,6 +11,7 @@ import pyrfc3339 from .immutable_dict import ImmutableDict +from .immutable_list import ImmutableList from .edn_lex import Keyword, Symbol from .edn_parse import TaggedElement @@ -92,6 +93,8 @@ def udump(obj, return '({})'.format(seq(obj, **kwargs)) elif isinstance(obj, list): return '[{}]'.format(seq(obj, **kwargs)) + elif isinstance(obj, ImmutableList): + return '[{}]'.format(seq(obj, **kwargs)) elif isinstance(obj, set) or isinstance(obj, frozenset): if sort_sets: obj = sorted(obj) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index 07ad728..9257453 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -10,6 +10,7 @@ from .edn_lex import tokens, lex from .immutable_dict import ImmutableDict +from .immutable_list import ImmutableList if sys.version_info[0] == 3: @@ -56,12 +57,12 @@ def p_term_leaf(p): def p_empty_vector(p): """vector : VECTOR_START VECTOR_END""" - p[0] = [] + p[0] = ImmutableList([]) def p_vector(p): """vector : VECTOR_START expressions VECTOR_END""" - p[0] = p[2] + p[0] = ImmutableList(p[2]) def p_empty_list(p): From 7b7aef3a7c80d1f49786ddfc5779e54e58de8451 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Tue, 16 May 2017 12:10:52 -0300 Subject: [PATCH 13/65] conform to flake8 --- edn_format/immutable_list.py | 7 +++++-- tests.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 9004a16..47f966f 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -3,6 +3,7 @@ import collections + class ImmutableList(collections.MutableSequence, collections.Hashable): def __init__(self, wrapped_list, copy=True): self._list = list(wrapped_list) if copy else wrapped_list @@ -22,7 +23,8 @@ def _call_wrapped_list_method(self, method, *args): getattr(new_list, method)(*args) return ImmutableList(new_list, copy=False) - # collection.MutableSequence methods https://docs.python.org/2/library/collections.html#collections-abstract-base-classes + # collection.MutableSequence methods + # https://docs.python.org/2/library/collections.html#collections-abstract-base-classes def __getitem__(self, index): return self._list[index] @@ -36,7 +38,8 @@ def __setitem__(self, *args): def __len__(self): return len(self._list) - # collection.Hashable methods https://docs.python.org/2/library/collections.html#collections-abstract-base-classes + # collection.Hashable methods + # https://docs.python.org/2/library/collections.html#collections-abstract-base-classes def __hash__(self): if self._hash is None: diff --git a/tests.py b/tests.py index fcb0f96..9664006 100644 --- a/tests.py +++ b/tests.py @@ -125,7 +125,9 @@ def test_parser(self): self.check_parse('\\', r'"\\"') self.check_parse(["abc", "123"], '["abc", "123"]') self.check_parse({"key": "value"}, '{"key" "value"}') - self.check_parse(frozenset({ImmutableList([u"ab", u"cd"]), ImmutableList([u"ef"])}), '#{["ab", "cd"], ["ef"]}') + self.check_parse(frozenset({ImmutableList([u"ab", u"cd"]), + ImmutableList([u"ef"])}), + '#{["ab", "cd"], ["ef"]}') def check_roundtrip(self, data_input, **kw): self.assertEqual(data_input, loads(dumps(data_input, **kw))) From 4de31308d511580b46095af95be843d141c80717 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Tue, 23 May 2017 15:50:39 -0300 Subject: [PATCH 14/65] add test --- tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests.py b/tests.py index 9664006..d8d02a9 100644 --- a/tests.py +++ b/tests.py @@ -352,5 +352,24 @@ def test_equality(self): self.assertTrue(Symbol("db/id") == Symbol("db/id")) +class ImmutableListTest(unittest.TestCase): + def test_list(self): + x = ImmutableList([1,2,3]) + self.assertTrue(x == [1, 2, 3]) + + self.assertTrue(x.append(4) == None) + self.assertTrue(x.extend(x) == None) + self.assertTrue(x.reverse() == None) + self.assertTrue(x.remove(1) == None) + + self.assertTrue(x.pop(0) == 1) + self.assertTrue(x.index(1) == 0) + self.assertTrue(x.count(3) == 1) + self.assertTrue(x == [1, 2, 3]) + self.assertTrue(x.insert(0, 0) == [0, 1, 2, 3]) + + y = ImmutableList([3, 1, 4]) + self.assertTrue(y.sort() == [1, 3, 4]) + if __name__ == "__main__": unittest.main() From 2f0632ec2a9ae53522ccc033b04ed9ab05e34a4a Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Tue, 23 May 2017 19:36:52 -0300 Subject: [PATCH 15/65] add placeholder file --- conda.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 conda.txt diff --git a/conda.txt b/conda.txt new file mode 100644 index 0000000..e69de29 From 294c7f7a4018debd95e72a6e10dbc043f653dfa3 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Tue, 23 May 2017 19:42:34 -0300 Subject: [PATCH 16/65] workaround for limitation on the image downstream --- conda.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/conda.txt b/conda.txt index e69de29..e079f8a 100644 --- a/conda.txt +++ b/conda.txt @@ -0,0 +1 @@ +pytest From 3779ef72e384219bf01fd9c858b012b205bbbfd6 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:39:07 -0300 Subject: [PATCH 17/65] remove conda.txt file --- conda.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 conda.txt diff --git a/conda.txt b/conda.txt deleted file mode 100644 index e079f8a..0000000 --- a/conda.txt +++ /dev/null @@ -1 +0,0 @@ -pytest From 0a45a22a4fcb16a1109066f3f1708c7822b5ee8c Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:39:20 -0300 Subject: [PATCH 18/65] add docstring --- edn_format/immutable_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 47f966f..ded932e 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -6,6 +6,7 @@ class ImmutableList(collections.MutableSequence, collections.Hashable): def __init__(self, wrapped_list, copy=True): + """Returns an immutable version of the given list. Optionally creates a shallow copy.""" self._list = list(wrapped_list) if copy else wrapped_list self._hash = None From c86cf6f3432a30626cecdb5536e85c3baa25e59b Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:41:57 -0300 Subject: [PATCH 19/65] descend from Sequence instead of MutableSequence --- edn_format/immutable_list.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index ded932e..4c90c88 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -4,7 +4,7 @@ import collections -class ImmutableList(collections.MutableSequence, collections.Hashable): +class ImmutableList(collections.Sequence, collections.Hashable): def __init__(self, wrapped_list, copy=True): """Returns an immutable version of the given list. Optionally creates a shallow copy.""" self._list = list(wrapped_list) if copy else wrapped_list @@ -24,18 +24,12 @@ def _call_wrapped_list_method(self, method, *args): getattr(new_list, method)(*args) return ImmutableList(new_list, copy=False) - # collection.MutableSequence methods + # collection.Sequence methods # https://docs.python.org/2/library/collections.html#collections-abstract-base-classes def __getitem__(self, index): return self._list[index] - def __delitem__(self, *args): - return self._call_wrapped_list_method("__delitem__", *args) - - def __setitem__(self, *args): - return self._call_wrapped_list_method("__setitem__", *args) - def __len__(self): return len(self._list) From 4d4cd74c5a94f8070420c509b265886fccdefb61 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:42:25 -0300 Subject: [PATCH 20/65] copy explicitly --- edn_format/immutable_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 4c90c88..2d3895f 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -7,7 +7,7 @@ class ImmutableList(collections.Sequence, collections.Hashable): def __init__(self, wrapped_list, copy=True): """Returns an immutable version of the given list. Optionally creates a shallow copy.""" - self._list = list(wrapped_list) if copy else wrapped_list + self._list = wrapped_list.copy() if copy else wrapped_list self._hash = None def __repr__(self): @@ -20,7 +20,7 @@ def __eq__(self, other): return self._list == other def _call_wrapped_list_method(self, method, *args): - new_list = list(self._list) + new_list = self._list.copy() getattr(new_list, method)(*args) return ImmutableList(new_list, copy=False) From 88a00786d8a0f1b1d854788bb535453c419a3854 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:42:38 -0300 Subject: [PATCH 21/65] remove MutableSequence tests --- tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests.py b/tests.py index d8d02a9..3a4d59c 100644 --- a/tests.py +++ b/tests.py @@ -357,12 +357,6 @@ def test_list(self): x = ImmutableList([1,2,3]) self.assertTrue(x == [1, 2, 3]) - self.assertTrue(x.append(4) == None) - self.assertTrue(x.extend(x) == None) - self.assertTrue(x.reverse() == None) - self.assertTrue(x.remove(1) == None) - - self.assertTrue(x.pop(0) == 1) self.assertTrue(x.index(1) == 0) self.assertTrue(x.count(3) == 1) self.assertTrue(x == [1, 2, 3]) From bc88cf385ed95649d08def358562333ed2b47a44 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:45:49 -0300 Subject: [PATCH 22/65] bump version --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2047af8..ad8397e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # `edn_format` Changelog +## v0.5.14 (2018/08/22) + +* Fix vector parser to use ImmutableList + ## v0.5.13 (2017/10/08) * Convert requirements from exact to minimum diff --git a/setup.py b/setup.py index 77e9864..9a0c2fe 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup setup(name="edn_format", - version="0.5.13", + version="0.5.14", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From a93d33d02cb6dd675a8dbfddb05f65349da19249 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:50:51 -0300 Subject: [PATCH 23/65] fix formatting --- tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 3a4d59c..7add395 100644 --- a/tests.py +++ b/tests.py @@ -354,7 +354,7 @@ def test_equality(self): class ImmutableListTest(unittest.TestCase): def test_list(self): - x = ImmutableList([1,2,3]) + x = ImmutableList([1, 2, 3]) self.assertTrue(x == [1, 2, 3]) self.assertTrue(x.index(1) == 0) @@ -365,5 +365,6 @@ def test_list(self): y = ImmutableList([3, 1, 4]) self.assertTrue(y.sort() == [1, 3, 4]) + if __name__ == "__main__": unittest.main() From dd3cdeb00a8b04ceac40b47c24d1243942c43a38 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Wed, 22 Aug 2018 21:54:43 -0300 Subject: [PATCH 24/65] use copy.copy() as .copy() is not available for lists on python2 --- edn_format/immutable_list.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 2d3895f..5686847 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -2,12 +2,13 @@ from __future__ import absolute_import, division, print_function, unicode_literals import collections +import copy as _copy class ImmutableList(collections.Sequence, collections.Hashable): def __init__(self, wrapped_list, copy=True): """Returns an immutable version of the given list. Optionally creates a shallow copy.""" - self._list = wrapped_list.copy() if copy else wrapped_list + self._list = _copy.copy(wrapped_list) if copy else wrapped_list self._hash = None def __repr__(self): @@ -20,7 +21,7 @@ def __eq__(self, other): return self._list == other def _call_wrapped_list_method(self, method, *args): - new_list = self._list.copy() + new_list = _copy.copy(self._list) getattr(new_list, method)(*args) return ImmutableList(new_list, copy=False) From db832f647e8fe0ad8386c26f297f8402e18407f0 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Fri, 24 Aug 2018 18:56:56 -0300 Subject: [PATCH 25/65] remove redundant tests --- tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests.py b/tests.py index 7add395..b4a627a 100644 --- a/tests.py +++ b/tests.py @@ -359,7 +359,6 @@ def test_list(self): self.assertTrue(x.index(1) == 0) self.assertTrue(x.count(3) == 1) - self.assertTrue(x == [1, 2, 3]) self.assertTrue(x.insert(0, 0) == [0, 1, 2, 3]) y = ImmutableList([3, 1, 4]) From 356e4a6b6c3eaaaba0f3b104e996f3ceb4376348 Mon Sep 17 00:00:00 2001 From: Konrad Scorciapino Date: Fri, 24 Aug 2018 18:57:22 -0300 Subject: [PATCH 26/65] simplify condition --- edn_format/edn_dump.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/edn_format/edn_dump.py b/edn_format/edn_dump.py index 56c8a22..0f06153 100644 --- a/edn_format/edn_dump.py +++ b/edn_format/edn_dump.py @@ -91,9 +91,7 @@ def udump(obj, return unicode_escape(obj) elif isinstance(obj, tuple): return '({})'.format(seq(obj, **kwargs)) - elif isinstance(obj, list): - return '[{}]'.format(seq(obj, **kwargs)) - elif isinstance(obj, ImmutableList): + elif isinstance(obj, (list, ImmutableList)): return '[{}]'.format(seq(obj, **kwargs)) elif isinstance(obj, set) or isinstance(obj, frozenset): if sort_sets: From 2d8484329d2760a82bc3b4f00e73bf4f0dd62e79 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 25 Aug 2018 00:16:31 +0200 Subject: [PATCH 27/65] Raise custom exceptions on syntax errors (#46) --- edn_format/__init__.py | 2 ++ edn_format/edn_lex.py | 5 +++-- edn_format/edn_parse.py | 7 ++++--- edn_format/exceptions.py | 6 ++++++ tests.py | 7 ++++++- 5 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 edn_format/exceptions.py diff --git a/edn_format/__init__.py b/edn_format/__init__.py index 2d2d26a..c15737e 100644 --- a/edn_format/__init__.py +++ b/edn_format/__init__.py @@ -5,9 +5,11 @@ from .edn_parse import parse as loads from .edn_parse import add_tag, remove_tag, TaggedElement from .edn_dump import dump as dumps +from .exceptions import EDNDecodeError from .immutable_dict import ImmutableDict __all__ = ( + 'EDNDecodeError', 'ImmutableDict', 'Keyword', 'Symbol', diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index a84a60c..ac0a3af 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -12,6 +12,7 @@ import ply.lex +from .exceptions import EDNDecodeError from .immutable_dict import ImmutableDict @@ -204,7 +205,7 @@ def t_FLOAT(t): if 'e' in t.value or 'E' in t.value: matches = re.search('[eE]([+-]?\d+)M?$', t.value) if matches is None: - raise SyntaxError('Invalid float : {}'.format(t.value)) + raise EDNDecodeError('Invalid float : {}'.format(t.value)) e_value = int(matches.group(1)) if t.value.endswith('M'): ctx = decimal.getcontext() @@ -251,7 +252,7 @@ def t_SYMBOL(t): def t_error(t): - raise SyntaxError( + raise EDNDecodeError( "Illegal character '{c}' with lexpos {p} in the area of ...{a}...".format( c=t.value[0], p=t.lexpos, a=t.value[0:100])) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index 07ad728..006ddd7 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -9,6 +9,7 @@ import pyrfc3339 from .edn_lex import tokens, lex +from .exceptions import EDNDecodeError from .immutable_dict import ImmutableDict @@ -93,7 +94,7 @@ def p_map(p): """map : MAP_START expressions MAP_OR_SET_END""" terms = p[2] if len(terms) % 2 != 0: - raise SyntaxError('Even number of terms required for map') + raise EDNDecodeError('Even number of terms required for map') # partition terms in pairs p[0] = ImmutableDict(dict([terms[i:i + 2] for i in range(0, len(terms), 2)])) @@ -144,9 +145,9 @@ def p_expression_tagged_element(p): def p_error(p): if p is None: - raise SyntaxError('EOF Reached') + raise EDNDecodeError('EOF Reached') else: - raise SyntaxError(p) + raise EDNDecodeError(p) def parse(text, input_encoding='utf-8'): diff --git a/edn_format/exceptions.py b/edn_format/exceptions.py new file mode 100644 index 0000000..c3bbcb4 --- /dev/null +++ b/edn_format/exceptions.py @@ -0,0 +1,6 @@ +# -*- coding: UTF-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + + +class EDNDecodeError(ValueError): + pass diff --git a/tests.py b/tests.py index 993b13c..938a181 100644 --- a/tests.py +++ b/tests.py @@ -11,7 +11,8 @@ import pytz from edn_format import edn_lex, edn_parse, \ - loads, dumps, Keyword, Symbol, TaggedElement, ImmutableDict, add_tag + loads, dumps, Keyword, Symbol, TaggedElement, ImmutableDict, add_tag, \ + EDNDecodeError class ConsoleTest(unittest.TestCase): @@ -261,6 +262,10 @@ def test_chars(self): edn_data = "\\{}".format(ch) self.assertEqual(ch, loads(edn_data), edn_data) + def test_exceptions(self): + with self.assertRaises(EDNDecodeError): + loads("{") + def test_keyword_keys(self): unchanged = ( None, From f53f98dcc7b26cb8cd67cdc5575293a2d2fa96a8 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 25 Aug 2018 00:17:08 +0200 Subject: [PATCH 28/65] Add CONTRIBUTING.md (#48) --- CONTRIBUTING.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cadb748 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing to edn\_format # + +## Setup ## + +1. Make sure you have a working Python installation (2 or 3). You may want to + use a [Virtualenv][] or a similar tool to create an isolated environment. +2. Install `edn_format`’s dependencies: `pip install -r requirements.txt` +3. Install `flake8`: `pip install flake8` + +[Virtualenv]: https://virtualenv.pypa.io/en/stable/#virtualenv + +## Tests ## + +Run unit tests with: + + python tests.py + +Run a linter over the code with: + + flake8 --max-line-length=100 --exclude=parsetab.py tests.py From a6e974d4b5cddc6f1db95ae0fa1be96a91c8a04c Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sun, 26 Aug 2018 22:11:57 +0200 Subject: [PATCH 29/65] Handle fractions https://github.com/edn-format/edn/issues/64 --- edn_format/edn_dump.py | 3 +++ edn_format/edn_lex.py | 9 +++++++++ edn_format/edn_parse.py | 1 + tests.py | 22 +++++++++++++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/edn_format/edn_dump.py b/edn_format/edn_dump.py index 0f06153..3093f33 100644 --- a/edn_format/edn_dump.py +++ b/edn_format/edn_dump.py @@ -3,6 +3,7 @@ import datetime import decimal +import fractions import itertools import re import sys @@ -105,6 +106,8 @@ def udump(obj, pairs = ((Keyword(k) if isinstance(k, (bytes, basestring)) else k, v) for k, v in pairs) return '{{{}}}'.format(seq(itertools.chain.from_iterable(pairs), **kwargs)) + elif isinstance(obj, fractions.Fraction): + return '{}/{}'.format(obj.numerator, obj.denominator) elif isinstance(obj, datetime.datetime): return '#inst "{}"'.format(pyrfc3339.generate(obj, microseconds=True)) elif isinstance(obj, datetime.date): diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index ac0a3af..bd88865 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -5,6 +5,7 @@ # see http://stackoverflow.com/a/24519338 import codecs import decimal +import fractions import logging import re import sys @@ -93,6 +94,7 @@ def __str__(self): 'BOOLEAN', 'INTEGER', 'FLOAT', + 'RATIO', 'SYMBOL', 'KEYWORD', 'VECTOR_START', @@ -215,6 +217,13 @@ def t_FLOAT(t): return t +def t_RATIO(t): + r"""-?\d+/\d+""" + numerator, denominator = t.value.split("/", 1) + t.value = fractions.Fraction(int(numerator), int(denominator)) + return t + + def t_INTEGER(t): r"""[+-]?\d+N?""" if t.value.endswith('N'): diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index c2be09d..91a1e5b 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -52,6 +52,7 @@ def p_term_leaf(p): | NIL | KEYWORD | SYMBOL + | RATIO | WHITESPACE""" p[0] = p[1] diff --git a/tests.py b/tests.py index 8f7fcc1..3a8a87c 100644 --- a/tests.py +++ b/tests.py @@ -6,6 +6,7 @@ from uuid import uuid4 import random import datetime +import fractions import unittest import pytz @@ -74,6 +75,9 @@ def test_lexer(self): "LexToken(MAP_OR_SET_END,'}',1,21)]", "{ :a false, :b false }") + self.check_lex("[LexToken(RATIO,Fraction(2, 3),1,0)]", + "2/3") + def check_parse(self, expected_output, actual_input): self.assertEqual(expected_output, edn_parse.parse(actual_input)) @@ -129,6 +133,8 @@ def test_parser(self): self.check_parse(frozenset({ImmutableList([u"ab", u"cd"]), ImmutableList([u"ef"])}), '#{["ab", "cd"], ["ef"]}') + self.check_parse(fractions.Fraction(2, 3), "2/3") + self.check_parse((2, Symbol('/'), 3), "(2 / 3)") def check_roundtrip(self, data_input, **kw): self.assertEqual(data_input, loads(dumps(data_input, **kw))) @@ -234,7 +240,10 @@ def test_round_trip_same(self): '#date "19/07/1984"', '#{{"a" 1}}', '#{{"a" #{{:b 2}}}}', - '"|"' + '"|"', + "/", + "1/3", + '"1/3"', ) class TagDate(TaggedElement): @@ -269,6 +278,17 @@ def test_exceptions(self): with self.assertRaises(EDNDecodeError): loads("{") + def test_fractions(self): + for edn_data in ( + '0/1', + '1/1', + '1/2', + '1/3', + '-5/3', + '99999999999999999999999999999999999/999999999999999999999999991', + ): + self.assertEqual(edn_data, dumps(loads(edn_data)), edn_data) + def test_keyword_keys(self): unchanged = ( None, From f5096dcdbe0bacc10b07175a7b267d470621ad9a Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Fri, 3 Aug 2018 22:28:58 +0200 Subject: [PATCH 30/65] Handle #_ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #4. Note it doesn’t support top-level #_ usage such as in: foo #_ bar --- edn_format/edn_lex.py | 14 +++++++----- edn_format/edn_parse.py | 49 ++++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index ac0a3af..fc2026e 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -102,7 +102,8 @@ def __str__(self): 'MAP_START', 'SET_START', 'MAP_OR_SET_END', - 'TAG') + 'TAG', + 'DISCARD_TAG') PARTS = {} PARTS["non_nums"] = r"\w.*+!\-_?$%&=:#<>@" @@ -138,7 +139,7 @@ def __str__(self): "[{all}]+" ")").format(**PARTS) TAG = (r"\#" - r"\w" + r"[a-zA-Z]" # https://github.com/edn-format/edn/issues/30#issuecomment-8540641 "(" "[{all}]*" r"\/" @@ -147,6 +148,8 @@ def __str__(self): "[{all}]*" ")").format(**PARTS) +DISCARD_TAG = r"\#\_" + t_VECTOR_START = r'\[' t_VECTOR_END = r'\]' t_LIST_START = r'\(' @@ -228,9 +231,10 @@ def t_COMMENT(t): pass # ignore -def t_DISCARD(t): - r'\#_\S+\b' - pass # ignore +@ply.lex.TOKEN(DISCARD_TAG) +def t_DISCARD_TAG(t): + t.value = t.value[1:] + return t @ply.lex.TOKEN(TAG) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index c2be09d..329584e 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -56,41 +56,21 @@ def p_term_leaf(p): p[0] = p[1] -def p_empty_vector(p): - """vector : VECTOR_START VECTOR_END""" - p[0] = ImmutableList([]) - - def p_vector(p): """vector : VECTOR_START expressions VECTOR_END""" p[0] = ImmutableList(p[2]) -def p_empty_list(p): - """list : LIST_START LIST_END""" - p[0] = tuple() - - def p_list(p): """list : LIST_START expressions LIST_END""" p[0] = tuple(p[2]) -def p_empty_set(p): - """set : SET_START MAP_OR_SET_END""" - p[0] = frozenset() - - def p_set(p): """set : SET_START expressions MAP_OR_SET_END""" p[0] = frozenset(p[2]) -def p_empty_map(p): - """map : MAP_START MAP_OR_SET_END""" - p[0] = ImmutableDict({}) - - def p_map(p): """map : MAP_START expressions MAP_OR_SET_END""" terms = p[2] @@ -100,14 +80,20 @@ def p_map(p): p[0] = ImmutableDict(dict([terms[i:i + 2] for i in range(0, len(terms), 2)])) -def p_expressions_expressions_expression(p): - """expressions : expressions expression""" - p[0] = p[1] + [p[2]] +def p_discarded_expressions(p): + """discarded_expressions : DISCARD_TAG expression discarded_expressions + |""" + p[0] = [] + +def p_expressions_expression_expressions(p): + """expressions : expression expressions""" + p[0] = [p[1]] + p[2] -def p_expressions_expression(p): - """expressions : expression""" - p[0] = [p[1]] + +def p_expressions_empty(p): + """expressions : discarded_expressions""" + p[0] = [] def p_expression(p): @@ -119,6 +105,11 @@ def p_expression(p): p[0] = p[1] +def p_expression_discard_expression_expression(p): + """expression : DISCARD_TAG expression expression""" + p[0] = p[3] + + def p_expression_tagged_element(p): """expression : TAG expression""" tag = p[1] @@ -144,9 +135,13 @@ def p_expression_tagged_element(p): p[0] = output +def eof(): + raise EDNDecodeError('EOF Reached') + + def p_error(p): if p is None: - raise EDNDecodeError('EOF Reached') + eof() else: raise EDNDecodeError(p) From fde947afd78372171ced65cc2a9dfac8354016d0 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sun, 26 Aug 2018 23:11:21 +0200 Subject: [PATCH 31/65] Add tests for #_ --- tests.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests.py b/tests.py index 8f7fcc1..29562d5 100644 --- a/tests.py +++ b/tests.py @@ -133,6 +133,12 @@ def test_parser(self): def check_roundtrip(self, data_input, **kw): self.assertEqual(data_input, loads(dumps(data_input, **kw))) + def check_eof(self, data_input, **kw): + with self.assertRaises(EDNDecodeError) as ctx: + loads(data_input, **kw) + + self.assertEqual('EOF Reached', str(ctx.exception)) + def test_dump(self): self.check_roundtrip({1, 2, 3}) self.check_roundtrip({1, 2, 3}, sort_sets=True) @@ -339,6 +345,57 @@ def misordered_set_sequence(): set(seq), sort_sets=True) + def test_discard(self): + for expected, edn_data in ( + ('[x]', '[x #_ z]'), + ('[z]', '[#_ x z]'), + ('[x z]', '[x #_ y z]'), + ('{1 4}', '{1 #_ 2 #_ 3 4}'), + ('[1 2]', '[1 #_ [ #_ [ #_ [ #_ [ #_ 42 ] ] ] ] 2 ]'), + ('[1 2 11]', '[1 2 #_ #_ #_ #_ 4 5 6 #_ 7 #_ #_ 8 9 10 11]'), + ('()', '(#_(((((((1))))))))'), + ('[6]', '[#_ #_ #_ #_ #_ 1 2 3 4 5 6]'), + ('[4]', '[#_ #_ 1 #_ 2 3 4]'), + ('{:a 1}', '{:a #_:b 1}'), + ('[42]', '[42 #_ {:a [1 2 3 4] true false 1 #inst "2017"}]'), + ('#{1}', '#{1 #_foo}'), + ('"#_ foo"', '"#_ foo"'), + ('["#" _]', '[\#_]'), + ('[_]', '[#_\#_]'), + ('[1]', '[1 #_\n\n42]'), + ('{}', '{#_ 1}'), + ): + self.assertEqual(expected, dumps(loads(edn_data)), edn_data) + + def test_discard_syntax_errors(self): + for edn_data in ('#_', '#_ #_ 1', '#inst #_ 2017', '[#_]'): + with self.assertRaises(EDNDecodeError): + loads(edn_data) + + def test_discard_all(self): + for edn_data in ( + '42', '-1', 'nil', 'true', 'false', '"foo"', '\\space', '\\a', + ':foo', ':foo/bar', '[]', '{}', '#{}', '()', '(a)', '(a b)', + '[a [[[b] c]] 2]', '#inst "2017"', + ): + self.assertEqual([1], loads('[1 #_ {}]'.format(edn_data)), edn_data) + self.assertEqual([1], loads('[#_ {} 1]'.format(edn_data)), edn_data) + + self.check_eof('#_ {}'.format(edn_data)) + + for coll in ('[%s]', '(%s)', '{%s}', '#{%s}'): + expected = coll % "" + edn_data = coll % '#_ {}'.format(edn_data) + self.assertEqual(expected, dumps(loads(edn_data)), edn_data) + + def test_chained_discards(self): + for expected, edn_data in ( + ('[]', '[#_ 1 #_ 2 #_ 3]'), + ('[]', '[#_ #_ 1 2 #_ 3]'), + ('[]', '[#_ #_ #_ 1 2 3]'), + ): + self.assertEqual(expected, dumps(loads(edn_data)), edn_data) + class EdnInstanceTest(unittest.TestCase): def test_hashing(self): From 6740ed8ee3c9efce0a42c3a881e948f2266c708d Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 7 Sep 2018 15:08:17 -0700 Subject: [PATCH 32/65] Tweak README regarding caveats + mention @bfontaine contributor --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ecd5cf9..042a246 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,14 @@ See `tests.py` for full details. Almost all features of EDN have been implemented, including custom tagged elements. -But expect bugs since this has not yet been used in production. +However, I personally don't use this in production, even though many users do, esp. the active contributors below. ## Contributors ## Special thanks to the following contributors for making this library usable: +- [@bfontaine](https://github.com/bfontaine) - [@marianoguerra](https://github.com/marianoguerra) - [@bitemyapp](https://github.com/bitemyapp) - [@jashugan](https://github.com/jashugan) From b721edd1a052b1c2c7b28b89c3c227b381136e1c Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 7 Sep 2018 15:11:21 -0700 Subject: [PATCH 33/65] Bump version to 0.6.0 Thanks @bfontaine and @konr! --- README.md | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 042a246..6c784a8 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ usable: To release a new version: -1. Bump up the version number in `setup.py`, e.g. `0.5.13` -2. Create a git tag: `git tag -a v0.5.13 -m 'Version 0.5.13'` +1. Bump up the version number in `setup.py`, e.g. `0.6.0` +2. Create a git tag: `git tag -a v0.6.0 -m 'Version 0.6.0'` 3. Push git tag: `git push origin master --tags` 4. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. 5. Run `python setup.py sdist upload` diff --git a/setup.py b/setup.py index 9a0c2fe..729d48e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup setup(name="edn_format", - version="0.5.14", + version="0.6.0", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From 629b52d3910fed7bc1f8a9722f97a59bf5fcd7c5 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sun, 9 Sep 2018 14:09:36 +0200 Subject: [PATCH 34/65] Add v0.6.0 to the Changelog --- CHANGELOG.md | 12 +++++++++--- README.md | 3 +-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad8397e..38140c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,14 @@ # `edn_format` Changelog -## v0.5.14 (2018/08/22) - -* Fix vector parser to use ImmutableList +## v0.6.0 (2018/09/08) + +* Fix vector parser to use `ImmutableList` +* Fix parsing of exact-precision floats with a negative exposant +* Support all ASCII characters +* Add a `sort_sets` optional argument to `dumps` +* Raise custom exceptions on syntax errors +* Support fractions +* Support `#_` ## v0.5.13 (2017/10/08) diff --git a/README.md b/README.md index 6c784a8..04ec063 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,7 @@ See `tests.py` for full details. ## Caveats ## -Almost all features of EDN have been implemented, including custom -tagged elements. +All features of EDN have been implemented, including custom tagged elements. However, I personally don't use this in production, even though many users do, esp. the active contributors below. From 0dfb8bc4f61332cfdfd651a7ecd908ba9acb2789 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Wed, 19 Sep 2018 13:16:07 -0700 Subject: [PATCH 35/65] Run Travis tests on latest Python 3.7 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fb12675..76b7ab1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.7" - - "3.6" + - "3.7" install: - "pip install -r requirements.txt" - "pip install flake8" From 22e16bd1dfa4337200213f84ea27d525b088be35 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Wed, 19 Sep 2018 13:20:57 -0700 Subject: [PATCH 36/65] Use dist: xenial --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 76b7ab1..9c3f252 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: xenial # Required for Python 3.7, see https://github.com/travis-ci/travis-ci/issues/9069 language: python python: - "2.7" From 841a479cc112995b7e8c18b545f73dcfd56f6700 Mon Sep 17 00:00:00 2001 From: Carlo Zancanaro Date: Thu, 20 Sep 2018 10:48:38 +1000 Subject: [PATCH 37/65] Use mutable data structures to improve parsing time Previously a list was used to hold the intermediate results during parsing of "expressions", but each expression was prepended by copying, Fixing this takes parsing from quadratic to linear time. --- edn_format/edn_parse.py | 10 ++++++---- edn_format/immutable_list.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index 6da18cb..a027d1d 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -4,6 +4,7 @@ import datetime import sys import uuid +from collections import deque import ply.yacc import pyrfc3339 @@ -78,23 +79,24 @@ def p_map(p): if len(terms) % 2 != 0: raise EDNDecodeError('Even number of terms required for map') # partition terms in pairs - p[0] = ImmutableDict(dict([terms[i:i + 2] for i in range(0, len(terms), 2)])) + p[0] = ImmutableDict((terms[i], terms[i+1]) for i in range(0, len(terms), 2)) def p_discarded_expressions(p): """discarded_expressions : DISCARD_TAG expression discarded_expressions |""" - p[0] = [] + p[0] = deque([]) def p_expressions_expression_expressions(p): """expressions : expression expressions""" - p[0] = [p[1]] + p[2] + p[2].appendleft(p[1]) + p[0] = p[2] def p_expressions_empty(p): """expressions : discarded_expressions""" - p[0] = [] + p[0] = deque([]) def p_expression(p): diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 5686847..7b934f5 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -8,7 +8,7 @@ class ImmutableList(collections.Sequence, collections.Hashable): def __init__(self, wrapped_list, copy=True): """Returns an immutable version of the given list. Optionally creates a shallow copy.""" - self._list = _copy.copy(wrapped_list) if copy else wrapped_list + self._list = list(wrapped_list) if copy else wrapped_list self._hash = None def __repr__(self): From 32c124b641673d21956b6adb653af001e89ba149 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 21 Sep 2018 13:00:51 -0700 Subject: [PATCH 38/65] Bump version to 0.6.1 --- CHANGELOG.md | 5 +++++ README.md | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38140c1..2158e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # `edn_format` Changelog +## v0.6.1 (2018/09/21) + +* Use mutable data structures to improve parsing time +* Run Travis tests on Python 3.7 + ## v0.6.0 (2018/09/08) * Fix vector parser to use `ImmutableList` diff --git a/README.md b/README.md index 04ec063..007a119 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ usable: To release a new version: -1. Bump up the version number in `setup.py`, e.g. `0.6.0` -2. Create a git tag: `git tag -a v0.6.0 -m 'Version 0.6.0'` +1. Bump up the version number in `setup.py`, e.g. `0.6.1` +2. Create a git tag: `git tag -a v0.6.1 -m 'Version 0.6.1'` 3. Push git tag: `git push origin master --tags` 4. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. 5. Run `python setup.py sdist upload` From c14529e215492e999446c4ad76078bd0a334020e Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 21 Sep 2018 13:04:18 -0700 Subject: [PATCH 39/65] Update setup.py to 0.6.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 729d48e..56b416e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup setup(name="edn_format", - version="0.6.0", + version="0.6.1", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From f6e260cc23b1dab93a16ff2d8962feb336d458f4 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Thu, 27 Sep 2018 12:17:26 +0200 Subject: [PATCH 40/65] Various code simplifications (#53) --- edn_format/edn_lex.py | 3 ++- edn_format/edn_parse.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index 06b69cb..a38295f 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -273,7 +273,8 @@ def t_error(t): def lex(text=None): kwargs = {} if __debug__: - kwargs = dict(debug=True, debuglog=logging.getLogger(__name__)) + kwargs["debug"] = True + kwargs["debuglog"] = logging.getLogger(__name__) lex = ply.lex.lex(reflags=re.UNICODE, **kwargs) if text is not None: lex.input(text) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index a027d1d..c755523 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -26,7 +26,7 @@ start = 'expression' -_serializers = dict({}) +_serializers = {} class TaggedElement(object): @@ -85,7 +85,7 @@ def p_map(p): def p_discarded_expressions(p): """discarded_expressions : DISCARD_TAG expression discarded_expressions |""" - p[0] = deque([]) + p[0] = deque() def p_expressions_expression_expressions(p): @@ -96,7 +96,7 @@ def p_expressions_expression_expressions(p): def p_expressions_empty(p): """expressions : discarded_expressions""" - p[0] = deque([]) + p[0] = deque() def p_expression(p): @@ -119,11 +119,14 @@ def p_expression_tagged_element(p): element = p[2] if tag == 'inst': - if len(element) == 10 and element.count('-') == 2: + length = len(element) + hyphens_count = element.count('-') + + if length == 10 and hyphens_count == 2: output = datetime.datetime.strptime(element, '%Y-%m-%d').date() - elif len(element) == 7 and element.count('-') == 1: + elif length == 7 and hyphens_count == 1: output = datetime.datetime.strptime(element, '%Y-%m').date() - elif len(element) == 4 and element.count('-') == 0: + elif length == 4 and hyphens_count == 0: output = datetime.datetime.strptime(element, '%Y').date() else: output = pyrfc3339.parse(element) @@ -155,6 +158,6 @@ def parse(text, input_encoding='utf-8'): kwargs = ImmutableDict({}) if __debug__: - kwargs = dict({'debug': True}) + kwargs = dict(debug=True) p = ply.yacc.yacc(**kwargs) return p.parse(text, lexer=lex()) From 990573478292d9260711bbe6421baf7efa649513 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Thu, 27 Sep 2018 14:24:31 +0200 Subject: [PATCH 41/65] Parse nil and booleans as symbols (#54) --- edn_format/edn_lex.py | 26 ++++++++------------------ edn_format/edn_parse.py | 2 -- tests.py | 14 ++++++++------ 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index a38295f..650c032 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -90,8 +90,6 @@ def __str__(self): tokens = ('WHITESPACE', 'CHAR', 'STRING', - 'NIL', - 'BOOLEAN', 'INTEGER', 'FLOAT', 'RATIO', @@ -189,21 +187,6 @@ def t_STRING(t): return t -def t_NIL(t): - """nil""" - t.value = None - return t - - -def t_BOOLEAN(t): - r"""(true|false)(?=([,\s\])}]|$))""" - if t.value == "false": - t.value = False - elif t.value == "true": - t.value = True - return t - - def t_FLOAT(t): r"""[+-]?\d+(?:\.\d+([eE][+-]?\d+)?|([eE][+-]?\d+))M?""" e_value = 0 @@ -260,7 +243,14 @@ def t_KEYWORD(t): @ply.lex.TOKEN(SYMBOL) def t_SYMBOL(t): - t.value = Symbol(t.value) + if t.value == "nil": + t.value = None + elif t.value == "true": + t.value = True + elif t.value == "false": + t.value = False + else: + t.value = Symbol(t.value) return t diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index c755523..e202c9c 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -49,8 +49,6 @@ def p_term_leaf(p): | STRING | INTEGER | FLOAT - | BOOLEAN - | NIL | KEYWORD | SYMBOL | RATIO diff --git a/tests.py b/tests.py index f0bc9b7..702bdee 100644 --- a/tests.py +++ b/tests.py @@ -31,16 +31,16 @@ def check_lex(self, expected_output, actual_input): self.assertEqual(expected_output, str(list(edn_lex.lex(actual_input)))) def test_lexer(self): - self.check_lex("[LexToken(NIL,None,1,0)]", + self.check_lex("[LexToken(SYMBOL,None,1,0)]", "nil") - self.check_lex("[LexToken(BOOLEAN,True,1,0)]", + self.check_lex("[LexToken(SYMBOL,True,1,0)]", "true") self.check_lex("[LexToken(INTEGER,123,1,0)]", "123") self.check_lex( "[LexToken(INTEGER,456,1,0), " + - "LexToken(NIL,None,1,4), " + - "LexToken(BOOLEAN,False,1,8)]", + "LexToken(SYMBOL,None,1,4), " + + "LexToken(SYMBOL,False,1,8)]", "456 nil false") self.check_lex("[LexToken(CHAR,'c',1,0)]", r"\c") @@ -65,13 +65,15 @@ def test_lexer(self): "prefix/name") self.check_lex("[LexToken(SYMBOL,Symbol(true.),1,0)]", "true.") + self.check_lex("[LexToken(SYMBOL,Symbol(nil.),1,0)]", + "nil.") self.check_lex("[LexToken(SYMBOL,Symbol($:ABC?),1,0)]", "$:ABC?") self.check_lex("[LexToken(MAP_START,'{',1,0), " "LexToken(KEYWORD,Keyword(a),1,2), " - "LexToken(BOOLEAN,False,1,5), " + "LexToken(SYMBOL,False,1,5), " "LexToken(KEYWORD,Keyword(b),1,12), " - "LexToken(BOOLEAN,False,1,15), " + "LexToken(SYMBOL,False,1,15), " "LexToken(MAP_OR_SET_END,'}',1,21)]", "{ :a false, :b false }") From ed68fce709d8e5e9a7b8ac08c47bc78b1113e7f4 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 4 Jan 2019 09:39:05 -0800 Subject: [PATCH 42/65] Add Vagrantfile for local dev --- .gitignore | 3 ++- README.md | 24 ++++++++++++++++++++++++ Vagrantfile | 15 +++++++++++++++ install.sh | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Vagrantfile create mode 100755 install.sh diff --git a/.gitignore b/.gitignore index 6bbbb28..ce01ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ dist edn_format.egg-info/ *_flymake.py venv -build \ No newline at end of file +build +.vagrant \ No newline at end of file diff --git a/README.md b/README.md index 007a119..15fc32e 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,30 @@ usable: - [@jashugan](https://github.com/jashugan) - [@exilef](https://github.com/exilef) +## Local Dev ## + +```bash +# 1. One-time: Install Vagrant +# +# macOS +# Install Homebrew from https://brew.sh +# brew cask install virtualbox vagrant +# +# All OSes +# Install from https://www.vagrantup.com/downloads.html + +# 2. One-time: Install Vagrant plugin +vagrant plugin install vagrant-vbguest + +# 3. This is all you need +vagrant up + +# 4. To access the dev environment via ssh +vagrant ssh +cd /vagrant +python tests.py +``` + ## Contributor Notes ## To release a new version: diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..059a62a --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,15 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# https://www.vagrantup.com/docs/ +Vagrant.configure("2") do |config| + config.vm.box = "ubuntu/cosmic64" # https://cloud-images.ubuntu.com + + config.vm.provider "virtualbox" do |vb| + vb.customize ["modifyvm", :id, "--memory", "4096"] + vb.name = "edn_format" + end + + # https://www.vagrantup.com/docs/provisioning/shell.html + config.vm.provision "shell", path: "install.sh", privileged: false, sensitive: true +end diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..2b54eb6 --- /dev/null +++ b/install.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Set working directory to current repository +cd /vagrant + +# Update apt packages list +sudo env DEBIAN_FRONTEND=noninteractive apt-get update + +# Install C++ compiler and Python dependencies +# https://github.com/pyenv/pyenv/wiki/Common-build-problems +sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y make build-essential \ + libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl libncurses5-dev \ + libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev git + +# Install pyenv +# https://github.com/pyenv/pyenv-installer +curl -sL https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash + +echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bashrc +echo 'eval "$(pyenv init -)"' >> ~/.bashrc +echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc + +export PATH="$HOME/.pyenv/bin:$PATH" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +# Install Python 3 +pyenv install 3.7.2 +pyenv global 3.7.2 +pyenv shell 3.7.2 +pip install -U pip # always ensure latest version + +# Install dependencies - Python libraries +pip install -r requirements.txt + +# Run tests +python tests.py From c9aa6659c40b3b85ff0fdeb15c21e37440176d67 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 4 Jan 2019 09:44:15 -0800 Subject: [PATCH 43/65] Use collections.abc instead of collections Fixes #55 --- README.md | 3 +++ edn_format/immutable_dict.py | 4 ++-- edn_format/immutable_list.py | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 15fc32e..e14db96 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,10 @@ vagrant up # 4. To access the dev environment via ssh vagrant ssh cd /vagrant +# To run tests python tests.py +# To check Python warnings +python -Wall -c 'import edn_format' ``` ## Contributor Notes ## diff --git a/edn_format/immutable_dict.py b/edn_format/immutable_dict.py index 9fc3335..03fb89c 100644 --- a/edn_format/immutable_dict.py +++ b/edn_format/immutable_dict.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals -import collections +import collections.abc -class ImmutableDict(collections.Mapping): +class ImmutableDict(collections.abc.Mapping): def __init__(self, somedict): self.dict = dict(somedict) # make a copy self.hash = None diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 7b934f5..8b4643a 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals -import collections +import collections.abc import copy as _copy -class ImmutableList(collections.Sequence, collections.Hashable): +class ImmutableList(collections.abc.Sequence, collections.abc.Hashable): def __init__(self, wrapped_list, copy=True): """Returns an immutable version of the given list. Optionally creates a shallow copy.""" self._list = list(wrapped_list) if copy else wrapped_list From 67d0f1521d24befbcbeba19487d9adaf8241b844 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 4 Jan 2019 09:49:14 -0800 Subject: [PATCH 44/65] Fix build issues https://travis-ci.org/swaroopch/edn_format/jobs/475411817 --- README.md | 2 ++ edn_format/edn_lex.py | 2 +- install.sh | 1 + tests.py | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e14db96..e44f17b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ cd /vagrant python tests.py # To check Python warnings python -Wall -c 'import edn_format' +# Code style +flake8 --max-line-length=100 --exclude=parsetab.py . ``` ## Contributor Notes ## diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index 650c032..f9322e4 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -191,7 +191,7 @@ def t_FLOAT(t): r"""[+-]?\d+(?:\.\d+([eE][+-]?\d+)?|([eE][+-]?\d+))M?""" e_value = 0 if 'e' in t.value or 'E' in t.value: - matches = re.search('[eE]([+-]?\d+)M?$', t.value) + matches = re.search(r'[eE]([+-]?\d+)M?$', t.value) if matches is None: raise EDNDecodeError('Invalid float : {}'.format(t.value)) e_value = int(matches.group(1)) diff --git a/install.sh b/install.sh index 2b54eb6..81bd2ff 100755 --- a/install.sh +++ b/install.sh @@ -32,6 +32,7 @@ pip install -U pip # always ensure latest version # Install dependencies - Python libraries pip install -r requirements.txt +pip install flake8 # Run tests python tests.py diff --git a/tests.py b/tests.py index 702bdee..e9ff0b6 100644 --- a/tests.py +++ b/tests.py @@ -382,8 +382,8 @@ def test_discard(self): ('[42]', '[42 #_ {:a [1 2 3 4] true false 1 #inst "2017"}]'), ('#{1}', '#{1 #_foo}'), ('"#_ foo"', '"#_ foo"'), - ('["#" _]', '[\#_]'), - ('[_]', '[#_\#_]'), + ('["#" _]', r'[\#_]'), + ('[_]', r'[#_\#_]'), ('[1]', '[1 #_\n\n42]'), ('{}', '{#_ 1}'), ): From a1ca4d833ed28e7710aceac61ab4a1764b25ee38 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 4 Jan 2019 09:53:45 -0800 Subject: [PATCH 45/65] Fix Python 2.x compatibility https://travis-ci.org/swaroopch/edn_format/jobs/475413453 --- edn_format/immutable_dict.py | 7 +++++-- edn_format/immutable_list.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/edn_format/immutable_dict.py b/edn_format/immutable_dict.py index 03fb89c..bbbe3d3 100644 --- a/edn_format/immutable_dict.py +++ b/edn_format/immutable_dict.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals -import collections.abc +try: + import collections.abc as collections_abc +except ImportError: + import collections as collections_abc -class ImmutableDict(collections.abc.Mapping): +class ImmutableDict(collections_abc.Mapping): def __init__(self, somedict): self.dict = dict(somedict) # make a copy self.hash = None diff --git a/edn_format/immutable_list.py b/edn_format/immutable_list.py index 8b4643a..80dae45 100644 --- a/edn_format/immutable_list.py +++ b/edn_format/immutable_list.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals -import collections.abc +try: + import collections.abc as collections_abc +except ImportError: + import collections as collections_abc import copy as _copy -class ImmutableList(collections.abc.Sequence, collections.abc.Hashable): +class ImmutableList(collections_abc.Sequence, collections_abc.Hashable): def __init__(self, wrapped_list, copy=True): """Returns an immutable version of the given list. Optionally creates a shallow copy.""" self._list = list(wrapped_list) if copy else wrapped_list From 3a84231a4ba71691c19f60855935533d0376408a Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 4 Jan 2019 09:59:12 -0800 Subject: [PATCH 46/65] Bump version to 0.6.2 --- README.md | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e44f17b..381abd2 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,8 @@ flake8 --max-line-length=100 --exclude=parsetab.py . To release a new version: -1. Bump up the version number in `setup.py`, e.g. `0.6.1` -2. Create a git tag: `git tag -a v0.6.1 -m 'Version 0.6.1'` +1. Bump up the version number in `setup.py`, e.g. `0.6.2` +2. Create a git tag: `git tag -a v0.6.2 -m 'Version 0.6.2'` 3. Push git tag: `git push origin master --tags` 4. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. 5. Run `python setup.py sdist upload` diff --git a/setup.py b/setup.py index 56b416e..64b3901 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup setup(name="edn_format", - version="0.6.1", + version="0.6.2", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From 38c5eae27e7fe8731a900ea278ea70f9e8d921b9 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Fri, 4 Jan 2019 11:09:23 -0800 Subject: [PATCH 47/65] DRY the requirements Move from setup.py to only requirements.txt --- setup.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 64b3901..f145bfa 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,15 @@ - try: from setuptools import setup except ImportError: from distutils.core import setup +try: + from pip.req import parse_requirements +except ImportError: + from pip._internal.req import parse_requirements + +requirements = [str(ir.req) for ir in parse_requirements('requirements.txt', session=False)] + setup(name="edn_format", version="0.6.2", author="Swaroop C H", @@ -11,11 +17,7 @@ description="EDN format reader and writer in Python", long_description=open('README.md').read(), url="https://github.com/swaroopch/edn_format", - install_requires=[ - "pytz>=2016.10", - "pyRFC3339>=0.2", - "ply>=3.10", - ], + install_requires=requirements, license="Apache 2.0", packages=['edn_format'], # http://pypi.python.org/pypi?%3Aaction=list_classifiers From 8a4469a71f809e7a3ad4939487b6d754961cbe17 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Mon, 14 Jan 2019 14:22:34 -0800 Subject: [PATCH 48/65] Add badge for PyPI version --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 381abd2..7f63b48 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Implements the [EDN format](https://github.com/edn-format/edn) in Python. [![Build Status](https://travis-ci.org/swaroopch/edn_format.svg?branch=master)](https://travis-ci.org/swaroopch/edn_format) +[![PyPI version](https://img.shields.io/pypi/v/edn_format.svg)](https://pypi.org/project/edn_format/) ## Installation ## From fb54e2c9821cd31f1b8643c1058e9b99e1969c5e Mon Sep 17 00:00:00 2001 From: LeXofLeviafan Date: Mon, 25 Mar 2019 20:12:18 +0300 Subject: [PATCH 49/65] add support for unicode char literals --- edn_format/edn_lex.py | 7 ++++--- tests.py | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index f9322e4..b98d2a1 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -21,6 +21,7 @@ long = int basestring = str unicode = str +_bytes = (bytes if sys.version_info[0] == 2 else lambda s: bytes(s, 'utf-8')) ESCAPE_SEQUENCE_RE = re.compile(r''' @@ -174,9 +175,9 @@ def t_WHITESPACE(t): def t_CHAR(t): - # from "!" to "~" = all printable ASCII chars except the space - r"(\\[!-~])" - t.value = t.value[1] + # uXXXX hex code or from "!" to "~" = all printable ASCII chars except the space + r"(\\u[0-9A-Fa-f]{4}|\\[!-~\w])" + t.value = (t.value[1] if len(t.value) == 2 else _bytes(t.value).decode('raw_unicode_escape')) return t diff --git a/tests.py b/tests.py index e9ff0b6..73d2998 100644 --- a/tests.py +++ b/tests.py @@ -107,6 +107,10 @@ def test_parser(self): r"\c") self.check_parse("\n", r"\newline") + self.check_parse(u"Σ", + u"\\Σ") + self.check_parse(u"λ", + r"\u03bB") self.check_parse(Keyword("abc"), ":abc") self.check_parse([Keyword("abc"), 1, True, None], From 84303240e660022cf8a8adc50e882dcfda5b410a Mon Sep 17 00:00:00 2001 From: LeXofLeviafan Date: Mon, 25 Mar 2019 21:37:51 +0300 Subject: [PATCH 50/65] fixed indentation in test_sort_keys --- tests.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests.py b/tests.py index 73d2998..f3404ee 100644 --- a/tests.py +++ b/tests.py @@ -339,18 +339,18 @@ def test_keyword_keys(self): loads(dumps({Keyword("a"): {"b": 2}, 3: 4}, keyword_keys=True))) def test_sort_keys(self): - cases = ( - ('{"a" 4 "b" 5 "c" 3}', OrderedDict([("c", 3), ("b", 5), ("a", 4)])), - ('{1 0 2 0 "a" 0}', {"a": 0, 1: 0, 2: 0}), - ('[{"a" 1 "b" 1}]', [OrderedDict([("b", 1), ("a", 1)])]), - ) - - for expected, data in cases: - self.check_dumps(expected, data, sort_keys=True) - - self.check_dumps('{:a 1 :b 1 :c 1 :d 1}', - {"a": 1, "d": 1, "b": 1, "c": 1}, - sort_keys=True, keyword_keys=True) + cases = ( + ('{"a" 4 "b" 5 "c" 3}', OrderedDict([("c", 3), ("b", 5), ("a", 4)])), + ('{1 0 2 0 "a" 0}', {"a": 0, 1: 0, 2: 0}), + ('[{"a" 1 "b" 1}]', [OrderedDict([("b", 1), ("a", 1)])]), + ) + + for expected, data in cases: + self.check_dumps(expected, data, sort_keys=True) + + self.check_dumps('{:a 1 :b 1 :c 1 :d 1}', + {"a": 1, "d": 1, "b": 1, "c": 1}, + sort_keys=True, keyword_keys=True) def test_sort_sets(self): def misordered_set_sequence(): From a83e61d492f38964d16069e1addd1e5a33fcbf58 Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Wed, 3 Apr 2019 20:08:11 +0300 Subject: [PATCH 51/65] combined version_info checks in edn_format/edn_lex.py Co-Authored-By: LeXofLeviafan --- edn_format/edn_lex.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index b98d2a1..768b464 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -21,7 +21,9 @@ long = int basestring = str unicode = str -_bytes = (bytes if sys.version_info[0] == 2 else lambda s: bytes(s, 'utf-8')) + _bytes = lambda s: bytes(s, 'utf-8') +else: + _bytes = bytes ESCAPE_SEQUENCE_RE = re.compile(r''' From 1a668524b8bc5bf3742229bccfacac8de828e87d Mon Sep 17 00:00:00 2001 From: LeXofLeviafan Date: Wed, 3 Apr 2019 20:18:33 +0300 Subject: [PATCH 52/65] fixed comment in char token definition --- edn_format/edn_lex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index 768b464..bed28cf 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -177,7 +177,7 @@ def t_WHITESPACE(t): def t_CHAR(t): - # uXXXX hex code or from "!" to "~" = all printable ASCII chars except the space + # uXXXX hex code or from "!" to "~" = all printable ASCII chars except the space or unicode word chars r"(\\u[0-9A-Fa-f]{4}|\\[!-~\w])" t.value = (t.value[1] if len(t.value) == 2 else _bytes(t.value).decode('raw_unicode_escape')) return t From 434a8e8eeb1799493e739ebddef1ac92643fc29f Mon Sep 17 00:00:00 2001 From: LeXofLeviafan Date: Wed, 3 Apr 2019 20:44:16 +0300 Subject: [PATCH 53/65] made changes according to flake8 demands --- edn_format/edn_lex.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index bed28cf..04c6372 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -21,7 +21,7 @@ long = int basestring = str unicode = str - _bytes = lambda s: bytes(s, 'utf-8') + def _bytes(s): return bytes(s, 'utf-8') else: _bytes = bytes @@ -177,7 +177,8 @@ def t_WHITESPACE(t): def t_CHAR(t): - # uXXXX hex code or from "!" to "~" = all printable ASCII chars except the space or unicode word chars + # uXXXX hex code or from "!" to "~" = all printable ASCII chars except the space + # or unicode word chars r"(\\u[0-9A-Fa-f]{4}|\\[!-~\w])" t.value = (t.value[1] if len(t.value) == 2 else _bytes(t.value).decode('raw_unicode_escape')) return t From 258fa17c965b4b7115963b5dfe2da2437bf5be0c Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Tue, 16 Apr 2019 15:21:48 +0530 Subject: [PATCH 54/65] Bump version to 0.6.3 --- README.md | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7f63b48..d33a7f6 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ flake8 --max-line-length=100 --exclude=parsetab.py . To release a new version: -1. Bump up the version number in `setup.py`, e.g. `0.6.2` -2. Create a git tag: `git tag -a v0.6.2 -m 'Version 0.6.2'` +1. Bump up the version number in `setup.py`, e.g. `0.6.3` +2. Create a git tag: `git tag -a v0.6.3 -m 'Version 0.6.3'` 3. Push git tag: `git push origin master --tags` 4. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. 5. Run `python setup.py sdist upload` diff --git a/setup.py b/setup.py index f145bfa..0ff0442 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ requirements = [str(ir.req) for ir in parse_requirements('requirements.txt', session=False)] setup(name="edn_format", - version="0.6.2", + version="0.6.3", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From 8878b35715187ff5bc9309fbf4f2e0d5b4f962ad Mon Sep 17 00:00:00 2001 From: Swaroop C H Date: Tue, 16 Apr 2019 17:33:26 +0530 Subject: [PATCH 55/65] Travis : Upgrade Ubuntu version to latest LTS, bionic --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9c3f252..c6da741 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: xenial # Required for Python 3.7, see https://github.com/travis-ci/travis-ci/issues/9069 +dist: bionic # latest LTS as per https://wiki.ubuntu.com/Releases language: python python: - "2.7" From 727105f52a33a5f7808f05488910e885164eb5ce Mon Sep 17 00:00:00 2001 From: Swaroop Chitlur Date: Sun, 23 Jun 2019 22:40:39 -0700 Subject: [PATCH 56/65] Use tag signing --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d33a7f6..721eb33 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,10 @@ flake8 --max-line-length=100 --exclude=parsetab.py . To release a new version: -1. Bump up the version number in `setup.py`, e.g. `0.6.3` -2. Create a git tag: `git tag -a v0.6.3 -m 'Version 0.6.3'` -3. Push git tag: `git push origin master --tags` -4. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. -5. Run `python setup.py sdist upload` +1. Ensure you have [setup GPG](https://help.github.com/en/articles/generating-a-new-gpg-key) +2. Bump up the version number in `setup.py`, e.g. `0.6.3` +3. Create a git tag: `git tag -s v0.6.3 -m 'Version 0.6.3'` (use [signed tags](https://help.github.com/en/articles/signing-tags)) +4. Verify git tag: `git tag -v v0.6.3` +5. Push git tag: `git push origin master --tags` +6. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. +7. Run `python setup.py sdist upload` From 3ac726eba893705415aa4676f547fa03c5653822 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 14 Sep 2019 20:26:06 +0200 Subject: [PATCH 57/65] Add an edn_parse.tag decorator, + docs and tests on tags (#59) --- edn_format/__init__.py | 3 ++- edn_format/edn_parse.py | 34 ++++++++++++++++++++++++++++++++-- tests.py | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/edn_format/__init__.py b/edn_format/__init__.py index e1e438f..5f0790e 100644 --- a/edn_format/__init__.py +++ b/edn_format/__init__.py @@ -3,7 +3,7 @@ from .edn_lex import Keyword, Symbol from .edn_parse import parse as loads -from .edn_parse import add_tag, remove_tag, TaggedElement +from .edn_parse import add_tag, remove_tag, tag, TaggedElement from .edn_dump import dump as dumps from .exceptions import EDNDecodeError from .immutable_dict import ImmutableDict @@ -20,4 +20,5 @@ 'dumps', 'loads', 'remove_tag', + 'tag', ) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index e202c9c..9259605 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -34,16 +34,46 @@ def __str__(self): raise NotImplementedError("To be implemented by derived classes") -def add_tag(tag_name, tag_class): +def add_tag(tag_name, tag_fn_or_class): + """ + Add a custom tag handler. + """ assert isinstance(tag_name, basestring) - _serializers[tag_name] = tag_class + _serializers[tag_name] = tag_fn_or_class def remove_tag(tag_name): + """ + Remove a custom tag handler. + """ assert isinstance(tag_name, basestring) del _serializers[tag_name] +def tag(tag_name): + """ + Decorator for a function or a class to use as a tagged element. + + Example: + + @tag("dog") + def parse_dog(name): + return { + "kind": "dog", + "name": name, + "message": "woof-woof", + } + + parse("#dog \"Max\"") + # => {"kind": "dog", "name": "Max", "message": "woof-woof"} + """ + + def _tag_decorator(fn_or_cls): + _serializers[tag_name] = fn_or_cls + return fn_or_cls + return _tag_decorator + + def p_term_leaf(p): """term : CHAR | STRING diff --git a/tests.py b/tests.py index f3404ee..b7a62f0 100644 --- a/tests.py +++ b/tests.py @@ -12,7 +12,8 @@ import pytz from edn_format import edn_lex, edn_parse, \ - loads, dumps, Keyword, Symbol, TaggedElement, ImmutableDict, ImmutableList, add_tag, \ + loads, dumps, Keyword, Symbol, ImmutableDict, ImmutableList, \ + TaggedElement, add_tag, remove_tag, tag, \ EDNDecodeError @@ -422,6 +423,36 @@ def test_chained_discards(self): ): self.assertEqual(expected, dumps(loads(edn_data)), edn_data) + def test_custom_tags(self): + @tag("dog") + def parse_dog(name): + return { + "kind": "dog", + "name": name, + "message": "woof-woof", + } + + @tag("cat") + class Cat(TaggedElement): + def __init__(self, name): + self.name = name + + dog = loads("#dog \"Max\"") + self.assertEqual( + {"kind": "dog", "name": "Max", "message": "woof-woof"}, dog) + + cat = loads("#cat \"Alex\"") + self.assertIsInstance(cat, Cat) + self.assertEqual("Alex", cat.name) + + remove_tag("cat") + self.assertRaises(NotImplementedError, lambda: loads("#cat \"Alex\"")) + + add_tag("cat", Cat) + cat = loads("#cat \"Alex\"") + self.assertIsInstance(cat, Cat) + self.assertEqual("Alex", cat.name) + class EdnInstanceTest(unittest.TestCase): def test_hashing(self): From 969f0622e1d4f13e5d39bb5c5dcb2bc347465f39 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 14 Sep 2019 20:42:28 +0200 Subject: [PATCH 58/65] Contributor Notes: replace the deprecated commands --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 721eb33..72ae3fb 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,12 @@ flake8 --max-line-length=100 --exclude=parsetab.py . To release a new version: -1. Ensure you have [setup GPG](https://help.github.com/en/articles/generating-a-new-gpg-key) +1. Ensure you have [setup GPG](https://help.github.com/en/articles/generating-a-new-gpg-key) and [`twine`](https://pypi.org/project/twine/) 2. Bump up the version number in `setup.py`, e.g. `0.6.3` 3. Create a git tag: `git tag -s v0.6.3 -m 'Version 0.6.3'` (use [signed tags](https://help.github.com/en/articles/signing-tags)) 4. Verify git tag: `git tag -v v0.6.3` 5. Push git tag: `git push origin master --tags` -6. Make sure you have a [~/.pypirc file](http://docs.python.org/2/distutils/packageindex.html#pypirc) with your PyPI credentials. -7. Run `python setup.py sdist upload` +6. Clean your `dist/` directory if it already exists +7. Package the release: `python setup.py sdist bdist_wheel` +8. Check the package: `twine check dist/*` +9. Upload the package: `twine upload dist/*` From a885ee5771658bdd0307cecf91ffa9cb07560ee4 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sat, 14 Sep 2019 20:42:35 +0200 Subject: [PATCH 59/65] Version 0.6.4 --- CHANGELOG.md | 12 ++++++++++++ setup.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2158e38..c6f4d15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # `edn_format` Changelog +## v0.6.4 (2019/09/14) + +* Add an `edn_parse.tag` decorator + +## v0.6.3 (2019/04/16) + +* Add support for Unicode char literals + +## v0.6.2 (2019/01/04) + +* Parse `nil` and booleans as symbols + ## v0.6.1 (2018/09/21) * Use mutable data structures to improve parsing time diff --git a/setup.py b/setup.py index 0ff0442..2888878 100644 --- a/setup.py +++ b/setup.py @@ -11,11 +11,12 @@ requirements = [str(ir.req) for ir in parse_requirements('requirements.txt', session=False)] setup(name="edn_format", - version="0.6.3", + version="0.6.4", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", long_description=open('README.md').read(), + long_description_content_type='text/markdown', url="https://github.com/swaroopch/edn_format", install_requires=requirements, license="Apache 2.0", From bd49d44bc58e1e783b23717877e9d3c080be181e Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Wed, 2 Oct 2019 20:28:35 +0200 Subject: [PATCH 60/65] Move contributor notes in CONTRIBUTING.md (#61) --- CONTRIBUTING.md | 46 +++++++++++++++++++++++++++++++++++++++++--- README.md | 51 ++----------------------------------------------- 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cadb748..2237531 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,8 +2,30 @@ ## Setup ## -1. Make sure you have a working Python installation (2 or 3). You may want to - use a [Virtualenv][] or a similar tool to create an isolated environment. +1. Make sure you have a working Python installation (2 or 3). Use either + [Virtualenv][], Vagrant, or a similar tool to create an isolated + environment: + ```bash + # 1. One-time: Install Vagrant + # + # macOS + # Install Homebrew from https://brew.sh + # brew cask install virtualbox vagrant + # + # All OSes + # Install from https://www.vagrantup.com/downloads.html + + # 2. One-time: Install Vagrant plugin + vagrant plugin install vagrant-vbguest + + # 3. This is all you need + vagrant up + + # 4. To access the dev environment via ssh + vagrant ssh + cd /vagrant + ``` + 2. Install `edn_format`’s dependencies: `pip install -r requirements.txt` 3. Install `flake8`: `pip install flake8` @@ -15,6 +37,24 @@ Run unit tests with: python tests.py +Check Python warnings with: + + python -Wall -c 'import edn_format' + Run a linter over the code with: - flake8 --max-line-length=100 --exclude=parsetab.py tests.py + flake8 --max-line-length=100 --exclude=parsetab.py . + +## Release a new version ## + +To release a new version: + +1. Ensure you have [setup GPG](https://help.github.com/en/articles/generating-a-new-gpg-key) and [`twine`](https://pypi.org/project/twine/) +2. Bump up the version number in `setup.py`, e.g. `0.6.3` +3. Create a git tag: `git tag -s v0.6.3 -m 'Version 0.6.3'` (use [signed tags](https://help.github.com/en/articles/signing-tags)) +4. Verify git tag: `git tag -v v0.6.3` +5. Push git tag: `git push origin master --tags` +6. Clean your `dist/` directory if it already exists +7. Package the release: `python setup.py sdist bdist_wheel` +8. Check the package: `twine check dist/*` +9. Upload the package: `twine upload dist/*` diff --git a/README.md b/README.md index 72ae3fb..9211429 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Implements the [EDN format](https://github.com/edn-format/edn) in Python. +All features of EDN are implemented, including custom tagged elements. + [![Build Status](https://travis-ci.org/swaroopch/edn_format.svg?branch=master)](https://travis-ci.org/swaroopch/edn_format) [![PyPI version](https://img.shields.io/pypi/v/edn_format.svg)](https://pypi.org/project/edn_format/) @@ -25,12 +27,6 @@ false, it may be a bug. See `tests.py` for full details. -## Caveats ## - -All features of EDN have been implemented, including custom tagged elements. - -However, I personally don't use this in production, even though many users do, esp. the active contributors below. - ## Contributors ## Special thanks to the following contributors for making this library @@ -41,46 +37,3 @@ usable: - [@bitemyapp](https://github.com/bitemyapp) - [@jashugan](https://github.com/jashugan) - [@exilef](https://github.com/exilef) - -## Local Dev ## - -```bash -# 1. One-time: Install Vagrant -# -# macOS -# Install Homebrew from https://brew.sh -# brew cask install virtualbox vagrant -# -# All OSes -# Install from https://www.vagrantup.com/downloads.html - -# 2. One-time: Install Vagrant plugin -vagrant plugin install vagrant-vbguest - -# 3. This is all you need -vagrant up - -# 4. To access the dev environment via ssh -vagrant ssh -cd /vagrant -# To run tests -python tests.py -# To check Python warnings -python -Wall -c 'import edn_format' -# Code style -flake8 --max-line-length=100 --exclude=parsetab.py . -``` - -## Contributor Notes ## - -To release a new version: - -1. Ensure you have [setup GPG](https://help.github.com/en/articles/generating-a-new-gpg-key) and [`twine`](https://pypi.org/project/twine/) -2. Bump up the version number in `setup.py`, e.g. `0.6.3` -3. Create a git tag: `git tag -s v0.6.3 -m 'Version 0.6.3'` (use [signed tags](https://help.github.com/en/articles/signing-tags)) -4. Verify git tag: `git tag -v v0.6.3` -5. Push git tag: `git push origin master --tags` -6. Clean your `dist/` directory if it already exists -7. Package the release: `python setup.py sdist bdist_wheel` -8. Check the package: `twine check dist/*` -9. Upload the package: `twine upload dist/*` From 0d4c2136663b57bef913b321313f2651ca6649a3 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Wed, 2 Oct 2019 20:29:02 +0200 Subject: [PATCH 61/65] Add edn_format.loads_all to parse all expressions in a string (#60) --- README.md | 2 + edn_format/__init__.py | 3 +- edn_format/edn_parse.py | 18 +++++- tests.py | 119 +++++++++++++++++++++------------------- 4 files changed, 82 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 9211429..f5b0ec2 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ All features of EDN are implemented, including custom tagged elements. '#{1 2 3}' >>> edn_format.loads("[1 true nil]") [1, True, None] +>>> edn_format.loads_all("1 2 3 4") +[1, 2, 3, 4] ``` diff --git a/edn_format/__init__.py b/edn_format/__init__.py index 5f0790e..660391b 100644 --- a/edn_format/__init__.py +++ b/edn_format/__init__.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals from .edn_lex import Keyword, Symbol -from .edn_parse import parse as loads +from .edn_parse import parse as loads, parse_all as loads_all from .edn_parse import add_tag, remove_tag, tag, TaggedElement from .edn_dump import dump as dumps from .exceptions import EDNDecodeError @@ -19,6 +19,7 @@ 'add_tag', 'dumps', 'loads', + 'loads_all', 'remove_tag', 'tag', ) diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index 9259605..c43c034 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -24,7 +24,7 @@ if tokens: pass -start = 'expression' +start = 'expressions' _serializers = {} @@ -180,7 +180,10 @@ def p_error(p): raise EDNDecodeError(p) -def parse(text, input_encoding='utf-8'): +def parse_all(text, input_encoding='utf-8'): + """ + Parse all objects from the text and return a (possibly empty) list. + """ if not isinstance(text, unicode): text = text.decode(input_encoding) @@ -188,4 +191,13 @@ def parse(text, input_encoding='utf-8'): if __debug__: kwargs = dict(debug=True) p = ply.yacc.yacc(**kwargs) - return p.parse(text, lexer=lex()) + expressions = p.parse(text, lexer=lex()) + return list(expressions) + + +def parse(text, input_encoding='utf-8'): + """ + Parse one object from the text. Return None if the text is empty. + """ + expressions = parse_all(text, input_encoding=input_encoding) + return expressions[0] if expressions else None diff --git a/tests.py b/tests.py index b7a62f0..0a5fd79 100644 --- a/tests.py +++ b/tests.py @@ -84,64 +84,67 @@ def test_lexer(self): def check_parse(self, expected_output, actual_input): self.assertEqual(expected_output, edn_parse.parse(actual_input)) + def check_parse_all(self, expected_output, actual_input): + self.assertEqual(expected_output, edn_parse.parse_all(actual_input)) + def check_dumps(self, expected_output, actual_input, **kw): self.assertEqual(expected_output, dumps(actual_input, **kw)) - def test_parser(self): - self.check_parse(1, - "1") - self.check_parse(Symbol("a*b"), - 'a*b') - self.check_parse("ab", - '"ab"') - self.check_parse('a"b', - r'"a\"b"') - self.check_parse("blah\n", - '"blah\n"') - self.check_parse([1, 2, 3], - "[1 2 3]") - self.check_parse({1, 2, 3}, - "#{1 2 3}") - self.check_parse([1, True, None], - "[1 true nil]") - self.check_parse("c", - r"\c") - self.check_parse("\n", - r"\newline") - self.check_parse(u"Σ", - u"\\Σ") - self.check_parse(u"λ", - r"\u03bB") - self.check_parse(Keyword("abc"), - ":abc") - self.check_parse([Keyword("abc"), 1, True, None], - "[:abc 1 true nil]") - self.check_parse((Keyword("abc"), 1, True, None), - "(:abc 1 true nil)") - self.check_parse(tuple(), "()") - self.check_parse(set(), "#{}") - self.check_parse({}, "{}") - self.check_parse([], "[]") - self.check_parse({"a": [1, 2, 3]}, - '{"a" [1 2 3]}') - self.check_parse(datetime.datetime(2012, 12, 22, 19, 40, 18, 0, - tzinfo=pytz.utc), - '#inst "2012-12-22T19:40:18Z"') - self.check_parse(datetime.date(2011, 10, 9), - '#inst "2011-10-09"') - self.check_parse("|", "\"|\"") - self.check_parse("%", "\"%\"") - self.check_parse(['bl"ah'], r"""["bl\"ah"]""") - self.check_parse("blah\n", '"blah\n"') - self.check_parse('"', r'"\""') - self.check_parse('\\', r'"\\"') - self.check_parse(["abc", "123"], '["abc", "123"]') - self.check_parse({"key": "value"}, '{"key" "value"}') - self.check_parse(frozenset({ImmutableList([u"ab", u"cd"]), - ImmutableList([u"ef"])}), - '#{["ab", "cd"], ["ef"]}') - self.check_parse(fractions.Fraction(2, 3), "2/3") - self.check_parse((2, Symbol('/'), 3), "(2 / 3)") + def test_parser_single_expressions(self): + for expected, edn_string in ( + (1, "1"), + (Symbol("a*b"), 'a*b'), + ("ab", '"ab"'), + ('a"b', r'"a\"b"'), + ("blah\n", '"blah\n"'), + ([1, 2, 3], "[1 2 3]"), + ({1, 2, 3}, "#{1 2 3}"), + ([1, True, None], "[1 true nil]"), + ("c", r"\c"), + ("\n", r"\newline"), + (u"Σ", u"\\Σ"), + (u"λ", r"\u03bB"), + (Keyword("abc"), ":abc"), + ([Keyword("abc"), 1, True, None], "[:abc 1 true nil]"), + ((Keyword("abc"), 1, True, None), "(:abc 1 true nil)"), + (tuple(), "()"), + (set(), "#{}"), + ({}, "{}"), + ([], "[]"), + ({"a": [1, 2, 3]}, '{"a" [1 2 3]}'), + (datetime.datetime(2012, 12, 22, 19, 40, 18, 0, tzinfo=pytz.utc), + '#inst "2012-12-22T19:40:18Z"'), + (datetime.date(2011, 10, 9), + '#inst "2011-10-09"'), + ("|", "\"|\""), + ("%", "\"%\""), + (['bl"ah'], r"""["bl\"ah"]"""), + ("blah\n", '"blah\n"'), + ('"', r'"\""'), + ('\\', r'"\\"'), + (["abc", "123"], '["abc", "123"]'), + ({"key": "value"}, '{"key" "value"}'), + (frozenset({ImmutableList([u"ab", u"cd"]), ImmutableList([u"ef"])}), + '#{["ab", "cd"], ["ef"]}'), + (fractions.Fraction(2, 3), "2/3"), + ((2, Symbol('/'), 3), "(2 / 3)"), + ): + self.check_parse(expected, edn_string) + self.check_parse_all([expected], edn_string) + + def test_parser_multiple_expressions(self): + for expected, edn_string in ( + ([], ""), + ([], " ,,,, ,, , "), + ([1], ",,,,,,,1,,,,,,,,,"), + ([1, 2], "1 2"), + ([1, 2], "1 2"), + ([True, 42, False, Symbol('end')], "true 42 false end"), + ([Symbol("a*b"), 42], 'a*b 42'), + ): + self.check_parse_all(expected, edn_string) + if expected: + self.check_parse(expected[0], edn_string) def check_roundtrip(self, data_input, **kw): self.assertEqual(data_input, loads(dumps(data_input, **kw))) @@ -152,6 +155,10 @@ def check_eof(self, data_input, **kw): self.assertEqual('EOF Reached', str(ctx.exception)) + def check_mismatched_delimiters(self): + for bad_string in ("[", "(", "{", "(((((())", '"', '"\\"'): + self.check_eof(bad_string) + def test_dump(self): self.check_roundtrip({1, 2, 3}) self.check_roundtrip({1, 2, 3}, sort_sets=True) @@ -408,7 +415,7 @@ def test_discard_all(self): self.assertEqual([1], loads('[1 #_ {}]'.format(edn_data)), edn_data) self.assertEqual([1], loads('[#_ {} 1]'.format(edn_data)), edn_data) - self.check_eof('#_ {}'.format(edn_data)) + self.assertEqual(None, loads('#_ {}'.format(edn_data))) for coll in ('[%s]', '(%s)', '{%s}', '#{%s}'): expected = coll % "" From 9199f9b872c8e8c0749a1b2e503eed5b23340515 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Fri, 1 Nov 2019 18:21:44 +0100 Subject: [PATCH 62/65] Disallow 0-prefixed integers From the spec: > No integer other than 0 may begin with 0. --- edn_format/edn_lex.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index 04c6372..71df129 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -215,7 +215,9 @@ def t_RATIO(t): def t_INTEGER(t): - r"""[+-]?\d+N?""" + # "No integer other than 0 may begin with 0." + # https://github.com/edn-format/edn#integers + r"""[+-]?(?:0|[1-9]\d*)N?""" if t.value.endswith('N'): t.value = t.value[:-1] t.value = int(t.value) From eba86863ee6a45ea0d6d03d2bd69037c321f5485 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Fri, 1 Nov 2019 18:11:30 +0100 Subject: [PATCH 63/65] Parse hexadecimal notation Fixes #62. --- edn_format/edn_lex.py | 9 ++++++++- edn_format/edn_parse.py | 1 + tests.py | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py index 71df129..836e84c 100644 --- a/edn_format/edn_lex.py +++ b/edn_format/edn_lex.py @@ -94,6 +94,7 @@ def __str__(self): 'CHAR', 'STRING', 'INTEGER', + 'HEX_INTEGER', 'FLOAT', 'RATIO', 'SYMBOL', @@ -217,13 +218,19 @@ def t_RATIO(t): def t_INTEGER(t): # "No integer other than 0 may begin with 0." # https://github.com/edn-format/edn#integers - r"""[+-]?(?:0|[1-9]\d*)N?""" + r"""[+-]?(?:0(?!x)|[1-9]\d*)N?""" if t.value.endswith('N'): t.value = t.value[:-1] t.value = int(t.value) return t +def t_HEX_INTEGER(t): + r"""[+-]?0x[A-F0-9]+""" + t.value = int(t.value, 16) + return t + + def t_COMMENT(t): r'[;][^\n]*' pass # ignore diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py index c43c034..c4e75a4 100644 --- a/edn_format/edn_parse.py +++ b/edn_format/edn_parse.py @@ -77,6 +77,7 @@ def _tag_decorator(fn_or_cls): def p_term_leaf(p): """term : CHAR | STRING + | HEX_INTEGER | INTEGER | FLOAT | KEYWORD diff --git a/tests.py b/tests.py index 0a5fd79..14839a9 100644 --- a/tests.py +++ b/tests.py @@ -38,6 +38,8 @@ def test_lexer(self): "true") self.check_lex("[LexToken(INTEGER,123,1,0)]", "123") + self.check_lex("[LexToken(HEX_INTEGER,1,1,0)]", + "0x1") self.check_lex( "[LexToken(INTEGER,456,1,0), " + "LexToken(SYMBOL,None,1,4), " + @@ -93,6 +95,8 @@ def check_dumps(self, expected_output, actual_input, **kw): def test_parser_single_expressions(self): for expected, edn_string in ( (1, "1"), + (16768115, "0xFFDC73"), + (Symbol("xFF"), "xFF"), (Symbol("a*b"), 'a*b'), ("ab", '"ab"'), ('a"b', r'"a\"b"'), @@ -138,6 +142,7 @@ def test_parser_multiple_expressions(self): ([], " ,,,, ,, , "), ([1], ",,,,,,,1,,,,,,,,,"), ([1, 2], "1 2"), + ([0, Symbol("x1"), 1], "0 x1 0x1"), ([1, 2], "1 2"), ([True, 42, False, Symbol('end')], "true 42 false end"), ([Symbol("a*b"), 42], 'a*b 42'), From 09add5d4a317bd6daa54f8298370df539f2ad3d2 Mon Sep 17 00:00:00 2001 From: Baptiste Fontaine Date: Sun, 3 Nov 2019 20:52:23 +0100 Subject: [PATCH 64/65] Release v0.6.5 --- CHANGELOG.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6f4d15..53cc10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # `edn_format` Changelog +## v0.6.5 (2019/11/03) + +* Parse integers in hexadecimal notation +* Disallow `0`-prefixed integers other than zero itself + ## v0.6.4 (2019/09/14) * Add an `edn_parse.tag` decorator diff --git a/setup.py b/setup.py index 2888878..f17917a 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ requirements = [str(ir.req) for ir in parse_requirements('requirements.txt', session=False)] setup(name="edn_format", - version="0.6.4", + version="0.6.5", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python", From 7dd6a1333d992a01c97849b51cb1ec5c4d9a2c58 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Fri, 22 Nov 2019 20:05:28 -0300 Subject: [PATCH 65/65] Bump version to 0.6.5-nu --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 195fd68..a6871c9 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup(name="edn_format", - version="0.5.16-nu", + version="0.6.5-nu", author="Swaroop C H", author_email="swaroop@swaroopch.com", description="EDN format reader and writer in Python",