From e94735a40944bd0790cf46ec7e17a27e7c34f281 Mon Sep 17 00:00:00 2001 From: Peter Schoener Date: Thu, 4 May 2023 18:43:13 -0700 Subject: [PATCH 01/28] add `to_list()` for `TokenTree` Conversely to `TokenList.to_tree()`, this allows us to convert a tree back to a list, preserving order, for example in case we need to reconstruct the surface form of a sentence or phrase. --- conllu/models.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/conllu/models.py b/conllu/models.py index 8de46bd..25e5b3b 100644 --- a/conllu/models.py +++ b/conllu/models.py @@ -247,23 +247,26 @@ def __eq__(self, other: T.Any) -> bool: and self.metadata == other.metadata return False - def serialize(self) -> str: - if not self.token or "id" not in self.token: - raise ParseException("Could not serialize tree, missing 'id' field.") - - def flatten_tree(root_token: TokenTree, token_list: T.List[Token] = []) -> T.List[Token]: + def to_list(self): + def _to_list(root_token: TokenTree, token_list: T.List[Token] = []) -> T.List[Token]: token_list.append(root_token.token) for child_token in root_token.children: - flatten_tree(child_token, token_list) + _to_list(child_token, token_list) return token_list - tokens = flatten_tree(self) - tokens = sorted(tokens, key=lambda t: t['id']) - tokenlist = TokenList(tokens, self.metadata) + if not self.token or "id" not in self.token: + raise ParseException("Could not flatten tree; missing 'id' field.") - return serialize(tokenlist) + token_list = _to_list(self) + token_list = sorted(token_list, key=lambda t: t['id']) + token_list = TokenList(token_list, self.metadata) + + return token_list + + def serialize(self) -> str: + return serialize(self.to_list()) def print_tree(self, depth: int = 0, indent: int = 4, exclude_fields: T.Sequence[str] = DEFAULT_EXCLUDE_FIELDS) -> None: From 3d80a2ae8fa45b7483d6e05d8020677628eb6397 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jul 2023 01:22:27 +0000 Subject: [PATCH 02/28] Bump cryptography from 41.0.0 to 41.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.0 to 41.0.2. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.0...41.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e78ded3..0ff199a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==41.0.0 +cryptography==41.0.2 # via secretstorage distlib==0.3.6 # via virtualenv From 67e24323cf836d94de39acca359077e53c12c8d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:05:21 +0000 Subject: [PATCH 03/28] Bump pygments from 2.13.0 to 2.15.0 Bumps [pygments](https://github.com/pygments/pygments) from 2.13.0 to 2.15.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.13.0...2.15.0) --- updated-dependencies: - dependency-name: pygments dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0ff199a..5349786 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -92,7 +92,7 @@ pycparser==2.21 # via cffi pyflakes==3.0.1 # via flake8 -pygments==2.13.0 +pygments==2.15.0 # via # readme-renderer # rich From 4d51d48942a39f00370c1bdd2092274183508e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 20:45:19 +0000 Subject: [PATCH 04/28] Bump certifi from 2022.12.7 to 2023.7.22 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5349786..bb80b85 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ build==0.9.0 # via pip-tools cachetools==5.2.0 # via tox -certifi==2022.12.7 +certifi==2023.7.22 # via requests cffi==1.15.1 # via cryptography From b524d02d46ef5f60838b32d5ce781f1f85f39aea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 02:22:52 +0000 Subject: [PATCH 05/28] Bump cryptography from 41.0.2 to 41.0.3 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.2 to 41.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.2...41.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index bb80b85..035be23 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==41.0.2 +cryptography==41.0.3 # via secretstorage distlib==0.3.6 # via virtualenv From 41a48e1c1510edbf27dd1d1590f8ccf992db5cc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:00:51 +0000 Subject: [PATCH 06/28] Bump cryptography from 41.0.3 to 41.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 035be23..639a2f8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==41.0.3 +cryptography==41.0.4 # via secretstorage distlib==0.3.6 # via virtualenv From 2fdca0661577c707135aaaab18ab91ad6d1997bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 02:33:01 +0000 Subject: [PATCH 07/28] Bump urllib3 from 1.26.13 to 1.26.17 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.13 to 1.26.17. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.13...1.26.17) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 639a2f8..ccf1f1c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -122,7 +122,7 @@ twine==4.0.2 # via -r requirements-dev.in typing-extensions==4.4.0 # via mypy -urllib3==1.26.13 +urllib3==1.26.17 # via # requests # twine From 537e304c941591b622b471e18b9f85c86dc5d128 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 01:18:11 +0000 Subject: [PATCH 08/28] Bump urllib3 from 1.26.17 to 1.26.18 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.17 to 1.26.18. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.17...1.26.18) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ccf1f1c..a20f78b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -122,7 +122,7 @@ twine==4.0.2 # via -r requirements-dev.in typing-extensions==4.4.0 # via mypy -urllib3==1.26.17 +urllib3==1.26.18 # via # requests # twine From 3f7a0e9a8458e3b9d78ba713117edf545932ca9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 00:09:55 +0000 Subject: [PATCH 09/28] Bump cryptography from 41.0.4 to 41.0.6 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.4 to 41.0.6. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.4...41.0.6) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a20f78b..e4b0877 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==41.0.4 +cryptography==41.0.6 # via secretstorage distlib==0.3.6 # via virtualenv From 0a1c9d43a1152ade210ca2a860ecee8e57361b50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:08:04 +0000 Subject: [PATCH 10/28] Bump cryptography from 41.0.6 to 42.0.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.6 to 42.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.6...42.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e4b0877..05dd11e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==41.0.6 +cryptography==42.0.0 # via secretstorage distlib==0.3.6 # via virtualenv From 065c5b82940cf108bb7df2cd50ce346a33f71ec0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Feb 2024 00:54:21 +0000 Subject: [PATCH 11/28] Bump cryptography from 42.0.0 to 42.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.0 to 42.0.2. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.0...42.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 05dd11e..cc292bb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==42.0.0 +cryptography==42.0.2 # via secretstorage distlib==0.3.6 # via virtualenv From b565c08bfec47d56cd7e4bbd12ceaacdf0ecca77 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 20:46:55 +0000 Subject: [PATCH 12/28] Bump cryptography from 42.0.2 to 42.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.2 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.2...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cc292bb..4bf7458 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,7 @@ commonmark==0.9.1 # via rich coverage==7.0.1 # via -r requirements-dev.in -cryptography==42.0.2 +cryptography==42.0.4 # via secretstorage distlib==0.3.6 # via virtualenv From a1474c50933ad6b2414eb08761208cc5d5305455 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 01:52:47 +0000 Subject: [PATCH 13/28] Bump idna from 3.4 to 3.7 Bumps [idna](https://github.com/kjd/idna) from 3.4 to 3.7. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.4...v3.7) --- updated-dependencies: - dependency-name: idna dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4bf7458..f30fb7c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,7 +40,7 @@ filelock==3.8.2 # virtualenv flake8==6.0.0 # via -r requirements-dev.in -idna==3.4 +idna==3.7 # via requests importlib-metadata==5.2.0 # via From 1feaac96a3fffd4e54392ad2e476dcdd0b194727 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 05:27:10 +0000 Subject: [PATCH 14/28] --- updated-dependencies: - dependency-name: requests dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f30fb7c..04d63a3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -102,7 +102,7 @@ pytest==7.2.0 # via -r requirements-dev.in readme-renderer==37.3 # via twine -requests==2.31.0 +requests==2.32.0 # via # requests-toolbelt # twine From e0d5bb0df56986754455b59f084210438767eecd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 01:31:00 +0000 Subject: [PATCH 15/28] Bump urllib3 from 1.26.18 to 1.26.19 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.18 to 1.26.19. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.19/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.18...1.26.19) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 04d63a3..287ab58 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -122,7 +122,7 @@ twine==4.0.2 # via -r requirements-dev.in typing-extensions==4.4.0 # via mypy -urllib3==1.26.18 +urllib3==1.26.19 # via # requests # twine From bd6d3f10610843ac316219810d700e8d83de9017 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:05:34 +0000 Subject: [PATCH 16/28] Bump zipp from 3.11.0 to 3.19.1 Bumps [zipp](https://github.com/jaraco/zipp) from 3.11.0 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](https://github.com/jaraco/zipp/compare/v3.11.0...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 287ab58..2fdc99e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -134,7 +134,7 @@ wheel==0.38.4 # via # -r requirements-dev.in # pip-tools -zipp==3.11.0 +zipp==3.19.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From bbd7e2e4d5c17b36f460eff9ffffc37b3b8abf43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 00:32:39 +0000 Subject: [PATCH 17/28] Bump certifi from 2023.7.22 to 2024.7.4 Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.7.22 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2023.07.22...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2fdc99e..4062554 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ build==0.9.0 # via pip-tools cachetools==5.2.0 # via tox -certifi==2023.7.22 +certifi==2024.7.4 # via requests cffi==1.15.1 # via cryptography From 2be49b9682c42a7341eeb13ec89574c2bbd59eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 11:28:02 +0200 Subject: [PATCH 18/28] Add .python-version to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4c1f20f..c2e1a99 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .tox /build /dist +.python-version From 44fcd3951353638f62b00fa388da712dff8cb0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 11:28:42 +0200 Subject: [PATCH 19/28] Downgrade virtualenv for python 3.6 support. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 7b70848..a4f7171 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,8 @@ envlist = py36,py37,py38,py39,py310 coverage,flake8,isort,mypy +requires = + virtualenv<=20.21.1 [testenv] allowlist_externals = From fba0d14acaffbd3436fdf37a13bcb7500caef789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 11:28:58 +0200 Subject: [PATCH 20/28] Upgrade all dev the dependencies. --- requirements-dev.txt | 122 ++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 60 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4062554..57d6847 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,140 +1,142 @@ # -# This file is autogenerated by pip-compile with python 3.11 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile requirements-dev.in # -attrs==22.2.0 - # via pytest -bleach==5.0.1 - # via readme-renderer -build==0.9.0 +build==1.2.1 # via pip-tools -cachetools==5.2.0 +cachetools==5.3.3 # via tox certifi==2024.7.4 # via requests -cffi==1.15.1 +cffi==1.16.0 # via cryptography -chardet==5.1.0 +chardet==5.2.0 # via tox -charset-normalizer==2.1.1 +charset-normalizer==3.3.2 # via requests -click==8.1.3 +click==8.1.7 # via pip-tools colorama==0.4.6 # via tox -commonmark==0.9.1 - # via rich -coverage==7.0.1 +coverage==7.6.0 # via -r requirements-dev.in -cryptography==42.0.4 +cryptography==42.0.8 # via secretstorage -distlib==0.3.6 +distlib==0.3.8 # via virtualenv -docutils==0.19 +docutils==0.21.2 # via readme-renderer -filelock==3.8.2 +filelock==3.15.4 # via # tox # virtualenv -flake8==6.0.0 +flake8==7.1.0 # via -r requirements-dev.in idna==3.7 # via requests -importlib-metadata==5.2.0 - # via - # keyring - # twine -iniconfig==1.1.1 +importlib-metadata==8.0.0 + # via twine +iniconfig==2.0.0 # via pytest -isort==5.11.4 +isort==5.13.2 # via -r requirements-dev.in -jaraco-classes==3.2.3 +jaraco-classes==3.4.0 + # via keyring +jaraco-context==5.3.0 + # via keyring +jaraco-functools==4.0.1 # via keyring jeepney==0.8.0 # via # keyring # secretstorage -keyring==23.13.1 +keyring==25.2.1 # via twine +markdown-it-py==3.0.0 + # via rich mccabe==0.7.0 # via flake8 -more-itertools==9.0.0 - # via jaraco-classes -mypy==0.991 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==10.3.0 + # via + # jaraco-classes + # jaraco-functools +mypy==1.10.1 # via -r requirements-dev.in -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via mypy -packaging==22.0 +nh3==0.2.18 + # via readme-renderer +packaging==24.1 # via # build # pyproject-api # pytest # tox -pep517==0.13.0 - # via build -pip-tools==6.12.1 +pip-tools==7.4.1 # via -r requirements-dev.in -pkginfo==1.9.2 +pkginfo==1.10.0 # via twine -platformdirs==2.6.0 +platformdirs==4.2.2 # via # tox # virtualenv -pluggy==1.0.0 +pluggy==1.5.0 # via # pytest # tox -pycodestyle==2.10.0 +pycodestyle==2.12.0 # via flake8 -pycparser==2.21 +pycparser==2.22 # via cffi -pyflakes==3.0.1 +pyflakes==3.2.0 # via flake8 -pygments==2.15.0 +pygments==2.18.0 # via # readme-renderer # rich -pyproject-api==1.2.1 +pyproject-api==1.7.1 # via tox -pytest==7.2.0 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pytest==8.2.2 # via -r requirements-dev.in -readme-renderer==37.3 +readme-renderer==44.0 # via twine -requests==2.32.0 +requests==2.32.3 # via # requests-toolbelt # twine -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==12.6.0 +rich==13.7.1 # via twine secretstorage==3.3.3 # via keyring -six==1.16.0 - # via bleach -tox==4.0.17 +tox==4.16.0 # via -r requirements-dev.in -twine==4.0.2 +twine==5.1.1 # via -r requirements-dev.in -typing-extensions==4.4.0 +typing-extensions==4.12.2 # via mypy -urllib3==1.26.19 +urllib3==2.2.2 # via # requests # twine -virtualenv==20.17.1 +virtualenv==20.26.3 # via tox -webencodings==0.5.1 - # via bleach -wheel==0.38.4 +wheel==0.43.0 # via # -r requirements-dev.in # pip-tools -zipp==3.19.1 +zipp==3.19.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From afcbb2f5bd93b420f8da238cd9087f76a70fc582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 11:37:13 +0200 Subject: [PATCH 21/28] Test in 3.11 and 3.12 too. --- .github/workflows/index.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml index b39fdf9..e85eaab 100644 --- a/.github/workflows/index.yml +++ b/.github/workflows/index.yml @@ -7,13 +7,14 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip From 5c1e9f8f8dc20f066f8b1aada0d60c6b4703f77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 11:44:54 +0200 Subject: [PATCH 22/28] Upgrade to latest versions of github actions. --- .github/workflows/index.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml index e85eaab..532138e 100644 --- a/.github/workflows/index.yml +++ b/.github/workflows/index.yml @@ -9,16 +9,15 @@ jobs: matrix: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip - pip install tox tox-gh-actions + python -m pip install tox tox-gh-actions - name: Run tests - run: | - tox + run: tox \ No newline at end of file From c09fbb3d66ff5a6038e7ce89bb492b45da1fa667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 12:56:28 +0200 Subject: [PATCH 23/28] Convert to pyproject.toml instead of setup.py. --- pyproject.toml | 63 ++++++++++++++++++++++++++++++++++++++++++++ requirements-dev.in | 1 + requirements-dev.txt | 4 +++ setup.cfg | 34 ------------------------ setup.py | 37 -------------------------- tox.ini | 4 ++- 6 files changed, 71 insertions(+), 72 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..df41dda --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "conllu" +version = "4.5.3" +description = "CoNLL-U Parser parses a CoNLL-U formatted string into a nested python dictionary" +readme = "README.md" +requires-python = ">=3.6" +authors = [{name = "Emil Stenström", email = "emil@emilstenstrom.se"}] +license = {file = "LICENSE"} +keywords = ["conllu", "conll", "conll-u", "parser", "nlp"] +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://github.com/EmilStenstrom/conllu/" + +[project.optional-dependencies] +test = ["tox"] + +[tool.setuptools.packages.find] +where = ["conllu"] +include = ["conllu"] + +[tool.setuptools.package-data] +"conllu" = ["py.typed"] + +[tool.flake8] +ignore = "E302, W503" +max-line-length = 119 + +[tool.coverage.report] +fail_under = 100 +exclude_lines = [ + "pragma: no cover", + "def __repr__", + 'if T\.TYPE_CHECKING', + 'TT\.Protocol', +] + +[tool.coverage.run] +branch = true +source = ["conllu"] +omit = [ + "conllu/__init__.py", +] + +[tool.isort] +line_length = 119 +multi_line_output = 5 +include_trailing_comma = true +known_first_party = ["conllu", "tests"] \ No newline at end of file diff --git a/requirements-dev.in b/requirements-dev.in index d448eeb..70015e7 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,5 +1,6 @@ coverage flake8 +flake8-pyproject isort mypy pip-tools diff --git a/requirements-dev.txt b/requirements-dev.txt index 57d6847..f79fbab 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -33,6 +33,10 @@ filelock==3.15.4 # tox # virtualenv flake8==7.1.0 + # via + # -r requirements-dev.in + # flake8-pyproject +flake8-pyproject==1.2.3 # via -r requirements-dev.in idna==3.7 # via requests diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 0b0418e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[flake8] -ignore = E302,W503 -max-line-length = 119 - -[nosetests] -logging-level = ERROR - -[bdist_wheel] -universal = 1 - -[metadata] -license_files = LICENSE - -[coverage:report] -fail_under = 100 -exclude_lines = - pragma: no cover - def __repr__ - if T\.TYPE_CHECKING - TT\.Protocol - -[coverage:run] -branch = True -source = conllu -omit = - conllu/__init__.py - -[isort] -line_length = 119 -multi_line_output = 5 -include_trailing_comma = True -known_first_party = - conllu - tests diff --git a/setup.py b/setup.py deleted file mode 100644 index e8b2a73..0000000 --- a/setup.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -from setuptools import setup # type: ignore - -VERSION = '4.5.3' - -with open(os.path.join(os.path.dirname(__file__), 'README.md')) as f: - description = f.read() - -setup( - name='conllu', - packages=["conllu"], - python_requires=">=3.6", - package_data={ - "": ["py.typed"] - }, - version=VERSION, - license='MIT License', - description='CoNLL-U Parser parses a CoNLL-U formatted string into a nested python dictionary', - long_description=description, - long_description_content_type="text/markdown", - author=u'Emil Stenström', - author_email="emil@emilstenstrom.se", - url='https://github.com/EmilStenstrom/conllu/', - keywords=['conllu', 'conll', 'conll-u', 'parser', 'nlp'], - classifiers=[ - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Operating System :: OS Independent", - ], -) diff --git a/tox.ini b/tox.ini index a4f7171..149d672 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,9 @@ commands = [testenv:flake8] # Note: Settings for flake8 exists in the setup.cfg file changedir = {toxinidir} -deps = flake8 +deps = + flake8 + flake8-pyproject commands = flake8 conllu tests {[cleanup]commands} From 1225e0ddb053a0cdf2a06619157b2d7f5ad32431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 13:02:32 +0200 Subject: [PATCH 24/28] Seems I need isolated build for pyproject. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 149d672..53b3d0c 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ envlist = coverage,flake8,isort,mypy requires = virtualenv<=20.21.1 +isolated_build = True [testenv] allowlist_externals = From a4995ec5401e9b3e31cf61a257b3b1add7366a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 13:06:28 +0200 Subject: [PATCH 25/28] Add py310 and fail on no env. --- tox.ini | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tox.ini b/tox.ini index 53b3d0c..51c0124 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,16 @@ envlist = coverage,flake8,isort,mypy requires = virtualenv<=20.21.1 + +[gh-actions] +# Note: coverage runs the tests, so no need for py on the last row +python = + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39 + 3.10: py310, coverage, flake8, isort, mypy +fail_on_no_env = True isolated_build = True [testenv] @@ -60,12 +70,3 @@ deps = mypy commands = mypy . {[cleanup]commands} - -[gh-actions] -# Note: coverage runs the tests, so no need for py on the last row -python = - 3.6: py36 - 3.7: py37 - 3.8: py38 - 3.9: py39 - 3.10: coverage, flake8, isort, mypy From bac045f09ce45829d4e6ec6c6e6aca5057f8dae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 13:07:20 +0200 Subject: [PATCH 26/28] Don't move isolated build. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 51c0124..65ab242 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ envlist = coverage,flake8,isort,mypy requires = virtualenv<=20.21.1 +isolated_build = True [gh-actions] # Note: coverage runs the tests, so no need for py on the last row @@ -19,7 +20,6 @@ python = 3.9: py39 3.10: py310, coverage, flake8, isort, mypy fail_on_no_env = True -isolated_build = True [testenv] allowlist_externals = From 1493d6c0de4f703589e7b29b09f51bbe93caaa7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 13:56:19 +0200 Subject: [PATCH 27/28] Put envs on separate lines and match GHA versions. --- .github/workflows/index.yml | 2 +- tox.ini | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml index 532138e..d3fc2ac 100644 --- a/.github/workflows/index.yml +++ b/.github/workflows/index.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/tox.ini b/tox.ini index 65ab242..cb0952b 100644 --- a/tox.ini +++ b/tox.ini @@ -5,8 +5,15 @@ [tox] envlist = - py36,py37,py38,py39,py310 - coverage,flake8,isort,mypy + py36 + py37 + py38 + py39 + py310 + coverage + flake8 + isort + mypy requires = virtualenv<=20.21.1 isolated_build = True @@ -18,7 +25,7 @@ python = 3.7: py37 3.8: py38 3.9: py39 - 3.10: py310, coverage, flake8, isort, mypy + 3.10: coverage, flake8, isort, mypy fail_on_no_env = True [testenv] From 73ae932a7a2f983b0f9bee0875d0f4397f77a72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Stenstr=C3=B6m?= Date: Fri, 12 Jul 2024 14:39:31 +0200 Subject: [PATCH 28/28] Replace custom test runner with pytest. --- pyproject.toml | 4 ++ runtests.py | 101 -------------------------------------- tests/helpers.py | 16 ------ tests/test_integration.py | 12 +++-- tox.ini | 39 ++++++--------- 5 files changed, 25 insertions(+), 147 deletions(-) delete mode 100644 runtests.py diff --git a/pyproject.toml b/pyproject.toml index df41dda..432b4b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,10 @@ include = ["conllu"] [tool.setuptools.package-data] "conllu" = ["py.typed"] +[tool.pytest.ini_options] +markers = [ + "integration: marks tests as integration tests (deselect with '-m \"not integration\"')", +] [tool.flake8] ignore = "E302, W503" max-line-length = 119 diff --git a/runtests.py b/runtests.py deleted file mode 100644 index f5ff5e6..0000000 --- a/runtests.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python - -# Reference: https://gist.github.com/fragmuffin/a245f59bdcd457936c3b51aa2ebb3f6c - -import functools -import re -import unittest - - -class MyTestRunner(unittest.runner.TextTestRunner): - - def __init__(self, *args, **kwargs): - """ - Append blacklist & whitelist attributes to TestRunner instance - """ - self.whitelist = set(kwargs.pop('whitelist', [])) - self.blacklist = set(kwargs.pop('blacklist', [])) - - super(MyTestRunner, self).__init__(*args, **kwargs) - - @classmethod - def test_iter(cls, suite): - """ - Iterate through test suites, and yield individual tests - """ - for test in suite: - if isinstance(test, unittest.TestSuite): - for t in cls.test_iter(test): - yield t - else: - yield test - - def run(self, testlist): - # Change given testlist into a TestSuite - suite = unittest.TestSuite() - - # Add each test in testlist, apply skip mechanism if necessary - for test in self.test_iter(testlist): - - # Determine if test should be skipped - skip = bool(self.whitelist) - test_labels = getattr(test, '_labels', set()) - if test_labels & self.whitelist: - skip = False - if test_labels & self.blacklist: - skip = True - - if skip: - # Test should be skipped. - # replace original method with function "skip" - test_method = getattr(test, test._testMethodName) - - # Create a "skip test" wrapper for the actual test method - @functools.wraps(test_method) - def skip_wrapper(*args, **kwargs): - raise unittest.SkipTest('label exclusion') - skip_wrapper.__unittest_skip__ = True - skip_wrapper.__unittest_skip_why__ = ", ".join(test_labels) - - setattr(test, test._testMethodName, skip_wrapper) - - suite.addTest(test) - - # Resume normal TextTestRunner function with the new test suite - return super(MyTestRunner, self).run(suite) - - -if __name__ == '__main__': - import argparse - import sys - - # ---- create commandline parser - parser = argparse.ArgumentParser(description='Find and run cqparts tests.') - - def label_list(value): - return re.split(r'\W+', value) - - parser.add_argument('-w', '--whitelist', dest='whitelist', - type=label_list, default=[], - help="list of labels to test (skip all others)") - parser.add_argument('-b', '--blacklist', dest='blacklist', - type=label_list, default=[], - help="list of labels to skip") - - args = parser.parse_args() - - # ---- Discover and run tests - - # Discover Tests - loader = unittest.TestLoader() - tests = loader.discover('.', pattern='test_*.py') - - # Run tests - testRunner = MyTestRunner( - blacklist=args.blacklist, - whitelist=args.whitelist, - verbosity=2, - ) - - result = testRunner.run(tests) - sys.exit(not result.wasSuccessful()) diff --git a/tests/helpers.py b/tests/helpers.py index fdd193b..9e6b297 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,22 +2,6 @@ from io import StringIO -def testlabel(*labels): - """ - Usage:: - @testlabel('quick') - class MyTest(unittest.TestCase): - def test_foo(self): - pass - """ - def inner(cls): - # append labels to class - cls._labels = set(labels) | getattr(cls, '_labels', set()) - - return cls - - return inner - def capture_print(func, args=None): f = StringIO() with redirect_stdout(f): diff --git a/tests/test_integration.py b/tests/test_integration.py index c49b74d..40e38a5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -3,13 +3,15 @@ from io import StringIO from textwrap import dedent +import pytest + from conllu import parse, parse_incr, parse_tree, parse_tree_incr from conllu.models import Token, TokenList from conllu.parser import parse_dict_value, parse_int_value -from tests.helpers import capture_print, testlabel +from tests.helpers import capture_print -@testlabel("integration") +@pytest.mark.integration class TestParse(unittest.TestCase): maxDiff = None @@ -152,7 +154,7 @@ def test_parse_tree_incr(self): self.assertEqual(parse_tree(self.data), list(parse_tree_incr(StringIO(self.data)))) -@testlabel("integration") +@pytest.mark.integration class TestTrickyCases(unittest.TestCase): maxDiff = None @@ -175,7 +177,7 @@ def test_parse_tree_and_serialize(self): self.assertEqual(parse_tree(testcase)[0].serialize(), testcase_without_range_and_elided.serialize()) -@testlabel("integration") +@pytest.mark.integration class TestParseCoNLLUPlus(unittest.TestCase): def test_parse_conllu_plus(self): # Note: global.columns affects both sentences @@ -241,7 +243,7 @@ def test_parse_conllu_plus(self): }) -@testlabel("integration") +@pytest.mark.integration class TestParseCoNLL2009(unittest.TestCase): def test_parse_CoNLL2009_1(self): data = dedent("""\ diff --git a/tox.ini b/tox.ini index cb0952b..dec858e 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ envlist = isort mypy requires = - virtualenv<=20.21.1 + virtualenv<=20.21.1 isolated_build = True [gh-actions] @@ -29,51 +29,40 @@ python = fail_on_no_env = True [testenv] -allowlist_externals = - find -setenv = - PYTHONWARNINGS=once::DeprecationWarning -commands = - python runtests.py - -[cleanup] -commands = - find {toxinidir}/conllu -type f -name "*.pyc" -delete - find {toxinidir}/conllu -type d -name "__pycache__" -delete - find {toxinidir} -type f -path "*.egg-info*" -delete - find {toxinidir} -type d -path "*.egg-info" -delete +package = wheel +wheel_build_env = .pkg +deps = pytest +commands = python -m pytest {posargs} [testenv:flake8] -# Note: Settings for flake8 exists in the setup.cfg file +# Note: Settings for flake8 exists in the pyproject.toml file changedir = {toxinidir} deps = flake8 flake8-pyproject commands = flake8 conllu tests - {[cleanup]commands} [testenv:isort] -# Note: Settings for isort exists in the setup.cfg file +# Note: Settings for isort exists in the pyproject.toml file changedir = {toxinidir} deps = isort commands = isort --check-only --diff conllu tests - {[cleanup]commands} [testenv:coverage] -# Note: Settings for coverage exists in the setup.cfg file +# Note: Settings for coverage exists in the pyproject.toml file changedir = {toxinidir} -deps = coverage +deps = pytest-coverage commands = - coverage run --branch runtests.py --blacklist integration + coverage run --branch -m pytest -m "not integration" coverage report -m --fail-under=100 - {[cleanup]commands} [testenv:mypy] -# Note: Settings for coverage exists in the setup.cfg file +# Note: Settings for coverage exists in the pyproject.toml file changedir = {toxinidir} -deps = mypy +deps = + mypy + pytest commands = mypy . - {[cleanup]commands}