From f88692144b142c6e8af61039ce6a9dba21a6dd3c Mon Sep 17 00:00:00 2001 From: Alexandru Fikl Date: Sat, 16 Nov 2024 15:12:00 +0200 Subject: [PATCH 1/5] pyproject: bump min papis version --- pyproject.toml | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0b933a4..e583051 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,20 @@ [build-system] build-backend = "hatchling.build" -requires = ["hatchling>=1.10.0"] +requires = [ "hatchling>=1.10" ] [project] name = "papis-zotero" -description = "Interact with Zotero using papis" version = "0.1.2" -authors = [{ name = "Alejandro Gallo", email = "aamsgallo@gmail.com" }] -maintainers = [{ name = "Alejandro Gallo", email = "aamsgallo@gmail.com" }] +description = "Interact with Zotero using papis" readme = "README.rst" -license = { text = "GPLv3" } +keywords = [ "bibtex", "biliography", "cli", "management", "papis", "zotero" ] +license = { text = "GPL-3.0-or-later" } +maintainers = [ { name = "Alejandro Gallo", email = "aamsgallo@gmail.com" } ] +authors = [ { name = "Alejandro Gallo", email = "aamsgallo@gmail.com" } ] +requires-python = ">=3.8" classifiers = [ - "Environment :: Console :: Curses", "Environment :: Console", + "Environment :: Console :: Curses", "Intended Audience :: Developers", "Intended Audience :: Education", "Intended Audience :: Science/Research", @@ -29,35 +31,34 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Utilities", ] -keywords = ["bibtex", "biliography", "cli", "management", "papis", "zotero"] -dependencies = ["papis==0.13"] - -[project.urls] -Repository = "https://github.com/papis/papis-zotero" +dependencies = [ "papis>=0.14,<0.15" ] [project.optional-dependencies] -"develop" = [ +develop = [ "flake8", "flake8-bugbear", + "flake8-pyproject", "flake8-quotes", - "Flake8-pyproject", "mypy>=0.7", "pep8-naming", "pytest", "pytest-cov", "python-coveralls", - "types-PyYAML", - "types-tqdm", + "types-pyyaml", ] +[project.urls] +Repository = "https://github.com/papis/papis-zotero" + [project.entry-points."papis.command"] -"zotero" = "papis_zotero:main" +zotero = "papis_zotero:main" [tool.flake8] -select = ["B", "D", "E", "F", "N", "Q", "W"] -extend-ignore = ["B019", "E123", "N818", "W503"] +select = [ "B", "D", "E", "F", "N", "Q", "W" ] +extend-ignore = [ "B019", "E123", "N818", "W503" ] max-line-length = 88 inline-quotes = "double" multiline-quotes = "double" @@ -65,7 +66,8 @@ multiline-quotes = "double" [tool.pytest.ini_options] norecursedirs = ".git doc build dist" python_files = "*.py" -markers = ["config_setup: setup for tmp_config", +markers = [ + "config_setup: setup for tmp_config", "library_setup: setup for tmp_library", ] From a2073798ecf608cfeb8854e8857f5c0f3f561a8a Mon Sep 17 00:00:00 2001 From: Alexandru Fikl Date: Sat, 16 Nov 2024 15:23:44 +0200 Subject: [PATCH 2/5] test: port to papis.testing --- Makefile | 4 +- pyproject.toml | 6 +- tests/conftest.py | 31 --- tests/resources/bibtex/zotero-library.bib | 2 +- tests/resources/bibtex_out.yaml | 2 +- tests/test_bibtex.py | 3 +- tests/test_sql.py | 3 +- tests/testlib.py | 261 ---------------------- 8 files changed, 11 insertions(+), 301 deletions(-) delete mode 100644 tests/conftest.py delete mode 100644 tests/testlib.py diff --git a/Makefile b/Makefile index 1c24248..1133b5e 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,10 @@ mypy: ## Run (strict) mypy checks .PHONY: mypy pytest: ## Run pytest test and doctests - python -m pytest -rswx --cov=papis_zotero -v -s tests + python -m pytest -rswx -v -s tests .PHONY: pytest ci-install: ## Run pip and install dependencies on CI - python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip hatchling wheel python -m pip install -e '.[develop]' .PHONY: ci-install diff --git a/pyproject.toml b/pyproject.toml index e583051..6de3057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,10 @@ inline-quotes = "double" multiline-quotes = "double" [tool.pytest.ini_options] +addopts = [ + "--doctest-modules", + "--cov=papis_zotero", +] norecursedirs = ".git doc build dist" python_files = "*.py" markers = [ @@ -76,4 +80,4 @@ strict = true show_column_numbers = true hide_error_codes = false pretty = true -warn_unused_ignores = false +warn_unused_ignores = true diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 3e01605..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -## -# This is copied from `papis/tests/conftest.py` and should be kept in sync as -# much as possible until we clean it up and can depend on newer versions of papis -## - -import pytest -from typing import Iterator - -from tests import testlib - - -@pytest.fixture() -def tmp_config( - request: pytest.FixtureRequest - ) -> Iterator[testlib.TemporaryConfiguration]: - marker = request.node.get_closest_marker("config_setup") - kwargs = marker.kwargs if marker else {} - - with testlib.TemporaryConfiguration(**kwargs) as config: - yield config - - -@pytest.fixture() -def tmp_library( - request: pytest.FixtureRequest - ) -> Iterator[testlib.TemporaryLibrary]: - marker = request.node.get_closest_marker("library_setup") - kwargs = marker.kwargs if marker else {} - - with testlib.TemporaryLibrary(**kwargs) as lib: - yield lib diff --git a/tests/resources/bibtex/zotero-library.bib b/tests/resources/bibtex/zotero-library.bib index 039d01a..bbd8d1f 100644 --- a/tests/resources/bibtex/zotero-library.bib +++ b/tests/resources/bibtex/zotero-library.bib @@ -12,7 +12,7 @@ @article{schaeffer_efficient_2013 author = {Schaeffer, Nathanaël}, urldate = {2023-02-26}, date = {2013-03}, - langid = {english}, + langid = {english} file = {Full Text:files/8/Schaeffer - 2013 - Efficient spherical harmonic transforms aimed at p.pdf:application/pdf}, } diff --git a/tests/resources/bibtex_out.yaml b/tests/resources/bibtex_out.yaml index d070d01..e95a810 100644 --- a/tests/resources/bibtex_out.yaml +++ b/tests/resources/bibtex_out.yaml @@ -12,7 +12,7 @@ journaltitle: Journal of Computational Physics langid: english month: 7 pages: 17--38 -ref: ReviewOfSummaSvard2014 +ref: svard_review_2014 tags: sbp title: Review of summation-by-parts schemes for initial–boundary-value problems type: article diff --git a/tests/test_bibtex.py b/tests/test_bibtex.py index ce8e104..48a9631 100644 --- a/tests/test_bibtex.py +++ b/tests/test_bibtex.py @@ -7,8 +7,7 @@ import papis.database import papis.document import papis_zotero.bibtex - -from .testlib import TemporaryLibrary +from papis.testing import TemporaryLibrary @pytest.mark.skipif(os.name == "nt", reason="encoding is incorrect on windows") diff --git a/tests/test_sql.py b/tests/test_sql.py index 4447266..303deba 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -6,8 +6,7 @@ import papis.yaml import papis.document import papis_zotero.sql - -from .testlib import TemporaryLibrary +from papis.testing import TemporaryLibrary @pytest.mark.skipif(os.name == "nt", reason="encoding is incorrect on windows") diff --git a/tests/testlib.py b/tests/testlib.py deleted file mode 100644 index 081112c..0000000 --- a/tests/testlib.py +++ /dev/null @@ -1,261 +0,0 @@ -## -# This is copied from `papis/tests/testlib.py` and should be kept in sync as -# much as possible until we clean it up and can depend on newer versions of papis -## - -import os -import random -import tempfile -from typing import Any, Dict, Sequence, Optional - -import pytest -import click -import click.testing - -import papis.logging - -papis.logging.setup() -random.seed(42) - -PAPIS_TEST_DOCUMENTS = [ - { - "author": "doc without files", - "title": "Title of doc without files", - "year": "1093", - "_test_files": 0, - }, - { - "author": "J. Krishnamurti", - "title": "Freedom from the known", - "year": "2009", - "_test_files": 1, - }, { - "author": "K. Popper", - "doi": "10.1021/ct5004252", - "title": "The open society", - "volume": "I", - "_test_files": 0, - }, { - "author": "Turing A. M.", - "doi": "10.1112/plms/s2-42.1.230", - "issue": "1", - "journal": "Proceedings of the London Mathematical Society", - "note": "First turing machine paper foundation of cs", - "pages": "230--265", - "title": "On Computable Numbers with an Application to the Entscheidungsproblem", # noqa: E501 - "url": "https://api.wiley.com/onlinelibrary/tdm/v1/articles/10.1112%2Fplms%2Fs2-42.1.230", # noqa: E501 - "volume": "s2-42", - "year": "1937", - "_test_files": 2, - }, - { - "author": "test_author", - "title": "Test Document 1 (wRkdff)", - "year": "2019", - "_test_files": 1 - }, - { - "author": "test_author", - "title": "Test Document 2 (ZD9QRz)", - "year": "2019", - "_test_files": 1 - }, -] - - -def create_random_file(filetype: Optional[str] = None, - prefix: Optional[str] = None, - suffix: Optional[str] = None, - dir: Optional[str] = None) -> str: - if filetype is None: - filetype = random.choice(["pdf", "epub", "djvu"]) - - # NOTE: these are chosen to match using 'filetype.guess' and are not valid - # files otherwise - if filetype == "pdf": - buf = b"%PDF-1.5%\n" - suffix = ".pdf" if suffix is None else suffix - elif filetype == "epub": - buf = bytes( - [0x50, 0x4B, 0x3, 0x4] - + [0x00 for i in range(26)] - + [0x6D, 0x69, 0x6D, 0x65, 0x74, 0x79, 0x70, 0x65, 0x61, 0x70, - 0x70, 0x6C, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x2F, - 0x65, 0x70, 0x75, 0x62, 0x2B, 0x7A, 0x69, 0x70] - + [0x00 for i in range(1)] - ) - suffix = ".epub" if suffix is None else suffix - elif filetype == "djvu": - buf = bytes( - [0x41, 0x54, 0x26, 0x54, 0x46, 0x4F, 0x52, 0x4D] - + [0x00, 0x00, 0x00, 0x00] - + [0x44, 0x4A, 0x56, 0x4D] - ) - suffix = ".djvu" if suffix is None else suffix - elif filetype == "text": - buf = b"papis-test-file-contents" - else: - raise ValueError("Unknown file type: '{}'".format(filetype)) - - with tempfile.NamedTemporaryFile( - dir=dir, suffix=suffix, prefix=prefix, - delete=False) as fd: - fd.write(buf) - - return fd.name - - -def populate_library(libdir: str) -> None: - import papis.id - from papis.document import Document - - for i, data in enumerate(PAPIS_TEST_DOCUMENTS): - doc_data = data.copy() - - folder_path = os.path.join(libdir, "test_doc_{}".format(i)) - os.makedirs(folder_path) - - # add files - num_test_files = doc_data.pop("_test_files") - if num_test_files: - doc_data["files"] = [ - os.path.basename(create_random_file(dir=folder_path)) - for _ in range(int(str(num_test_files))) - ] - - # create document - doc = Document(folder_path, doc_data) - doc[papis.id.key_name()] = papis.id.compute_an_id(doc) - doc.save() - - -class TemporaryConfiguration: - libname = "test" - - def __init__(self, - settings: Optional[Dict[str, Any]] = None, - overwrite: bool = False) -> None: - self.settings = settings - self.overwrite = overwrite - - self.libdir = "" - self.configdir = "" - self.configfile = "" - - self._tmpdir: Optional[tempfile.TemporaryDirectory[str]] = None - self._monkeypatch: Optional[pytest.MonkeyPatch] = None - - @property - def tmpdir(self) -> str: - assert self._tmpdir is not None - return self._tmpdir.name - - def create_random_file(self, - filetype: Optional[str] = None, - prefix: Optional[str] = None, - suffix: Optional[str] = None) -> str: - return create_random_file( - filetype, suffix=suffix, prefix=prefix, - dir=self.tmpdir) - - def __enter__(self) -> "TemporaryConfiguration": - if self._tmpdir is not None: - raise ValueError("{!r} cannot be nested".format(type(self).__name__)) - - # create directories and files - self._tmpdir = tempfile.TemporaryDirectory(prefix="papis-test-") - - self.libdir = os.path.join(self.tmpdir, "lib") - self.configdir = os.path.join(self.tmpdir, "papis") - self.configfile = os.path.join(self.configdir, "config") - - os.makedirs(self.libdir) - os.makedirs(self.configdir) - os.makedirs(os.path.join(self.configdir, "scripts")) - - # load settings - settings = { - self.libname: {"dir": self.libdir}, - "settings": {"default-library": self.libname} - } - - if self.settings is not None: - if self.overwrite: - settings = self.settings - else: - settings["settings"].update(self.settings) - - import configparser - with open(self.configfile, "w") as fd: - config = configparser.ConfigParser() - config.read_dict(settings) - config.write(fd) - - # monkeypatch environment - self._monkeypatch = pytest.MonkeyPatch() - self._monkeypatch.setenv("XDG_CONFIG_HOME", self.tmpdir) - self._monkeypatch.setenv("XDG_CACHE_HOME", self.tmpdir) - - # reload configuration - import papis.config - papis.config.set_config_file(self.configfile) - papis.config.reset_configuration() - - assert papis.config.get_config_folder() == self.configdir - assert papis.config.get_config_file() == self.configfile - - return self - - def __exit__(self, exc_type, exc_value, exc_traceback) -> None: # type: ignore - # cleanup - if self._monkeypatch: - self._monkeypatch.undo() - if self._tmpdir: - self._tmpdir.cleanup() - - # reset variables - self._tmpdir = None - self.libdir = "" - self.configdir = "" - self.configfile = "" - - -class TemporaryLibrary(TemporaryConfiguration): - def __init__(self, - settings: Optional[Dict[str, Any]] = None, - populate: bool = True) -> None: - super().__init__(settings=settings) - self.populate = populate - - def __enter__(self) -> "TemporaryLibrary": - super().__enter__() - - # initialize library - import papis.library - lib = papis.library.Library(self.libname, [self.libdir]) - - import papis.config - papis.config.set("default-library", self.libname) - papis.config.set_lib(lib) - - # populate library - if self.populate: - populate_library(self.libdir) - - return self - - -class PapisRunner(click.testing.CliRunner): - def __init__(self, **kwargs: Any) -> None: - if "mix_stderr" not in kwargs: - kwargs["mix_stderr"] = False - - super().__init__(**kwargs) - - def invoke(self, # type: ignore[override] - cli: click.Command, - args: Sequence[str], **kwargs: Any) -> click.testing.Result: - if "catch_exceptions" not in kwargs: - kwargs["catch_exceptions"] = False - - return super().invoke(cli, args, **kwargs) From b14bf45cf54549ed809107982d967008c16b46dd Mon Sep 17 00:00:00 2001 From: Alexandru Fikl Date: Sat, 16 Nov 2024 15:44:19 +0200 Subject: [PATCH 3/5] bibtex: format tags as a list --- papis_zotero/bibtex.py | 38 ++++++++++++++++----------------- papis_zotero/server.py | 2 +- papis_zotero/sql.py | 8 ++----- tests/resources/bibtex_out.yaml | 3 ++- 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/papis_zotero/bibtex.py b/papis_zotero/bibtex.py index ff8b465..b587da3 100644 --- a/papis_zotero/bibtex.py +++ b/papis_zotero/bibtex.py @@ -1,5 +1,6 @@ import os -from typing import Optional +import re +from typing import Any, Dict, Optional import papis.bibtex import papis.commands.add @@ -8,6 +9,8 @@ logger = papis.logging.get_logger(__name__) +RE_SEPARATOR = re.compile(r"\s*,\s*") + def add_from_bibtex(bib_file: str, out_folder: Optional[str] = None, @@ -18,28 +21,25 @@ def add_from_bibtex(bib_file: str, entries = papis.bibtex.bibtex_to_dict(bib_file) nentries = len(entries) for i, entry in enumerate(entries): - # cleanup tags - if "date" in entry: - date = str(entry.pop("date")).split("-") - entry["year"] = int(date[0]) # type: ignore[assignment] - entry["month"] = int(date[1]) # type: ignore[assignment] + result: Dict[str, Any] = entry.copy() - if "keywords" in entry: - entry["tags"] = entry.pop("keywords") + # cleanup date + if "date" in result: + date = str(result.pop("date")).split("-") + result["year"] = int(date[0]) + result["month"] = int(date[1]) - if "author" not in entry: - if "tags" in entry: - entry["tags"] = "{} unknown-author".format(entry["tags"]) - else: - entry["tags"] = "unknown-author" + # cleanup tags + if "keywords" in result: + result["tags"] = RE_SEPARATOR.split(result.pop("keywords")) - if "ref" in entry: - entry["ref"] = papis.bibtex.ref_cleanup(entry["ref"]) + if "ref" in result: + result["ref"] = papis.bibtex.ref_cleanup(result["ref"]) else: - entry["ref"] = papis.bibtex.create_reference(entry) + result["ref"] = papis.bibtex.create_reference(result) # get file - pdf_file = entry.pop("file", None) + pdf_file = result.pop("file", None) if pdf_file is not None: pdf_file = pdf_file.split(":")[1] pdf_file = os.path.join(*pdf_file.split("/")) @@ -53,8 +53,8 @@ def add_from_bibtex(bib_file: str, # add to library logger.info("[%4d/%-4d] Exporting item with ref '%s'.", - i, nentries, entry["ref"]) + i, nentries, result["ref"]) papis.commands.add.run([pdf_file] if pdf_file is not None else [], - data=entry, + data=result, link=link) diff --git a/papis_zotero/server.py b/papis_zotero/server.py index 5c391e5..fb36a0c 100644 --- a/papis_zotero/server.py +++ b/papis_zotero/server.py @@ -33,7 +33,7 @@ }]), _k("tags", [{ "key": "tags", - "action": lambda t: "; ".join(tag["tag"] for tag in t) + "action": lambda t: [tag["tag"] for tag in t] }]), _k("date", [ {"key": "year", "action": lambda d: int(d.split("-")[0])}, diff --git a/papis_zotero/sql.py b/papis_zotero/sql.py index 0c9c427..58e25f3 100644 --- a/papis_zotero/sql.py +++ b/papis_zotero/sql.py @@ -15,10 +15,6 @@ logger = papis.logging.get_logger(__name__) -# separator between multiple tags -# FIXME: this should be handled by papis -ZOTERO_TAG_DELIMITER = " " - # fuzzy date matching ISO_DATE_RE = re.compile(r"(?P\d{4})-?(?P\d{2})?-?(?P\d{2})?") @@ -162,11 +158,11 @@ def get_files(connection: sqlite3.Connection, item_id: str, item_key: str, """ -def get_tags(connection: sqlite3.Connection, item_id: str) -> Dict[str, str]: +def get_tags(connection: sqlite3.Connection, item_id: str) -> Dict[str, List[str]]: cursor = connection.cursor() cursor.execute(ZOTERO_QUERY_ITEM_TAGS, (item_id,)) - tags = ZOTERO_TAG_DELIMITER.join(str(row[0]) for row in cursor) + tags = [str(row[0]) for row in cursor] return {"tags": tags} if tags else {} diff --git a/tests/resources/bibtex_out.yaml b/tests/resources/bibtex_out.yaml index e95a810..cfb7531 100644 --- a/tests/resources/bibtex_out.yaml +++ b/tests/resources/bibtex_out.yaml @@ -13,7 +13,8 @@ langid: english month: 7 pages: 17--38 ref: svard_review_2014 -tags: sbp +tags: +- sbp title: Review of summation-by-parts schemes for initial–boundary-value problems type: article url: https://linkinghub.elsevier.com/retrieve/pii/S002199911400151X From a2376bae4f10333a9871cd22e970ae9c8962a028 Mon Sep 17 00:00:00 2001 From: Alexandru Fikl Date: Sat, 16 Nov 2024 15:44:35 +0200 Subject: [PATCH 4/5] fix: update mypy ignores --- papis_zotero/__init__.py | 6 +++--- papis_zotero/sql.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/papis_zotero/__init__.py b/papis_zotero/__init__.py index 1791840..ab66ab7 100644 --- a/papis_zotero/__init__.py +++ b/papis_zotero/__init__.py @@ -12,13 +12,13 @@ logger = papis.logging.get_logger(__name__) -@click.group("zotero") # type: ignore[arg-type] +@click.group("zotero") @click.help_option("-h", "--help") def main() -> None: """Zotero interface for papis.""" -@main.command("serve") # type: ignore[arg-type] +@main.command("serve") @click.help_option("-h", "--help") @click.option("--port", help="Port to listen to", @@ -56,7 +56,7 @@ def serve(address: str, port: int, set_list: List[Tuple[str, str]],) -> None: httpd.serve_forever() -@main.command("import") # type: ignore[arg-type] +@main.command("import") @click.help_option("-h", "--help") @click.option( "-f", diff --git a/papis_zotero/sql.py b/papis_zotero/sql.py index 58e25f3..a99fcef 100644 --- a/papis_zotero/sql.py +++ b/papis_zotero/sql.py @@ -217,7 +217,8 @@ def get_collections(connection: sqlite3.Connection, """.format(",".join(["?"] * len(papis_zotero.utils.ZOTERO_EXCLUDED_ITEM_TYPES))) -def add_from_sql(input_path: str, out_folder: Optional[str] = None, +def add_from_sql(input_path: str, + out_folder: Optional[str] = None, link: bool = False) -> None: """ :param inpath: path to zotero SQLite database "zoter.sqlite" and @@ -255,6 +256,7 @@ def add_from_sql(input_path: str, out_folder: Optional[str] = None, if out_folder is not None: papis.config.set_lib_from_name(out_folder) + folder_name = papis.config.getstring("add-folder-name") for i, (item_id, item_type, item_key, date_added) in enumerate(cursor, start=1): # convert fields date_added = ( @@ -280,7 +282,7 @@ def add_from_sql(input_path: str, out_folder: Optional[str] = None, i, items_count, item_key, out_folder) papis.commands.add.run(paths=files, data=item, link=link, - folder_name=papis.config.getstring("add-folder-name") + folder_name=folder_name ) logger.info("Finished exporting from '%s'.", input_path) From f4eacc79a08075dd47156d2d52578e8711305cd9 Mon Sep 17 00:00:00 2001 From: Alexandru Fikl Date: Sat, 16 Nov 2024 15:47:23 +0200 Subject: [PATCH 5/5] ci: add Python 3.13 to matrix --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ced8fe..01b4dc9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] fail-fast: False steps: