From a77f6bc55ca28882f281c4a6e5cab10541ae6e8d Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Fri, 9 Aug 2024 19:47:27 -0400 Subject: [PATCH 1/3] ci: enable test and lint on all branches --- .github/workflows/ci.yaml | 3 +-- .github/workflows/lint.yml | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d0befeee0..7b1f19d67e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,8 +2,7 @@ name: Test on: pull_request: push: - branches: - - master + env: PY_COLORS: 1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4e2a2db260..5579f08524 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,8 +3,6 @@ run-name: Lint code on: pull_request: push: - branches: - - master env: PYTHON_VERSION: 3.8 From 94708de2afd53a66c4b85d90759ebfc2b783df1f Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Fri, 9 Aug 2024 18:36:11 -0400 Subject: [PATCH 2/3] fix: enable tracebacks for "user"/custom sqlite functions A bit niche but I tried setting my bareasc prefix to an empty string, and was getting an obtuse error. This should help make clearer what is happening when queries fail. The exception is not properly raised up the stack in the first place because it happens across 2 FFI boundaries: the DB query (Python -> SQLite), and the custom DB function (SQLite -> Python). Thus Python cannot forwarded it back to itself through SQLite, and it's treated as an "unraisable" exception. We could override `sys.unraisablehook` to not print anything for the original exception, and store it in a global for the outer Python interpreter to fetch and raise properly, but that's pretty hacky, limited to a single DB instance and query at once, and risks swallowing other "unraisable" exceptions. Instead we just tell the user to look above for what Python prints. Sample output: ``` Exception ignored in: Traceback (most recent call last): File "site-packages/unidecode/__init__.py", line 60, in unidecode_expect_ascii bytestring = string.encode('ASCII') ^^^^^^^^^^^^^ AttributeError: 'bytes' object has no attribute 'encode' Traceback (most recent call last): File "site-packages/beets/dbcore/db.py", line 988, in query cursor = self.db._connection().execute(statement, subvals) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ sqlite3.OperationalError: user-defined function raised exception During handling of the above exception, another exception occurred: Traceback (most recent call last): File "site-packages/beets/__main__.py", line 9, in sys.exit(main()) ^^^^^^ File "site-packages/beets/ui/__init__.py", line 1865, in main _raw_main(args) File "site-packages/beets/ui/__init__.py", line 1852, in _raw_main subcommand.func(lib, suboptions, subargs) File "site-packages/beets/ui/commands.py", line 1599, in list_func list_items(lib, decargs(args), opts.album) File "site-packages/beets/ui/commands.py", line 1594, in list_items for item in lib.items(query): ^^^^^^^^^^^^^^^^ File "site-packages/beets/library.py", line 1695, in items return self._fetch(Item, query, sort or self.get_default_item_sort()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "site-packages/beets/library.py", line 1673, in _fetch return super()._fetch(model_cls, query, sort) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "site-packages/beets/dbcore/db.py", line 1301, in _fetch rows = tx.query(sql, subvals) ^^^^^^^^^^^^^^^^^^^^^^ File "site-packages/beets/dbcore/db.py", line 991, in query raise DBCustomFunctionError() beets.dbcore.db.DBCustomFunctionError: beets defined SQLite function failed; see the other errors above for details ``` --- beets/dbcore/db.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 566c116314..7e8ff01c5a 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -73,6 +73,16 @@ class DBAccessError(Exception): """ +class DBCustomFunctionError(Exception): + """A sqlite function registered by beets failed.""" + + def __init__(self): + super().__init__( + "beets defined SQLite function failed; " + "see the other errors above for details" + ) + + class FormattedMapping(Mapping[str, str]): """A `dict`-like formatted view of a model. @@ -970,6 +980,12 @@ def __exit__( self._mutated = False self.db._db_lock.release() + if ( + isinstance(exc_value, sqlite3.OperationalError) + and exc_value.args[0] == "user-defined function raised exception" + ): + raise DBCustomFunctionError() + def query(self, statement: str, subvals: Sequence = ()) -> List: """Execute an SQL statement with substitution values and return a list of rows from the database. @@ -1028,6 +1044,10 @@ def __init__(self, path, timeout: float = 5.0): "sqlite3 must be compiled with multi-threading support" ) + # Print tracebacks for exceptions in user defined functions + # See also `self.add_functions` and `DBCustomFunctionError`. + sqlite3.enable_callback_tracebacks(True) + self.path = path self.timeout = timeout From 9a70a2159543f1b2389e1d89b4dce41683ab5189 Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Fri, 9 Aug 2024 19:33:43 -0400 Subject: [PATCH 3/3] feat: mark SQLite custom functions as deterministic to allow caching --- beets/dbcore/db.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 7e8ff01c5a..92d73aa8fc 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -20,11 +20,12 @@ import os import re import sqlite3 +import sys import threading import time from abc import ABC from collections import defaultdict -from sqlite3 import Connection +from sqlite3 import Connection, sqlite_version_info from types import TracebackType from typing import ( Any, @@ -1143,9 +1144,14 @@ def bytelower(bytestring: Optional[AnyStr]) -> Optional[AnyStr]: return bytestring - conn.create_function("regexp", 2, regexp) - conn.create_function("unidecode", 1, unidecode) - conn.create_function("bytelower", 1, bytelower) + deterministic = {} + if sys.version_info >= (3, 8) and sqlite_version_info >= (3, 8, 3): + # Let sqlite make extra optimizations + deterministic["deterministic"] = True + + conn.create_function("regexp", 2, regexp, **deterministic) + conn.create_function("unidecode", 1, unidecode, **deterministic) + conn.create_function("bytelower", 1, bytelower, **deterministic) def _close(self): """Close the all connections to the underlying SQLite database