diff --git a/docs/logging.md b/docs/logging.md index d91c0ad4..e2f85e6e 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -96,7 +96,7 @@ def cleanup_sql(sql): return first_line + '(\n ' + ',\n '.join(columns) + '\n);' cog.out("```sql\n") -for table in ("conversations", "responses"): +for table in ("conversations", "responses", "responses_fts"): schema = db[table].schema cog.out(format(cleanup_sql(schema))) cog.out("\n") @@ -121,5 +121,11 @@ CREATE TABLE [responses] ( [duration_ms] INTEGER, [datetime_utc] TEXT ); +CREATE VIRTUAL TABLE [responses_fts] USING FTS5 ( + [prompt], + [response], + content=[responses] +); ``` +`responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table. \ No newline at end of file diff --git a/llm/migrations.py b/llm/migrations.py index d0c422eb..378d3eca 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -195,3 +195,8 @@ def m010_create_new_log_tables(db): pk="id", foreign_keys=(("conversation_id", "conversations", "id"),), ) + + +@migration +def m011_fts_for_responses(db): + db["responses"].enable_fts(["prompt", "response"], create_triggers=True) diff --git a/tests/test_migrate.py b/tests/test_migrate.py index 5fe3c3cb..e066e264 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -21,7 +21,9 @@ def test_migrate_blank(): db = sqlite_utils.Database(memory=True) migrate(db) - assert set(db.table_names()) == {"_llm_migrations", "conversations", "responses"} + assert set(db.table_names()).issuperset( + {"_llm_migrations", "conversations", "responses", "responses_fts"} + ) assert db["responses"].columns_dict == EXPECTED foreign_keys = db["responses"].foreign_keys @@ -65,7 +67,7 @@ def test_migrate_from_original_schema(has_record): } ) migrate(db) - expected_tables = {"_llm_migrations", "conversations", "responses"} + expected_tables = {"_llm_migrations", "conversations", "responses", "responses_fts"} if has_record: expected_tables.add("logs") - assert set(db.table_names()) == expected_tables + assert set(db.table_names()).issuperset(expected_tables)