diff --git a/libs/langchain-community/src/chat_models/ibm.ts b/libs/langchain-community/src/chat_models/ibm.ts index d0fae3ac15ce..dd468909a886 100644 --- a/libs/langchain-community/src/chat_models/ibm.ts +++ b/libs/langchain-community/src/chat_models/ibm.ts @@ -1,3 +1,5 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + import { AIMessage, AIMessageChunk, diff --git a/libs/langchain-community/src/chat_models/tests/ibm.test.ts b/libs/langchain-community/src/chat_models/tests/ibm.test.ts index 3fea7de8504b..8e04c1c26c6b 100644 --- a/libs/langchain-community/src/chat_models/tests/ibm.test.ts +++ b/libs/langchain-community/src/chat_models/tests/ibm.test.ts @@ -1,4 +1,5 @@ /* eslint-disable no-process-env */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import WatsonxAiMlVml_v1 from "@ibm-cloud/watsonx-ai/dist/watsonx-ai-ml/vml_v1.js"; import { ChatWatsonx, ChatWatsonxInput, WatsonxCallParams } from "../ibm.js"; import { authenticateAndSetInstance } from "../../utils/ibm.js"; diff --git a/libs/langchain-community/src/embeddings/tests/ibm.test.ts b/libs/langchain-community/src/embeddings/tests/ibm.test.ts index affa8491807f..05f033f6f1af 100644 --- a/libs/langchain-community/src/embeddings/tests/ibm.test.ts +++ b/libs/langchain-community/src/embeddings/tests/ibm.test.ts @@ -1,4 +1,5 @@ /* eslint-disable no-process-env */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { testProperties } from "../../llms/tests/ibm.test.js"; import { WatsonxEmbeddings } from "../ibm.js"; diff --git a/libs/langchain-community/src/llms/tests/ibm.test.ts b/libs/langchain-community/src/llms/tests/ibm.test.ts index e0d6f3e4b521..7dfaecd6361c 100644 --- a/libs/langchain-community/src/llms/tests/ibm.test.ts +++ b/libs/langchain-community/src/llms/tests/ibm.test.ts @@ -1,4 +1,5 @@ /* eslint-disable no-process-env */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import WatsonxAiMlVml_v1 from "@ibm-cloud/watsonx-ai/dist/watsonx-ai-ml/vml_v1.js"; import { WatsonxLLM, WatsonxInputLLM } from "../ibm.js"; import { authenticateAndSetInstance } from "../../utils/ibm.js"; diff --git a/libs/langchain-community/src/utils/ibm.ts b/libs/langchain-community/src/utils/ibm.ts index acbb86f1a304..ccbe1204ef60 100644 --- a/libs/langchain-community/src/utils/ibm.ts +++ b/libs/langchain-community/src/utils/ibm.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { WatsonXAI } from "@ibm-cloud/watsonx-ai"; import { IamAuthenticator, diff --git a/libs/langchain-community/src/vectorstores/libsql.ts b/libs/langchain-community/src/vectorstores/libsql.ts index 05c77da7489c..dfdaeaca167b 100644 --- a/libs/langchain-community/src/vectorstores/libsql.ts +++ b/libs/langchain-community/src/vectorstores/libsql.ts @@ -1,7 +1,7 @@ -import type { Client } from "@libsql/client"; -import { VectorStore } from "@langchain/core/vectorstores"; -import type { EmbeddingsInterface } from "@langchain/core/embeddings"; import { Document } from "@langchain/core/documents"; +import type { EmbeddingsInterface } from "@langchain/core/embeddings"; +import { VectorStore } from "@langchain/core/vectorstores"; +import type { Client, InStatement } from "@libsql/client"; /** * Interface for LibSQLVectorStore configuration options. @@ -82,23 +82,17 @@ export class LibSQLVectorStore extends VectorStore { for (let i = 0; i < rows.length; i += batchSize) { const chunk = rows.slice(i, i + batchSize); - const insertQueries = chunk.map((row) => ({ - sql: `INSERT INTO ${this.table} (content, metadata, ${this.column}) VALUES (?, ?, ?) RETURNING id`, - args: [row.content, row.metadata, row.embedding], + + const insertQueries: InStatement[] = chunk.map((row) => ({ + sql: `INSERT INTO ${this.table} (content, metadata, ${this.column}) VALUES (:content, :metadata, vector(:embedding)) RETURNING ${this.table}.rowid AS id`, + args: row, })); const results = await this.db.batch(insertQueries); - for (const result of results) { - if ( - result && - result.rows && - result.rows.length > 0 && - result.rows[0].id != null - ) { - ids.push(result.rows[0].id.toString()); - } - } + ids.push( + ...results.flatMap((result) => result.rows.map((row) => String(row.id))) + ); } return ids; @@ -123,11 +117,12 @@ export class LibSQLVectorStore extends VectorStore { const queryVector = `[${query.join(",")}]`; - const sql = ` - SELECT ${this.table}.id, ${this.table}.content, ${this.table}.metadata, vector_distance_cos(${this.table}.${this.column}, vector('${queryVector}')) AS distance - FROM vector_top_k('idx_${this.table}_${this.column}', vector('${queryVector}'), ${k}) AS top_k - JOIN ${this.table} ON top_k.rowid = ${this.table}.id - `; + const sql: InStatement = { + sql: `SELECT ${this.table}.rowid as id, ${this.table}.content, ${this.table}.metadata, vector_distance_cos(${this.table}.${this.column}, vector(:queryVector)) AS distance + FROM vector_top_k('idx_${this.table}_${this.column}', vector(:queryVector), CAST(:k AS INTEGER)) as top_k + JOIN ${this.table} ON top_k.rowid = ${this.table}.rowid`, + args: { queryVector, k }, + }; const results = await this.db.execute(sql); @@ -136,7 +131,7 @@ export class LibSQLVectorStore extends VectorStore { const metadata = JSON.parse(row.metadata); const doc = new Document({ - id: row.id, + id: String(row.id), metadata, pageContent: row.content, }); @@ -145,6 +140,32 @@ export class LibSQLVectorStore extends VectorStore { }); } + /** + * Deletes vectors from the store. + * @param {Object} params - Delete parameters. + * @param {string[] | number[]} [params.ids] - The ids of the vectors to delete. + * @returns {Promise} + */ + async delete(params: { + ids?: string[] | number[]; + deleteAll?: boolean; + }): Promise { + if (params.deleteAll) { + await this.db.execute(`DELETE FROM ${this.table}`); + } else if (params.ids !== undefined) { + await this.db.batch( + params.ids.map((id) => ({ + sql: `DELETE FROM ${this.table} WHERE rowid = :id`, + args: { id }, + })) + ); + } else { + throw new Error( + `You must provide an "ids" parameter or a "deleteAll" parameter.` + ); + } + } + /** * Creates a new LibSQLVectorStore instance from texts. * @param {string[]} texts - The texts to add to the store. diff --git a/libs/langchain-community/src/vectorstores/tests/libsql.int.test.ts b/libs/langchain-community/src/vectorstores/tests/libsql.int.test.ts index 63fe6cbe2df7..5dbec055afff 100644 --- a/libs/langchain-community/src/vectorstores/tests/libsql.int.test.ts +++ b/libs/langchain-community/src/vectorstores/tests/libsql.int.test.ts @@ -1,13 +1,14 @@ /* eslint-disable no-process-env */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { expect, test } from "@jest/globals"; -import { OpenAIEmbeddings } from "@langchain/openai"; import { Document } from "@langchain/core/documents"; +import { OpenAIEmbeddings } from "@langchain/openai"; import { createClient } from "@libsql/client"; +import { SyntheticEmbeddings } from "@langchain/core/utils/testing"; +import fs from "node:fs"; +import { LibSQLVectorStore, LibSQLVectorStoreArgs } from "../libsql.js"; -import { LibSQLVectorStore } from "../libsql.js"; - -test("can create and query", async () => { +test("can create and query (cloud)", async () => { const client = createClient({ url: process.env.LIBSQL_URL!, authToken: process.env.LIBSQL_AUTH_TOKEN, @@ -43,3 +44,290 @@ test("can create and query", async () => { const results = await vectorStore.similaritySearchWithScore("added first", 4); expect(results.length).toBe(4); }); + +describe("LibSQLVectorStore (local)", () => { + const client = createClient({ + url: "file:store.db", + }); + + const config: LibSQLVectorStoreArgs = { + db: client, + }; + + const embeddings = new SyntheticEmbeddings({ + vectorSize: 1024, + }); + + afterAll(async () => { + await client.close(); + if (fs.existsSync("store.db")) { + fs.unlinkSync("store.db"); + } + }); + + test("a document with content can be added", async () => { + await client.batch([ + `DROP TABLE IF EXISTS vectors;`, + `CREATE TABLE IF NOT EXISTS vectors ( + content TEXT, + metadata JSON, + embedding F32_BLOB(1024) + );`, + `CREATE INDEX IF NOT EXISTS idx_vectors_embedding + ON vectors (libsql_vector_idx(embedding));`, + ]); + + const store = new LibSQLVectorStore(embeddings, config); + + const ids = await store.addDocuments([ + { + pageContent: "hello", + metadata: { a: 1 }, + }, + ]); + + expect(ids).toHaveLength(1); + + const [id] = ids; + + expect(typeof id).toBe("string"); + + const resultSet = await client.execute(`SELECT * FROM vectors`); + + expect(resultSet.rows).toHaveLength(1); + + const [row] = resultSet.rows; + + expect(row.content).toBe("hello"); + expect(JSON.parse(row.metadata as string)).toEqual({ a: 1 }); + }); + + test("a document with spaces in the content can be added", async () => { + await client.batch([ + `DROP TABLE IF EXISTS vectors;`, + `CREATE TABLE IF NOT EXISTS vectors ( + content TEXT, + metadata JSON, + embedding F32_BLOB(1024) + );`, + `CREATE INDEX IF NOT EXISTS idx_vectors_embedding + ON vectors (libsql_vector_idx(embedding));`, + ]); + + const store = new LibSQLVectorStore(embeddings, config); + + const ids = await store.addDocuments([ + { + pageContent: "hello world", + metadata: { a: 1 }, + }, + ]); + + expect(ids).toHaveLength(1); + + const [id] = ids; + + expect(typeof id).toBe("string"); + + const resultSet = await client.execute(`SELECT * FROM vectors`); + + expect(resultSet.rows).toHaveLength(1); + + const [row] = resultSet.rows; + + expect(row.content).toBe("hello world"); + expect(JSON.parse(row.metadata as string)).toEqual({ a: 1 }); + }); + + test("a similarity search can be performed", async () => { + await client.batch([ + `DROP TABLE IF EXISTS vectors;`, + `CREATE TABLE IF NOT EXISTS vectors ( + content TEXT, + metadata JSON, + embedding F32_BLOB(1024) + );`, + `CREATE INDEX IF NOT EXISTS idx_vectors_embedding + ON vectors (libsql_vector_idx(embedding));`, + ]); + + const store = new LibSQLVectorStore(embeddings, config); + + const ids = await store.addDocuments([ + { + pageContent: "the quick brown fox", + metadata: { a: 1 }, + }, + { + pageContent: "jumped over the lazy dog", + metadata: { a: 2 }, + }, + { + pageContent: "hello world", + metadata: { a: 3 }, + }, + ]); + + expect(ids).toHaveLength(3); + expect(ids.every((id) => typeof id === "string")).toBe(true); + + const results1 = await store.similaritySearch("the quick brown dog", 2); + + expect(results1).toHaveLength(2); + expect( + results1.map((result) => result.id).every((id) => typeof id === "string") + ).toBe(true); + + const results2 = await store.similaritySearch("hello"); + + expect(results2).toHaveLength(3); + expect( + results2.map((result) => result.id).every((id) => typeof id === "string") + ).toBe(true); + }); + + test("a document can be deleted by id", async () => { + await client.batch([ + `DROP TABLE IF EXISTS vectors;`, + `CREATE TABLE IF NOT EXISTS vectors ( + content TEXT, + metadata JSON, + embedding F32_BLOB(1024) + );`, + `CREATE INDEX IF NOT EXISTS idx_vectors_embedding + ON vectors (libsql_vector_idx(embedding));`, + ]); + + const store = new LibSQLVectorStore(embeddings, config); + + const ids = await store.addDocuments([ + { + pageContent: "the quick brown fox", + metadata: { a: 1 }, + }, + { + pageContent: "jumped over the lazy dog", + metadata: { a: 2 }, + }, + { + pageContent: "hello world", + metadata: { a: 3 }, + }, + ]); + + expect(ids).toHaveLength(3); + expect(ids.every((id) => typeof id === "string")).toBe(true); + + const [id1, id2] = ids; + + await store.delete({ ids: [id1, id2] }); + + const resultSet = await client.execute(`SELECT * FROM vectors`); + + expect(resultSet.rows).toHaveLength(1); + + const [row] = resultSet.rows; + + expect(row.content).toBe("hello world"); + expect(JSON.parse(row.metadata as string)).toEqual({ a: 3 }); + }); + + test("all documents can be deleted", async () => { + await client.batch([ + `DROP TABLE IF EXISTS vectors;`, + `CREATE TABLE IF NOT EXISTS vectors ( + content TEXT, + metadata JSON, + embedding F32_BLOB(1024) + );`, + `CREATE INDEX IF NOT EXISTS idx_vectors_embedding + ON vectors (libsql_vector_idx(embedding));`, + ]); + + const store = new LibSQLVectorStore(embeddings, config); + + const ids = await store.addDocuments([ + { + pageContent: "the quick brown fox", + metadata: { a: 1 }, + }, + { + pageContent: "jumped over the lazy dog", + metadata: { a: 2 }, + }, + { + pageContent: "hello world", + metadata: { a: 3 }, + }, + ]); + + expect(ids).toHaveLength(3); + expect(ids.every((id) => typeof id === "string")).toBe(true); + + await store.delete({ + deleteAll: true, + }); + + const resultSet = await client.execute(`SELECT * FROM vectors`); + + expect(resultSet.rows).toHaveLength(0); + }); + + test("the table can have a custom id column name", async () => { + await client.batch([ + `DROP TABLE IF EXISTS vectors;`, + `CREATE TABLE IF NOT EXISTS vectors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content TEXT, + metadata JSON, + embedding F32_BLOB(1024) + );`, + `CREATE INDEX IF NOT EXISTS idx_vectors_embedding + ON vectors (libsql_vector_idx(embedding));`, + ]); + + const store = new LibSQLVectorStore(embeddings, config); + + const ids = await store.addDocuments([ + { + pageContent: "the quick brown fox", + metadata: { a: 1 }, + }, + { + pageContent: "jumped over the lazy dog", + metadata: { a: 2 }, + }, + { + pageContent: "hello world", + metadata: { a: 3 }, + }, + ]); + + expect(ids).toHaveLength(3); + expect(ids).toEqual(["1", "2", "3"]); + + const results = await store.similaritySearch("the quick brown dog", 2); + + expect(results).toHaveLength(2); + expect(results.map((result) => result.pageContent)).toEqual([ + "the quick brown fox", + "jumped over the lazy dog", + ]); + expect( + results.map((result) => result.id).every((id) => typeof id === "string") + ).toBe(true); + + const [id1, id2] = ids; + + await store.delete({ ids: [id1, id2] }); + + const resultSet = await client.execute(`SELECT * FROM vectors`); + + expect(resultSet.rows).toHaveLength(1); + + const [row] = resultSet.rows; + + expect(row.content).toBe("hello world"); + expect(JSON.parse(row.metadata as string)).toEqual({ a: 3 }); + }); +});