Skip to content

Commit

Permalink
feat(community): Add delete and allow default row id in libsql (#7053)
Browse files Browse the repository at this point in the history
Co-authored-by: jacoblee93 <jacoblee93@gmail.com>
  • Loading branch information
cdierkens and jacoblee93 authored Oct 29, 2024
1 parent 8378a57 commit e53531a
Show file tree
Hide file tree
Showing 7 changed files with 341 additions and 26 deletions.
2 changes: 2 additions & 0 deletions libs/langchain-community/src/chat_models/ibm.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
/* eslint-disable @typescript-eslint/no-explicit-any */

import {
AIMessage,
AIMessageChunk,
Expand Down
1 change: 1 addition & 0 deletions libs/langchain-community/src/chat_models/tests/ibm.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/* eslint-disable no-process-env */
/* eslint-disable @typescript-eslint/no-explicit-any */
import WatsonxAiMlVml_v1 from "@ibm-cloud/watsonx-ai/dist/watsonx-ai-ml/vml_v1.js";
import { ChatWatsonx, ChatWatsonxInput, WatsonxCallParams } from "../ibm.js";
import { authenticateAndSetInstance } from "../../utils/ibm.js";
Expand Down
1 change: 1 addition & 0 deletions libs/langchain-community/src/embeddings/tests/ibm.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/* eslint-disable no-process-env */
/* eslint-disable @typescript-eslint/no-explicit-any */
import { testProperties } from "../../llms/tests/ibm.test.js";
import { WatsonxEmbeddings } from "../ibm.js";

Expand Down
1 change: 1 addition & 0 deletions libs/langchain-community/src/llms/tests/ibm.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/* eslint-disable no-process-env */
/* eslint-disable @typescript-eslint/no-explicit-any */
import WatsonxAiMlVml_v1 from "@ibm-cloud/watsonx-ai/dist/watsonx-ai-ml/vml_v1.js";
import { WatsonxLLM, WatsonxInputLLM } from "../ibm.js";
import { authenticateAndSetInstance } from "../../utils/ibm.js";
Expand Down
1 change: 1 addition & 0 deletions libs/langchain-community/src/utils/ibm.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { WatsonXAI } from "@ibm-cloud/watsonx-ai";
import {
IamAuthenticator,
Expand Down
65 changes: 43 additions & 22 deletions libs/langchain-community/src/vectorstores/libsql.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { Client } from "@libsql/client";
import { VectorStore } from "@langchain/core/vectorstores";
import type { EmbeddingsInterface } from "@langchain/core/embeddings";
import { Document } from "@langchain/core/documents";
import type { EmbeddingsInterface } from "@langchain/core/embeddings";
import { VectorStore } from "@langchain/core/vectorstores";
import type { Client, InStatement } from "@libsql/client";

/**
* Interface for LibSQLVectorStore configuration options.
Expand Down Expand Up @@ -82,23 +82,17 @@ export class LibSQLVectorStore extends VectorStore {

for (let i = 0; i < rows.length; i += batchSize) {
const chunk = rows.slice(i, i + batchSize);
const insertQueries = chunk.map((row) => ({
sql: `INSERT INTO ${this.table} (content, metadata, ${this.column}) VALUES (?, ?, ?) RETURNING id`,
args: [row.content, row.metadata, row.embedding],

const insertQueries: InStatement[] = chunk.map((row) => ({
sql: `INSERT INTO ${this.table} (content, metadata, ${this.column}) VALUES (:content, :metadata, vector(:embedding)) RETURNING ${this.table}.rowid AS id`,
args: row,
}));

const results = await this.db.batch(insertQueries);

for (const result of results) {
if (
result &&
result.rows &&
result.rows.length > 0 &&
result.rows[0].id != null
) {
ids.push(result.rows[0].id.toString());
}
}
ids.push(
...results.flatMap((result) => result.rows.map((row) => String(row.id)))
);
}

return ids;
Expand All @@ -123,11 +117,12 @@ export class LibSQLVectorStore extends VectorStore {

const queryVector = `[${query.join(",")}]`;

const sql = `
SELECT ${this.table}.id, ${this.table}.content, ${this.table}.metadata, vector_distance_cos(${this.table}.${this.column}, vector('${queryVector}')) AS distance
FROM vector_top_k('idx_${this.table}_${this.column}', vector('${queryVector}'), ${k}) AS top_k
JOIN ${this.table} ON top_k.rowid = ${this.table}.id
`;
const sql: InStatement = {
sql: `SELECT ${this.table}.rowid as id, ${this.table}.content, ${this.table}.metadata, vector_distance_cos(${this.table}.${this.column}, vector(:queryVector)) AS distance
FROM vector_top_k('idx_${this.table}_${this.column}', vector(:queryVector), CAST(:k AS INTEGER)) as top_k
JOIN ${this.table} ON top_k.rowid = ${this.table}.rowid`,
args: { queryVector, k },
};

const results = await this.db.execute(sql);

Expand All @@ -136,7 +131,7 @@ export class LibSQLVectorStore extends VectorStore {
const metadata = JSON.parse(row.metadata);

const doc = new Document({
id: row.id,
id: String(row.id),
metadata,
pageContent: row.content,
});
Expand All @@ -145,6 +140,32 @@ export class LibSQLVectorStore extends VectorStore {
});
}

/**
* Deletes vectors from the store.
* @param {Object} params - Delete parameters.
* @param {string[] | number[]} [params.ids] - The ids of the vectors to delete.
* @returns {Promise<void>}
*/
async delete(params: {
ids?: string[] | number[];
deleteAll?: boolean;
}): Promise<void> {
if (params.deleteAll) {
await this.db.execute(`DELETE FROM ${this.table}`);
} else if (params.ids !== undefined) {
await this.db.batch(
params.ids.map((id) => ({
sql: `DELETE FROM ${this.table} WHERE rowid = :id`,
args: { id },
}))
);
} else {
throw new Error(
`You must provide an "ids" parameter or a "deleteAll" parameter.`
);
}
}

/**
* Creates a new LibSQLVectorStore instance from texts.
* @param {string[]} texts - The texts to add to the store.
Expand Down
Loading

0 comments on commit e53531a

Please sign in to comment.