Merge pull request #152 from llm-tools/loaders

Docs fixes
llm-tools · Nov 1, 2024 · 959e184 · 959e184
2 parents 2de6d3b + f1e4952
commit 959e184
Show file tree

Hide file tree

Showing 90 changed files with 3,650 additions and 3,603 deletions.
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
 ## 0.1.13 (2024-10-30)
 
 Updated how BaseModel handles conversations.

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.16",

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,11 +1,17 @@
-## 0.1.13 (2024-10-30)
-
-This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
-
-## 0.1.11 (2024-10-27)
-
-This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
-
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
+## 0.1.13 (2024-10-30)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.13"
+        "@llm-tools/embedjs-interfaces": "0.1.15"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,3 +1,13 @@
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
+### 🩹 Fixes
+
+-   renamed remaining instances if vectorDb to vectorDatabase ([ca79586](https://github.com/llm-tools/embedJs/commit/ca79586))
+
 ## 0.1.13 (2024-10-30)
 
 A small update made to how conversations are handled

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.13",
-        "@llm-tools/embedjs-utils": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
+        "@llm-tools/embedjs-utils": "0.1.15",
         "debug": "^4.3.7",
         "langchain": "^0.3.5",
         "md5": "^2.3.0",

diff --git a/core/embedjs/src/core/rag-application-builder.ts b/core/embedjs/src/core/rag-application-builder.ts
@@ -48,8 +48,8 @@ export class RAGApplicationBuilder {
     }
 
     /**
-     * The function setVectorDb sets a BaseVectorDatabase object
-     * @param {BaseVectorDatabase} vectorDb - The `vectorDb` parameter is an instance of the `BaseVectorDatabase` class, which
+     * The function setVectorDatabase sets a BaseVectorDatabase object
+     * @param {BaseVectorDatabase} vectorDatabase - The `vectorDatabase` parameter is an instance of the `BaseVectorDatabase` class, which
      * is used to store vectors in a database.
      * @returns The `this` object is being returned, which allows for method chaining.
      */

diff --git a/core/embedjs/src/core/rag-application.ts b/core/embedjs/src/core/rag-application.ts
@@ -23,7 +23,7 @@ export class RAGApplication {
     private readonly embeddingRelevanceCutOff: number;
     private readonly searchResultCount: number;
     private readonly systemMessage: string;
-    private readonly vectorDb: BaseVectorDatabase;
+    private readonly vectorDatabase: BaseVectorDatabase;
     private readonly store: BaseStore;
     private loaders: BaseLoader[];
     private model: BaseModel;
@@ -39,8 +39,8 @@ export class RAGApplication {
         this.systemMessage = cleanString(llmBuilder.getSystemMessage());
         this.debug(`Using system query template - "${this.systemMessage}"`);
 
-        this.vectorDb = llmBuilder.getVectorDatabase();
-        if (!this.vectorDb) throw new SyntaxError('VectorDb not set');
+        this.vectorDatabase = llmBuilder.getVectorDatabase();
+        if (!this.vectorDatabase) throw new SyntaxError('vectorDatabase not set');
 
         this.searchResultCount = llmBuilder.getSearchResultCount();
         this.embeddingRelevanceCutOff = llmBuilder.getEmbeddingRelevanceCutOff();
@@ -68,7 +68,7 @@ export class RAGApplication {
             this.debug('Initialized LLM class');
         }
 
-        await this.vectorDb.init({ dimensions: await RAGEmbedding.getEmbedding().getDimensions() });
+        await this.vectorDatabase.init({ dimensions: await RAGEmbedding.getEmbedding().getDimensions() });
         this.debug('Initialized vector database');
 
         if (this.store) {
@@ -290,8 +290,8 @@ export class RAGApplication {
             };
         });
 
-        this.debug(`Inserting chunks for loader ${loaderUniqueId} to vectorDb`);
-        return this.vectorDb.insertChunks(embedChunks);
+        this.debug(`Inserting chunks for loader ${loaderUniqueId} to vectorDatabase`);
+        return this.vectorDatabase.insertChunks(embedChunks);
     }
 
     /**
@@ -302,7 +302,7 @@ export class RAGApplication {
      * embeddings as a number.
      */
     public async getEmbeddingsCount(): Promise<number> {
-        return this.vectorDb.getVectorCount();
+        return this.vectorDatabase.getVectorCount();
     }
 
     /**
@@ -312,7 +312,7 @@ export class RAGApplication {
      * @returns The `deleteLoader` method returns a boolean value indicating the success of the operation.
      */
     public async deleteLoader(uniqueLoaderId: string) {
-        const deleteResult = await this.vectorDb.deleteKeys(uniqueLoaderId);
+        const deleteResult = await this.vectorDatabase.deleteKeys(uniqueLoaderId);
         if (this.store && deleteResult) await this.store.deleteLoaderMetadataAndCustomValues(uniqueLoaderId);
         this.loaders = this.loaders.filter((x) => x.getUniqueId() != uniqueLoaderId);
         return deleteResult;
@@ -324,7 +324,7 @@ export class RAGApplication {
      * @returns The `reset` function returns a boolean value indicating the result.
      */
     public async reset() {
-        await this.vectorDb.reset();
+        await this.vectorDatabase.reset();
         return true;
     }
 
@@ -341,7 +341,10 @@ export class RAGApplication {
      */
     public async getEmbeddings(cleanQuery: string) {
         const queryEmbedded = await RAGEmbedding.getEmbedding().embedQuery(cleanQuery);
-        const unfilteredResultSet = await this.vectorDb.similaritySearch(queryEmbedded, this.searchResultCount + 10);
+        const unfilteredResultSet = await this.vectorDatabase.similaritySearch(
+            queryEmbedded,
+            this.searchResultCount + 10,
+        );
         this.debug(`Query resulted in ${unfilteredResultSet.length} chunks before filteration...`);
 
         return unfilteredResultSet

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,11 +1,17 @@
-## 0.1.13 (2024-10-30)
-
-This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
-
-## 0.1.11 (2024-10-27)
-
-This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
-
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
+## 0.1.13 (2024-10-30)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,11 +1,17 @@
-## 0.1.13 (2024-10-30)
-
-This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
-
-## 0.1.11 (2024-10-27)
-
-This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
-
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
+## 0.1.13 (2024-10-30)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,11 +1,17 @@
-## 0.1.13 (2024-10-30)
-
-This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
-
-## 0.1.11 (2024-10-27)
-
-This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
-
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
+## 0.1.13 (2024-10-30)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,11 +1,17 @@
-## 0.1.13 (2024-10-30)
-
-This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
-
-## 0.1.11 (2024-10-27)
-
-This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
-
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
+## 0.1.13 (2024-10-30)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.12.0",
-        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
 ## 0.1.13 (2024-10-30)
 
 A small update made to how conversations are handled

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
         "lmdb": "^3.1.4"
     },
     "type": "module",

diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.1.15 and 0.1.14 (2024-11-01)
+
+### 🚀 Features
+
+-   refactored caches into stores ([5aec5ed](https://github.com/llm-tools/embedJs/commit/5aec5ed))
+
 ## 0.1.13 (2024-10-30)
 
 A small update made to how conversations are handled

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.13",
+    "version": "0.1.15",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "@llm-tools/embedjs-interfaces": "0.1.15",
         "debug": "^4.3.7",
         "mongodb": "^6.10.0"
     },