Merge pull request #145 from llm-tools/loaders

Confluence Loader

core/embedjs/src/cache/memory-cache.ts

@@ -20,7 +20,7 @@ export class MemoryCache implements BaseCache {
     }
 
     async hasLoader(loaderId: string): Promise<boolean> {
-        return this.loaderList.hasOwnProperty(loaderId);
+        return !!this.loaderList[loaderId];
     }
 
     async loaderCustomSet<T extends Record<string, unknown>>(loaderCombinedId: string, value: T): Promise<void> {
@@ -32,7 +32,7 @@ export class MemoryCache implements BaseCache {
     }
 
     async loaderCustomHas(loaderCombinedId: string): Promise<boolean> {
-        return this.loaderCustomValues.hasOwnProperty(loaderCombinedId);
+        return !!this.loaderCustomValues[loaderCombinedId];
     }
 
     async deleteLoader(loaderId: string): Promise<void> {

core/embedjs/src/core/rag-application-builder.ts

@@ -8,7 +8,7 @@ export class RAGApplicationBuilder {
     private loaders: BaseLoader[];
     private vectorDb: BaseDb;
     private cache: BaseCache;
-    private queryTemplate: string;
+    private systemMessage: string;
     private searchResultCount: number;
     private embeddingModel: BaseEmbeddings;
     private embeddingRelevanceCutOff: number;
@@ -19,7 +19,7 @@ export class RAGApplicationBuilder {
         this.searchResultCount = 7;
         this.model = SIMPLE_MODELS.OPENAI_GPT4_TURBO;
 
-        this.queryTemplate = `You are a helpful human like chat bot. Use relevant provided context and chat history to answer the query at the end. Answer in full.
+        this.systemMessage = `You are a helpful human like chat bot. Use relevant provided context and chat history to answer the query at the end. Answer in full.
         If you don't know the answer, just say that you don't know, don't try to make up an answer.
 
         Do not use words like context or training data when responding. You can say you do not have all the information but do not indicate that you are not a reliable source.`;
@@ -75,11 +75,8 @@ export class RAGApplicationBuilder {
         return this;
     }
 
-    setQueryTemplate(queryTemplate: string) {
-        // if (!queryTemplate.includes('{0}'))
-        //     throw new Error('queryTemplate must include a placeholder for the query using {0}');
-
-        this.queryTemplate = queryTemplate;
+    setSystemMessage(systemMessage: string) {
+        this.systemMessage = systemMessage;
         return this;
     }
 
@@ -124,8 +121,8 @@ export class RAGApplicationBuilder {
         return this.embeddingRelevanceCutOff;
     }
 
-    getQueryTemplate() {
-        return this.queryTemplate;
+    getSystemMessage() {
+        return this.systemMessage;
     }
 
     getCache() {

core/embedjs/src/core/rag-application.ts

@@ -21,7 +21,7 @@ export class RAGApplication {
     private readonly debug = createDebugMessages('embedjs:core');
     private readonly embeddingRelevanceCutOff: number;
     private readonly searchResultCount: number;
-    private readonly queryTemplate: string;
+    private readonly systemMessage: string;
     private readonly cache: BaseCache;
     private readonly vectorDb: BaseDb;
     private loaders: BaseLoader[];
@@ -34,8 +34,8 @@ export class RAGApplication {
         BaseLoader.setCache(this.cache);
         BaseModel.setCache(this.cache);
 
-        this.queryTemplate = cleanString(llmBuilder.getQueryTemplate());
-        this.debug(`Using system query template - "${this.queryTemplate}"`);
+        this.systemMessage = cleanString(llmBuilder.getSystemMessage());
+        this.debug(`Using system query template - "${this.systemMessage}"`);
 
         this.vectorDb = llmBuilder.getVectorDb();
         if (!this.vectorDb) throw new SyntaxError('VectorDb not set');
@@ -387,6 +387,6 @@ export class RAGApplication {
             `Query resulted in ${context.length} chunks after filteration; chunks from ${sources.length} unique sources.`,
         );
 
-        return this.model.query(this.queryTemplate, userQuery, context, options?.conversationId);
+        return this.model.query(this.systemMessage, userQuery, context, options?.conversationId);
     }
 }

docs/api-reference/overview.mdx

@@ -35,8 +35,8 @@ Create a EmbedJs `RAGApplication` using `RAGApplicationBuilder`. `RAGApplication
     This parameter is used to control what amounts to a relevant / contextual document when retrieving documents from the vector database.
     Documents below this cut off are not discarded. EmbedJs uses sane defaults for this parameter but you can customize.
 </ParamField>
-<ParamField path="setQueryTemplate" type="string">
-    This allows you to customize the query template used when querying the LLM.
+<ParamField path="setSystemMessage" type="string">
+    This allows you to customize the system message used when querying the LLM. The system message is included once with every call to the LLM besides the user query and chat history.
 </ParamField>
 
 ## Usage

examples/confluence/project.json

@@ -28,10 +28,7 @@
                 "development": {},
                 "production": {
                     "esbuildOptions": {
-                        "sourcemap": false,
-                        "outExtension": {
-                            ".js": ".js"
-                        }
+                        "sourcemap": false
                     }
                 }
             }

loaders/embedjs-loader-confluence/src/confluence-loader.ts

@@ -50,9 +50,14 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
 
     override async *getUnfilteredChunks() {
         for (const spaceKey of this.spaceNames) {
+            let count = 0;
+
             for await (const result of this.processSpace(spaceKey)) {
                 yield result;
+                count++;
             }
+
+            this.debug(`Space '${spaceKey}' had ${count} new pages`);
         }
     }
 
@@ -63,7 +68,7 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
             this.debug(`Confluence space '${spaceKey}' has '${spaceContent['page'].results.length}' root pages`);
 
             for (const { id } of spaceContent['page'].results) {
-                for await (const result of this.processPage(id, spaceKey)) {
+                for await (const result of this.processPage(id)) {
                     yield result;
                 }
             }
@@ -73,25 +78,24 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
         }
     }
 
-    private async *processPage(pageId: string, spaceKey: string) {
+    private async *processPage(pageId: string) {
         let confluenceVersion = 0;
         try {
-            const spaceProperties = await this.confluence.spaceProperties.getSpaceProperty({
-                spaceKey,
-                key: pageId,
+            const spaceProperties = await this.confluence.content.getContentById({
+                id: pageId,
                 expand: ['version'],
             });
 
             if (!spaceProperties.version.number) throw new Error('Version number not found in space properties...');
             confluenceVersion = spaceProperties.version.number;
         } catch (e) {
-            this.debug('Could not get space properties. Page will be SKIPPED...', pageId, e);
+            this.debug('Could not get page properties. Page will be SKIPPED!', pageId, e.response);
             return;
         }
 
         let doProcess = false;
-        if (!this.checkInCache(pageId)) {
-            this.debug(`Processing '${pageId}' in space '${spaceKey}' for the FIRST time...`);
+        if (!(await this.checkInCache(pageId))) {
+            this.debug(`Processing '${pageId}' for the FIRST time...`);
             doProcess = true;
         } else {
             const cacheVersion = (await this.getFromCache(pageId)).version;
@@ -107,7 +111,7 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
         }
 
         if (!doProcess) {
-            this.debug(`Skipping page '${pageId}' in space '${spaceKey}'`);
+            this.debug(`Skipping page '${pageId}'`);
             return;
         }
 
@@ -118,7 +122,7 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
             });
 
             if (!content.body.view.value) {
-                this.debug(`Page '${pageId}' in space '${spaceKey}' has empty content. Skipping...`);
+                this.debug(`Page '${pageId}' has empty content. Skipping...`);
                 return;
             }
 
@@ -130,7 +134,7 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
 
             if (content.children) {
                 for (const { id } of content.children.page.results) {
-                    for await (const result of this.processPage(id, spaceKey)) {
+                    for await (const result of this.processPage(id)) {
                         yield result;
                     }
                 }