📝 Docs update

panoratech · Sep 18, 2024 · bde0e64 · bde0e64
1 parent daa937f
commit bde0e64
Show file tree

Hide file tree

Showing 11 changed files with 172 additions and 14 deletions.
diff --git a/docs/images/chroma.png b/docs/images/chroma.png
diff --git a/docs/images/cohere.png b/docs/images/cohere.png
diff --git a/docs/images/pinecone.png b/docs/images/pinecone.png
diff --git a/docs/mint.json b/docs/mint.json
@@ -670,6 +670,12 @@
         "core-concepts/sync-strategies"
       ]
     },
+    {
+      "group": "RAG",
+      "pages": [
+        "rag/index"
+      ]
+    },
     {
       "group": "Recipes",
       "pages": [

diff --git a/docs/open-source/self_hosting/envVariables.mdx b/docs/open-source/self_hosting/envVariables.mdx
@@ -23,14 +23,21 @@ description: ""
 | REDIS_HOST     | redis  | The redis host  |
 | REDIS_PASS      | A3vniod98Zbuvn9u5  | The redis password |
 | REDIS_PORT   | 6379 |The redis port| 
+| EMAIL_SENDING_ADDRESS      |    |   |
+| SMTP_HOST      |    |   |
+| SMTP_PORT      |    |   |
+| SMTP_USER      |    |   |
+| SMTP_PASSWORD      |    |   |
 
 ## Errors & Analytics (Optional in Selfhost)
 
 | Variable | Example                           | Purpose                           |
 | -------- | ------------------------------------- | ------------------------------------- |
 | SENTRY_DSN     | | Sentry Dsn |
 | SENTRY_ENABLED      |  | Flag which indicates if you want to enable Sentry  |
-
+| POSTHOG_HOST      |    |   |
+| POSTHOG_KEY      |    |   |
+| PH_TELEMETRY      |    |   |
 
 ## Database
 
@@ -172,7 +179,48 @@ description: ""
 | SHOPIFY_ECOMMERCE_CLOUD_CLIENT_ID      |    |   |
 | SHOPIFY_ECOMMERCE_CLOUD_CLIENT_SECRET      |    |   |
 | SHOPIFY_ECOMMERCE_CLOUD_SUBDOMAIN      |    |   |
-
+| SQUARESPACE_ECOMMERCE_CLOUD_CLIENT_ID      |    |   |
+| SQUARESPACE_ECOMMERCE_CLOUD_CLIENT_SECRET      |    |   |
+| EBAY_ECOMMERCE_CLOUD_CLIENT_ID      |    |   |
+| EBAY_ECOMMERCE_CLOUD_CLIENT_SECRET      |    |   |
+| EBAY_ECOMMERCE_CLOUD_RUVALUE      |    |   |
+| FAIRE_ECOMMERCE_CLOUD_CLIENT_ID      |    |   |
+| FAIRE_ECOMMERCE_CLOUD_CLIENT_SECRET      |    |   |
+| WEBFLOW_ECOMMERCE_CLOUD_CLIENT_ID      |    |   |
+| WEBFLOW_ECOMMERCE_CLOUD_CLIENT_SECRET      |    |   |
+| AMAZON_ECOMMERCE_CLOUD_CLIENT_ID      |    |   |
+| AMAZON_ECOMMERCE_CLOUD_CLIENT_SECRET      |    |   |
+| AMAZON_ECOMMERCE_CLOUD_APPLICATION_ID      |    |   |
+| MICROSOFTDYNAMICSSALES_CRM_CLOUD_CLIENT_ID      |    |   |
+| MICROSOFTDYNAMICSSALES_CRM_CLOUD_CLIENT_SECRET      |    |   |
+| SNYK_CYBERSECURITY_CLOUD_CLIENT_ID      |    |   |
+| SNYK_CYBERSECURITY_CLOUD_CLIENT_SECRET      |    |   |
+| CROWDSTRIKE_CYBERSECURITY_CLOUD_CLIENT_ID      |    |   |
+| CROWDSTRIKE_CYBERSECURITY_CLOUD_CLIENT_SECRET      |    |   |
+| SALESFORCE_CRM_CLOUD_CLIENT_ID      |    |   |
+| SALESFORCE_CRM_CLOUD_CLIENT_SECRET      |    |   |
+
+## RAG
+| Variable | Example                           | Purpose                           |
+| -------- | ------------------------------------- | ------------------------------------- |
+| OPENAI_API_KEY      |    |   |
+| JINA_API_KEY      |    |   |
+| COHERE_API_KEY      |    |   |
+| UNSTRUCTURED_API_KEY      |    |   |
+| UNSTRUCTURED_API_URL      |    |   |
+| PINECONE_API_KEY      |    |   |
+| PINECONE_INDEX_NAME      |    |   |
+| QDRANT_BASE_URL      |    |   |
+| QDRANT_API_KEY      |    |   |
+| QDRANT_COLLECTION_NAME      |    |   |
+| CHROMADB_URL      |    |   |
+| CHROMADB_COLLECTION_NAME      |    |   |
+| WEAVIATE_URL      |    |   |
+| WEAVIATE_API_KEY      |    |   |
+| WEAVIATE_CLASS_NAME      |    |   |
+| TURBOPUFFER_API_KEY      |    |   |
+| MILVUS_ADDRESS      |    |   |
+| MILVUS_COLLECTION_NAME      |    |   |
 
 # Frontend
 
@@ -197,10 +245,9 @@ You can let them empty for now !
 | -------- | ------------------------------------- | ------------------------------------- |
 | MINIO_ROOT_USER     | myaccesskey13 |  |
 | MINIO_ROOT_PASSWORD      | mysecretkey12  |  |
-| AWS_S3_REGION      | us-east-1  |  |
-| S3_TCG_ATTACHMENTS_BUCKETNAME     | tcg-attachments|   |
-| BUCKET_TCG_ATTACHMENTS_USER      | BUCKET_TCG_ATTACHMENTS_USER01 |  |
-| BUCKET_TCG_ATTACHMENTS_PW      | BUCKET_TCG_ATTACHMENTS_PW01  |  |
+| AWS_S3_REGION      |    |   |
+| AWS_ACCESS_KEY_ID      |    |   |
+| AWS_SECRET_ACCESS_KEY      |    |   |
 
 
 
diff --git a/docs/rag/index.mdx b/docs/rag/index.mdx
@@ -0,0 +1,58 @@
+---
+title: "Embeddings & Chunks"
+description: "Use the Panora API to retrieve your documents embedding sand chunks for your LLMs."
+icon: "heart"
+---
+
+Once we've synced documents across File Storage systems, we embed and chunk them so you can power your RAG applications and enable advanced retrieval search.
+
+# Step 1: Import the code snippet
+
+<CodeGroup>
+    ```shell React
+    pnpm i @panora/sdk
+    ```
+</CodeGroup>
+
+#### Use the SDK
+
+<CodeGroup>
+    ```javascript React
+    import { Panora } from "@panora/sdk";
+
+    const panora = new Panora({
+        apiKey: "<YOUR_API_KEY_HERE>",
+    });
+
+    async function run() {
+    const result = await panora.rag.query({
+      xConnectionToken: "<value>",
+      queryBody: {
+      query: "When does Panora incorporated?",
+      topK: 3,
+      },
+    });
+    
+    // Handle the result
+    console.log(result)
+    }
+
+    run();
+    ```
+</CodeGroup>
+
+Congrats ! You should be able to get back your embeddings and chunks for the query !
+
+By default, for embedding we use **OpenAI ADA-002** model and **Pinecone** managed vector database for storing the chunks.
+
+# Step 2 (Optional): Choose your own Vector DB + Embedding Model
+ 
+In Configuration page, choose the RAG settings page and provide your own credentials for vector database and embedding model.
+
+<Frame>
+  <img src="/images/cohere.png" alt="Description of image" />
+</Frame>
+<br/>
+<Frame>
+  <img src="/images/chroma.png" alt="Description of image" />
+</Frame>
diff --git a/docs/recipes/add-custom-provider-creds.mdx b/docs/recipes/add-custom-provider-creds.mdx
@@ -1,5 +1,5 @@
 ---
-title: "Custom Credentials"
+title: "Custom OAuth"
 description: "By default, all connectors use Panora's managed developer applications. You have the option to use your own custom apps."
 icon: "gear-code"
 ---

diff --git a/docs/what-is-panora.mdx b/docs/what-is-panora.mdx
@@ -11,5 +11,5 @@ This integration process is streamlined through [Panora Magic Link](/core-concep
 Upon successful connection, Panora issues a connection token to your application, which you can use to make API requests to Panora's endpoints. This token grants you access to a wealth of standardized data for a sepcific third-party.<br/>
 
 
-What sets Panora apart is its advanced RAG (Retrieval-Augmented Generation) pipeline capabilities. As part of our service, you can leverage Panora's chunking and embedding functionalities for enhanced retrieval search. This feature allows you to efficiently process and search through large volumes of connected data, enabling more accurate and context-aware information retrieval in your applications.<br/>
+What sets Panora apart is its advanced RAG (Retrieval-Augmented Generation) pipeline capabilities. As part of our service, you can leverage Panora's [chunking and embedding functionalities for enhanced retrieval search](/rag/index). This feature allows you to efficiently process and search through large volumes of connected data, enabling more accurate and context-aware information retrieval in your applications.<br/>
 By integrating Panora, you're not just connecting data sources – you're unlocking the power of intelligent data processing and retrieval, all through a single, unified API.
diff --git a/packages/api/src/@core/rag/rag.controller.ts b/packages/api/src/@core/rag/rag.controller.ts
@@ -1,10 +1,10 @@
 import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard';
 import { ConnectionUtils } from '@@core/connections/@utils';
-import { ApiPostCustomResponse } from '@@core/utils/dtos/openapi.respone.dto';
+import { ApiPostArrayCustomResponse } from '@@core/utils/dtos/openapi.respone.dto';
 import { Body, Controller, Headers, Post, UseGuards } from '@nestjs/common';
 import { ApiBody, ApiHeader, ApiOperation } from '@nestjs/swagger';
+import { QueryBody, RagQueryOutput } from './rag.dto';
 import { RagService } from './rag.service';
-import { RagQueryOutput, QueryBody } from './rag.dto';
 
 @Controller('rag')
 export class RagController {
@@ -26,7 +26,7 @@ export class RagController {
     description: 'The connection token',
     example: 'b008e199-eda9-4629-bd41-a01b6195864a',
   })
-  @ApiPostCustomResponse(RagQueryOutput)
+  @ApiPostArrayCustomResponse(RagQueryOutput)
   @ApiBody({ type: QueryBody })
   @UseGuards(ApiKeyAuthGuard)
   async queryEmbeddings(

diff --git a/packages/api/src/main.ts b/packages/api/src/main.ts
@@ -33,6 +33,9 @@ function addSpeakeasyGroup(document: any) {
 
     if (groupName) {
       for (const method in document.paths[path]) {
+        if (groupName === 'rag.query') {
+          groupName = 'rag';
+        }
         document.paths[path][method]['x-speakeasy-group'] = groupName;
         if (
           groupName !== 'webhooks' &&

diff --git a/packages/api/swagger/swagger-spec.yaml b/packages/api/swagger/swagger-spec.yaml
@@ -49,8 +49,10 @@ paths:
           content:
             application/json:
               schema:
-                type: object
-      x-speakeasy-group: rag.query
+                type: array
+                items:
+                  $ref: '#/components/schemas/RagQueryOutput'
+      x-speakeasy-group: rag
   /filestorage/files:
     get:
       operationId: listFilestorageFile
@@ -9609,14 +9611,56 @@ components:
       in: header
       name: x-api-key
   schemas:
+    RagQueryOutput:
+      type: object
+      properties:
+        chunk:
+          type: string
+          example: |-
+
+            Date : 06/07/2023
+          nullable: false
+          description: The chunk which matches the embed query
+        metadata:
+          type: object
+          example:
+            blobType: ''
+            text: ATTESTATION
+          additionalProperties: true
+          nullable: true
+          description: The metadata tied to the chunk
+        score:
+          type: number
+          example: 0.87
+          nullable: true
+          description: The score
+        embedding:
+          example:
+            - -0.00442447886
+            - -0.00116857514
+            - 0.00869117491
+            - -0.0361584462
+            - -0.00220073434
+            - 0.00946036354
+            - -0.0101112155
+          nullable: true
+          description: The embedding of the relevant chunk
+          type: array
+          items:
+            type: number
+      required:
+        - chunk
+        - metadata
+        - score
+        - embedding
     QueryBody:
       type: object
       properties:
         query:
           type: string
           example: When does Panora incorporated?
           nullable: false
-          description: The query you want to received embeddings adn chunks for
+          description: The query you want to received embeddings and chunks for
         topK:
           type: number
           example: '3'