Add compatible utils functions

README.md

@@ -386,23 +386,30 @@ Now our backend is able to crawl a given URL, embed the content and index the em
 
 ### Get matches from embeddings
 
-To retrieve the most relevant documents from the index, we'll use the `query` function in the Pinecone SDK. This function takes a vector and returns the most similar vectors from the index. We'll use this function to retrieve the most relevant documents from the index, given some embeddings.
+To retrieve the most relevant documents from the index, we'll use the `query` function in the Pinecone SDK. This function takes a record and returns the most similar records from the index. We'll use this function to retrieve the most relevant documents from the index, given some embeddings.
 
 ```ts
 const getMatchesFromEmbeddings = async (
-  embeddings: number[],
-  topK: number,
+  embeddings: number[], 
+  topK: number, 
   namespace: string
 ): Promise<ScoredPineconeRecord[]> => {
   // Obtain a client for Pinecone
-  const pinecone = await getPineconeClient();
+  const pinecone = new Pinecone();
 
   // Retrieve the list of indexes
-  const indexes = await pinecone.listIndexes();
+  const indexes = await pinecone.listIndexes()
+
+  let exists = false
+  for (const index of indexes) {
+    if (index.name === process.env.PINECONE_INDEX!) {
+      exists = true
+    }
+  }
 
   // Check if the desired index is present, else throw an error
-  if (!indexes.includes(process.env.PINECONE_INDEX!)) {
-    throw new Error(`Index ${process.env.PINECONE_INDEX} does not exist`);
+  if (!exists) {
+    throw (new Error(`Index ${process.env.PINECONE_INDEX} does not exist`))
   }
 
   // Get the Pinecone index
@@ -420,8 +427,8 @@ const getMatchesFromEmbeddings = async (
     return queryResult.matches || [];
   } catch (e) {
     // Log the error and throw it
-    console.log("Error querying embeddings: ", e);
-    throw new Error(`Error querying embeddings: ${e}`);
+    console.log("Error querying embeddings: ", e)
+    throw (new Error(`Error querying embeddings: ${e}`,))
   }
 };
 ```

src/app/api/crawl/seed.ts

@@ -1,12 +1,10 @@
 import { getEmbeddings } from "@/utils/embeddings";
 import { Document, MarkdownTextSplitter, RecursiveCharacterTextSplitter } from "@pinecone-database/doc-splitter";
-import { utils as PineconeUtils, Vector } from "@pinecone-database/pinecone";
+import { Pinecone, PineconeRecord } from "@pinecone-database/pinecone";
 import md5 from "md5";
-import { getPineconeClient } from "@/utils/pinecone";
 import { Crawler, Page } from "./crawler";
 import { truncateStringByBytes } from "@/utils/truncateString"
-
-const { chunkedUpsert, createIndexIfNotExists } = PineconeUtils
+import { chunkedUpsert, createIndexIfNotExists } from "@/utils/pinecone";
 
 interface SeedOptions {
   splittingMethod: string
@@ -20,7 +18,7 @@ type DocumentSplitter = RecursiveCharacterTextSplitter | MarkdownTextSplitter
 async function seed(url: string, limit: number, indexName: string, options: SeedOptions) {
   try {
     // Initialize the Pinecone client
-    const pinecone = await getPineconeClient();
+    const pinecone = new Pinecone()
 
     // Destructure the options object
     const { splittingMethod, chunkSize, chunkOverlap } = options;
@@ -42,10 +40,10 @@ async function seed(url: string, limit: number, indexName: string, options: Seed
     await createIndexIfNotExists(pinecone!, indexName, 1536);
     const index = pinecone && pinecone.Index(indexName);
 
-    // Get the vector embeddings for the documents
+    // Get the record embeddings for the documents
     const vectors = await Promise.all(documents.flat().map(embedDocument));
 
-    // Upsert vectors into the Pinecone index
+    // Upsert records into the Pinecone index
     await chunkedUpsert(index!, vectors, '', 10);
 
     // Return the first document
@@ -56,7 +54,7 @@ async function seed(url: string, limit: number, indexName: string, options: Seed
   }
 }
 
-async function embedDocument(doc: Document): Promise<Vector> {
+async function embedDocument(doc: Document): Promise<PineconeRecord> {
   try {
     // Generate OpenAI embeddings for the document content
     const embedding = await getEmbeddings(doc.pageContent);
@@ -74,7 +72,7 @@ async function embedDocument(doc: Document): Promise<Vector> {
         url: doc.metadata.url as string, // The URL where the document was found
         hash: doc.metadata.hash as string // The hash of the document content
       }
-    } as Vector;
+    } as PineconeRecord;
   } catch (error) {
     console.log("Error embedding document: ", error)
     throw error

src/app/utils/pinecone.ts

@@ -1,4 +1,68 @@
-import { Pinecone, ScoredPineconeRecord } from "@pinecone-database/pinecone";
+import { Index, Pinecone, PineconeRecord, RecordMetadata, ScoredPineconeRecord } from "@pinecone-database/pinecone";
+
+const indexExists = async (
+  client: Pinecone,
+  indexName: string
+) => {
+  // Retrieve the list of indexes
+  const indexes = await client.listIndexes()
+
+  // Check if the desired index is present
+  return indexes.some(index => { return index.name === indexName })
+}
+
+const createIndexIfNotExists = async (
+  client: Pinecone,
+  indexName: string,
+  dimension: number
+) => {
+  try {
+    if (!indexExists(client, indexName)) {
+      console.log('Creating index and Waiting until is ready...', indexName);
+      await client.createIndex({
+        name: indexName,
+        dimension,
+        waitUntilReady: true
+      });
+      console.log('Index is ready.');
+    }
+  } catch (e) {
+    console.error('Error creating index', e);
+  }
+};
+
+const sliceIntoChunks = <T>(arr: T[], chunkSize: number) => {
+  return Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) =>
+    arr.slice(i * chunkSize, (i + 1) * chunkSize)
+  );
+};
+
+const chunkedUpsert = async (
+  index: Index<RecordMetadata>,
+  vectors: PineconeRecord[],
+  namespace: string,
+  chunkSize = 10
+) => {
+  // Split the records into chunks
+  const chunks = sliceIntoChunks<PineconeRecord>(vectors, chunkSize);
+
+  try {
+    // Upsert each chunk of records into the index
+    await Promise.allSettled(
+      chunks.map(async (chunk) => {
+        try {
+          await index.namespace(namespace).upsert(chunk);
+        } catch (e) {
+          console.log('Error upserting chunk', e);
+        }
+      })
+    );
+
+    return true;
+  } catch (e) {
+    throw new Error(`Error upserting vectors into index: ${e}`);
+  }
+};
 
 // The function `getMatchesFromEmbeddings` is used to retrieve matches for the given embeddings
 const getMatchesFromEmbeddings = async (embeddings: number[], topK: number, namespace: string): Promise<ScoredPineconeRecord[]> => {
@@ -8,15 +72,8 @@ const getMatchesFromEmbeddings = async (embeddings: number[], topK: number, name
   // Retrieve the list of indexes
   const indexes = await pinecone.listIndexes()
 
-  let exists = false
-  for (const index of indexes) {
-    if (index.name === process.env.PINECONE_INDEX!) {
-      exists = true
-    }
-  }
-
   // Check if the desired index is present, else throw an error
-  if (!exists) {
+  if (!indexExists(pinecone, process.env.PINECONE_INDEX!)) {
     throw (new Error(`Index ${process.env.PINECONE_INDEX} does not exist`))
   }
 
@@ -41,4 +98,8 @@ const getMatchesFromEmbeddings = async (embeddings: number[], topK: number, name
   }
 }
 
-export { getMatchesFromEmbeddings }
+export { 
+  createIndexIfNotExists,
+  chunkedUpsert,
+  getMatchesFromEmbeddings 
+}