How to get large json I want from my context and question

[lakshya29154]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import * as fs from "fs";
import { config } from "dotenv";
import { createHash } from "crypto";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
// import { PDFLoader } from "@langchain/community/document_loader/fs/pdf";
import { OpenAIEmbeddings } from "@langchain/openai";
import { OpenAI } from "@langchain/openai";
import { OpenAIChat } from "@langchain/openai";
import { PromptTemplate } from "@langchain/core/prompts";
import { Client } from "@opensearch-project/opensearch";
import { PDFLoader } from "langchain/document_loaders/fs/pdf";

config();


const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
const model = new OpenAI({ temperature: 0.1, ap5iKey: OPENAI_API_KEY });

const client = new Client({
  node: "http://localhost:9200",
});

client
  .info()
  .then((response) => console.log("Elasticsearch Cluster Info:", response))
  .catch((error) => console.error("Error connecting to Elasticsearch:", error));

const INDEX_NAME = "privateequityfile2";

// Function to create an index with mappings and settings
async function createIndex() {
  const indexBody = {
    settings: {
      number_of_shards: 1,
      number_of_replicas: 0,
    },
    mappings: {
      properties: {
        content: { type: "text" },
        embedding: { type: "dense_vector", dims: 1536 },
        pdfHash: { type: "keyword" },
      },
    },
  };

  try {
    const indexExists = await client.indices.exists({ index: INDEX_NAME });
    if (!indexExists.body) {
      const response = await client.indices.create({
        index: INDEX_NAME,
        body: indexBody,
      });
      console.log("Index created:", response);
    } else {
      console.log("Index already exists.");
    }
  } catch (error) {
    console.error("Error creating index:", error);
  }
}

function generateHash(text) {
  let texts = text.toString();
  return createHash("sha256").update(texts).digest("hex");
}

async function splitTextIntoChunks(text) {
  const splitter = new RecursiveCharacterTextSplitter({
    chunkSize: 1000,
    chunkOverlap: 200,
  });
  const documents = await splitter.createDocuments([text]);
  return documents.map((doc) => doc.pageContent);
}

async function generateEmbeddings(textChunks) {
  const embeddings = new OpenAIEmbeddings({
    apiKey: process.env.OPENAI_API_KEY,
  });
  const vectors = [];

  for (const chunk of textChunks) {
    console.log("Chunks",chunk)
    const vector = await embeddings.embedQuery(chunk);
    if (vector && vector.length > 0) {
      vectors.push({ chunk, vector });
    } else {
      console.error("Invalid or empty vector generated for chunk:", chunk);
    }
  }
  return vectors;
}

async function storeInOpenSearch(vectors, pdfHash) {
  for (const item of vectors) {
    if (!item.vector || item.vector.length === 0) {
      console.error("Invalid or empty vector data:", item);
      continue;
    }

    try {
      await client.index({
        index: INDEX_NAME,
        body: {
          content: item.chunk,
          embedding: item.vector,
          pdfHash: pdfHash,
        },
      });
    } catch (error) {
      console.error("Error storing data in Elasticsearch:", error);
      throw error;
    }
  }
}

async function checkProcessed(pdfHash) {
  try {
    const { body } = await client.search({
      index: INDEX_NAME,
      body: {
        size: 1,
        query: {
          match: { pdfHash: pdfHash },
        },
      },
    });

    return body.hits.total.value > 0;
  } catch (error) {
    console.error("Error checking if PDF is processed:", error);
    throw error;
  }
}

async function queryOpenSearch(queryVector, topK = 10) {
  try {
    const { body } = await client.search({
      index: INDEX_NAME,
      body: {
        size: topK,
        query: {
          script_score: {
            query: { match_all: {} },
            script: {
              source:
                "cosineSimilarity(params.query_vector, doc['embedding']) + 1.0",
              params: { query_vector: queryVector },
            },
          },
        },
      },
    });

    return body.hits.hits.map((hit) => ({
      chunk: hit._source.content,
      distance: 1 - hit._score, // Adjust score calculation if needed
    }));
  } catch (error) {
    console.error(
      "Error querying Elasticsearch:",
      error.meta.body.error.root_cause[0].reason
    );
    throw error;
  }
}

async function askOpenAI(question, context) {
  const chat = new OpenAIChat({
    apiKey: process.env.OPENAI_API_KEY,
    modelName: "gpt-4",
    temperature: 0.7,
  });

  const prompt = `${context}\n\nQuestion: ${question}`;
  try {
    const answer = await chat.call(prompt);
    return answer.trim();
  } catch (error) {
    console.error("Error querying OpenAI:", error);
    return "Error querying OpenAI";
  }
}

// temp add

const API_URL_PROMPT_TEMPLATE1 = `
You are an expert JSON formatter. I will provide you with a set of questions and corresponding data. 
Your task is to format the answers into valid JSON by following the given key-value structure. 
For each question, use the data provided to populate the appropriate values in the JSON format.

- Here are your questions: {question}
- Here is your data: {api_docs}

Please generate the output in the following JSON format:

{{
    "fund_size": "string",
    "fund_manager": "string",
    "fund_life_rate": "string",
    "management_fee": "string",
    "investment_period": "string",
    "minimum_commitment": "string",
    "maximum_commitment": "string",
    "fixed_interest_rate": "string",
    "general_partner_catch_up": "string"
}}

Make sure to:
- Accurately map the data to each corresponding key.
- Ensure the JSON structure is valid and properly formatted.
- Only use the provided data to populate the values.
`

  const API_URL_PROMPT_TEMPLATE = new PromptTemplate({
    inputVariables: ["api_docs", "question"],
    template: API_URL_PROMPT_TEMPLATE1,
  });

// end

export async function main1(filePath) {
  const specFilePath = filePath;
  const loader = new PDFLoader(specFilePath, {
    parsedItemSeparator: "",
  });
  const docs = await loader.load();
  //  console.log("This is docs",docs)
  // const jsonData = fs.readFileSync(specFilePath, "utf8");
  // const swaggerData = docs
  let hash = generateHash(docs);
  
  // Create the index if it doesn't exist
  await createIndex();

  const isProcessed = await checkProcessed(hash);
  if (isProcessed) {
    console.log(
      // "JSON has already been processed. Skipping embedding generation."
    );
  } else {
    const swaggerData1 = JSON.stringify(docs);
    const textChunks = await splitTextIntoChunks(swaggerData1);
    // console.log("Text chunks generated:", textChunks);

    const vectors = await generateEmbeddings(textChunks);
    // console.log("Embeddings generated:", vectors , typeof(vectors));
    await storeInOpenSearch(vectors,hash);
  }
  let question1 = ["What would be the minimum and maximum commitments?",
    "Who would be the fund manager?",
    "What would be the life of fund?",
    " What would be the management fees?",
    "What would be the investment period extension?",
    "What would be the minimum commitment?",
    "What would be the maximum commitment?",
    "What is the preferred fixed interest rate return?",
    "What is the General Partner CatchUp?"]

    question1 = question1.toString();
  const embeddings = new OpenAIEmbeddings({
    apiKey: process.env.OPENAI_API_KEY,
  });
  const queryVector = await embeddings.embedQuery(question1);

  const relevantChunks = await queryOpenSearch(queryVector);
  console.log("Relevant chunks:", relevantChunks);
  const context = relevantChunks.map((result) => result.chunk).join("\n\n");
  console.log("Context:", context);

  //approch 2
  const apiUrlPrompt = await API_URL_PROMPT_TEMPLATE.format({
    api_docs: context,
    question: question1,
  });
  const apiUrlResponse = await model.generate([apiUrlPrompt]);
  console.log("API URL Response:", JSON.stringify(apiUrlResponse, null, 1));
  console.log(apiUrlResponse.generations[0][0]);
  const apiUrl = apiUrlResponse.generations[0][0]?.text;
  console.log("Generated API URL:", apiUrlResponse.generations[0][0]?.text , "\n");
  // return apiUrl;
}

main1("OfferingData/private_equity_large1.pdf")  // Also

Description

Above I send a script which query based on question now I want that I will get answer in json format my json is so large
{
"companyDescription": "string",
"companyLogo": "string",
"companyLogoIcon": "string",
"companyProspectus": "string",
"customFields": [
{
"name": "string",
"value": "string"
}
],
"founded": "string",
"headquarters": "string",
"industryOrSector": "string",
"tokenPrice": "string",
"investmentType": "string",
"investorNationalityWhitelist": [
"string"
],
"issuerSignee": {
"name": "string",
"surname": "string"
},
"issuerWalletAddress": "string",
"issuerOrganizationId": "string",
"legalName": "string",
"newAccountApplication": "string",
"termsAndConditions": "string",
"newsletterOrUpdate": [
"string"
],
"name": "string",
"description": "string",
"descriptionHeader": "string",
"cusip": "string",
"details": "string",
"orderFundedMessage": "string",
"state": "Draft",
"subscriptionAgreement": "string",
"summaryCompanyDescription": "string",
"summaryImage": "string",
"team": [
{
"name": "string",
"photo": "string",
"title": "string"
}
],
"tokenAddress": "string",
"tokenDecimals": 0,
"tokenTicker": "string",
"tokenNetwork": "Mainnet",
"tokenType": "ERC3643",
"tokenizationAgent": {
"type": "None"
},
"transferAgent": {
"type": "None"
},
"website": "string",
"payments": {
"wireTransfer": {
"bankAccountNumber": "string",
"bankAddress": "string",
"bankName": "string",
"companyAddress": "string",
"companyName": "string",
"routingNumber": "string",
"swiftCode": "string"
},
"cryptoTransfer": {
"contractType": "TrueFi",
"cryptocurrency": "USDC",
"contractAddress": "string",
"contractNetwork": "Mainnet"
},
"achTransfer": true,
"noneTransfer": true
},
"primaryMarket": true,
"secondaryMarket": true,
"rfqEnabled": true,
"endDate": "2024-10-14T11:06:20.251Z",
"position": 0,
"restrictedForDirectInvestors": true,
"assetType": "None",
"privateEquityConfig": {
"fundManager": "string",
"termOfFund": 0,
"issueDate": "2024-10-14T11:06:20.251Z",
"closingDate": "2024-10-14T11:06:20.251Z",
"navLaunchPrice": 0,
"targetIRR": 0,
"managementFees": [
{
"from": 0,
"to": 0,
"percent": 0
}
],
"preferredReturn": 0,
"catchUp": 0,
"lpGpAllocation": 0,
"dividendFrequency": "string",
"fundCurrency": "string",
"targetAUM": 0,
"latestNav": 0,
"investmentSize": 0,
"sector": "string",
"generalPartnerCommitment": 0,
"fundManagerWalletAddress": "string",
"individualMinCommitment": 0,
"individualMaxCommitment": 0,
"institutionMinCommitment": 0,
"institutionMaxCommitment": 0,
"commitmentSchedule": {
"startDate": "2024-10-14T11:06:20.251Z",
"endDate": "2024-10-14T11:06:20.251Z"
},
"capitalCallsSchedule": [
{
"startDate": "2024-10-14T11:06:20.251Z",
"endDate": "2024-10-14T11:06:20.251Z"
}
],
"redemptionsSchedule": [
{
"startDate": "2024-10-14T11:06:20.251Z",
"endDate": "2024-10-14T11:06:20.251Z"
}
],
"delayForCapitalCallToRaise": 0,
"delayForCapitalCallToRaiseUnit": "Hours",
"termOfFundExtension": 0,
"paymentMethod": "StableCoins",
"paymentType": "USDC",
"enableTaApproval": true
},
"roundConfig": {
"endDate": "2024-10-14T11:06:20.251Z",
"investmentMinimum": 0,
"investmentSize": 0,
"tokenPrice": 0,
"latestNav": 0,
"fundTokenNetworks": [
"None"
],
"fundPaymentMethods": [
"wire"
],
"holdingMethod": [
"FundShare"
]
},
"brokerOrganizationId": "string",
"isMultiSig": true,
"isSubscribedForTaApproval": true,
"isDvP": true
}
Now curretkr I am sending question and based on that it get relevant embedding bit now I have these I want all so how can I get this all data from pdf

System Info

windows , node -20

[dosubot[bot]]

I found a similar discussion that might be helpful: Using langchain to get response from large json data ^[1].

To extract all data from a PDF and generate a large JSON object, you can use the PDFLoader class to load the PDF and then process the text to fit your JSON structure. Here's a general approach:

1. Load the PDF: Use PDFLoader to extract text and metadata from the PDF.

2. Split Text: Use RecursiveCharacterTextSplitter to split the text into manageable chunks.

3. Generate Embeddings: Use OpenAIEmbeddings to create embeddings for each chunk.

4. Store and Query: Store these embeddings in OpenSearch and query them to retrieve relevant chunks.

5. Format JSON: Use a PromptTemplate to format the extracted data into your desired JSON structure.

Here's a simplified example:

import { PDFLoader } from "langchain/document_loaders/fs/pdf";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { OpenAIEmbeddings } from "@langchain/openai";
import { PromptTemplate } from "@langchain/core/prompts";

// Load PDF
async function loadPdf(filePath) {
  const loader = new PDFLoader(filePath);
  const docs = await loader.load();
  return docs;
}

// Split text into chunks
async function splitText(text) {
  const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
  return await splitter.createDocuments([text]);
}

// Generate embeddings
async function generateEmbeddings(chunks) {
  const embeddings = new OpenAIEmbeddings({ apiKey: process.env.OPENAI_API_KEY });
  return await Promise.all(chunks.map(chunk => embeddings.embedQuery(chunk.pageContent)));
}

// Format JSON
async function formatJson(context, questions) {
  const template = new PromptTemplate({
    inputVariables: ["api_docs", "question"],
    template: `Your JSON template here...`
  });
  return await template.format({ api_docs: context, question: questions });
}

// Main function
async function main(filePath) {
  const docs = await loadPdf(filePath);
  const textChunks = await splitText(JSON.stringify(docs));
  const embeddings = await generateEmbeddings(textChunks);
  const context = textChunks.map(chunk => chunk.pageContent).join("\n\n");
  const jsonOutput = await formatJson(context, "Your questions here...");
  console.log(jsonOutput);
}

main("path/to/your/pdf.pdf");

This approach will help you extract and format the data from your PDF into a large JSON object. Adjust the PromptTemplate to match your specific JSON structure.

To continue talking to Dosu, mention @dosu.