From edd2919370eb727a846427d36d3184635cd57eed Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 12:28:54 -0700 Subject: [PATCH 01/23] feat: violations of expectations chain --- .../violation_of_expectation_chain.test.ts | 33 +++ .../chains/violation_of_expectation/types.ts | 95 +++++++ .../violation_of_expectation_chain.ts | 247 ++++++++++++++++++ .../violation_of_expectation_prompt.ts | 28 ++ 4 files changed, 403 insertions(+) create mode 100644 langchain/src/chains/tests/violation_of_expectation_chain.test.ts create mode 100644 langchain/src/chains/violation_of_expectation/types.ts create mode 100644 langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts create mode 100644 langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts diff --git a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts b/langchain/src/chains/tests/violation_of_expectation_chain.test.ts new file mode 100644 index 000000000000..56ee658bb925 --- /dev/null +++ b/langchain/src/chains/tests/violation_of_expectation_chain.test.ts @@ -0,0 +1,33 @@ +import { ChatOpenAI } from "../../chat_models/openai.js"; +import { OpenAIEmbeddings } from "../../embeddings/openai.js"; +import { HNSWLib } from "../../vectorstores/hnswlib.js"; +import { dummyMessages } from "../violation_of_expectation/types.js"; +import { ViolationOfExpectationChain } from "../violation_of_expectation/violation_of_expectation_chain.js"; + +test("should respond with the proper schema", async () => { + const vectorStore = await HNSWLib.fromTexts( + ["Mitochondria are the powerhouse of the cell", "Foo is red"], + [{ id: 2 }, { id: 1 }], + new OpenAIEmbeddings() + ); + const retriever = vectorStore.asRetriever(); + + const llm = new ChatOpenAI({ + modelName: "gpt-3.5-turbo", + }); + const chain = new ViolationOfExpectationChain({ + llm, + retriever, + }); + + const res = await chain.call({ + chat_history: dummyMessages, + }); + + console.log( + res.map((i) => ({ + explainedPredictionErrors: i.explainedPredictionErrors, + violationExplanation: i.violationExplanation, + })) + ); +}); diff --git a/langchain/src/chains/violation_of_expectation/types.ts b/langchain/src/chains/violation_of_expectation/types.ts new file mode 100644 index 000000000000..ea3cf3bf4b0f --- /dev/null +++ b/langchain/src/chains/violation_of_expectation/types.ts @@ -0,0 +1,95 @@ +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; +import { AIMessage, BaseMessage, HumanMessage } from "../../schema/index.js"; + +/** + * Contains the chunk of messages, along with the + * users response, which is the next message after the chunk. + */ +export type MessageChunkResult = { + chunkedMessages: BaseMessage[]; + /** + * User response can be undefined if the last message in + * the chat history was from the AI. + */ + userResponse?: HumanMessage; +}; + +export type PredictNextUserMessageResponse = { + userState: string; + predictedUserMessage: string; + insights: Array; +}; + +export const PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA = z.object({ + userState: z + .string() + .describe("Reasoning about the users internal mental state"), + predictedUserMessage: z + .string() + .describe("Likely possibility for the next user response"), + insights: z + .string() + .array() + .describe( + "A list of any additional data, relevant to the conversation that would be useful to improve prediction" + ), +}); + +export const PREDICT_NEXT_USER_MESSAGE_FUNCTION = { + name: "predictNextUserMessage", + description: "Predicts the next user message, along with insights.", + parameters: zodToJsonSchema(PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA), +}; + +export const PREDICTION_VIOLATIONS_ZOD_SCHEMA = z.object({ + violationExplanation: z + .string() + .optional() + .describe("How was the predication violated?"), + explainedPredictionErrors: z + .string() + .optional() + .array() + .describe("Explanations of how the prediction was violated and why"), + accuratePrediction: z + .boolean() + .describe("Whether or not there was a violation."), +}); + +export const PREDICTION_VIOLATIONS_FUNCTION = { + name: "predictionViolations", + description: + "Generates violations, errors and differences between the predicted user response, and the actual response, if any.", + parameters: zodToJsonSchema(PREDICTION_VIOLATIONS_ZOD_SCHEMA), +}; + +export const dummyMessages = [ + new HumanMessage( + "Hey, have you read the entry I wrote about college choices in my journal?" + ), + new AIMessage( + "Ah, the infamous journal of yours. Yeah, you left it open on the page. MIT, huh?" + ), + new HumanMessage( + "Yeah, but it's not just the prestige. I wrote about the bio-engineering program there, it's groundbreaking. Plus, the campus is near the Charles River." + ), + new AIMessage("Haha, the river! Planning to join the rowing team?"), + new HumanMessage( + "Nah, I was actually looking at their Rocket Team. Imagine, designing real rockets!" + ), + new AIMessage( + "Well, space is the final frontier. I, on the other hand, wrote about Stanford in my journal. The whole Silicon Valley ecosystem is exciting." + ), + new HumanMessage( + "True. Your love for entrepreneurship would fit well there. And it’s near Palo Alto, so your startup dreams could actually take off." + ), + new AIMessage( + "Yeah, but it’s also because the campus has those Rodin sculptures. Art amidst tech, you know?" + ), + new HumanMessage( + "Haha, I bet you also love that it's a bike-friendly campus. You and your vintage Schwinn." + ), + new AIMessage("Guilty as charged. So, East Coast or West Coast next year?"), + // new HumanMessage("Let’s see whose journal predicts the future better."), +]; diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts new file mode 100644 index 000000000000..1ad419d51376 --- /dev/null +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -0,0 +1,247 @@ +import { CallbackManagerForChainRun } from "../../callbacks/manager.js"; +import { ChatOpenAI } from "../../chat_models/openai.js"; +import { JsonOutputFunctionsParser } from "../../output_parsers/openai_functions.js"; +import { + BaseMessage, + ChainValues, + HumanMessage, + isBaseMessage, +} from "../../schema/index.js"; +import { BaseRetriever } from "../../schema/retriever.js"; +import { BaseChain, ChainInputs } from "../base.js"; +import { + MessageChunkResult, + PREDICTION_VIOLATIONS_FUNCTION, + PREDICT_NEXT_USER_MESSAGE_FUNCTION, + PredictNextUserMessageResponse, +} from "./types.js"; +import { + PREDICTION_VIOLATIONS_PROMPT, + PREDICT_NEXT_USER_MESSAGE_PROMPT, +} from "./violation_of_expectation_prompt.js"; + +export interface ViolationOfExpectationChainInput extends ChainInputs { + /** + * The retriever to use for retrieving stored + * thoughts and insights. + */ + retriever: BaseRetriever; + /** + * The LLM to use + */ + llm: ChatOpenAI; +} + +export class ViolationOfExpectationChain + extends BaseChain + implements ViolationOfExpectationChainInput +{ + static lc_name() { + return "ViolationOfExpectationChain"; + } + + _chainType(): string { + return "violation_of_expectation_chain"; + } + + chatHistoryKey = "chat_history"; + + thoughtsKey = "thoughts"; + + get inputKeys() { + return [this.chatHistoryKey]; + } + + get outputKeys() { + return [this.thoughtsKey]; + } + + retriever: BaseRetriever; + + llm: ChatOpenAI; + + constructor(fields: ViolationOfExpectationChainInput) { + super(fields); + this.retriever = fields.retriever; + this.llm = fields.llm; + } + + getChatHistoryString(chatHistory: BaseMessage[]): string { + return chatHistory + .map((chatMessage) => { + if (chatMessage._getType() === "human") { + return `Human: ${chatMessage.content}`; + } else if (chatMessage._getType() === "ai") { + return `AI: ${chatMessage.content}`; + } else { + return `${chatMessage.content}`; + } + }) + .join("\n"); + } + + getMessageChunks(chatHistory: BaseMessage[]): MessageChunkResult[] { + const newArray: MessageChunkResult[] = []; + const tempArray: BaseMessage[] = []; + + chatHistory.forEach((item, index) => { + tempArray.push(item); + if (item._getType() === "ai") { + let userResponse: BaseMessage | undefined = chatHistory[index + 1]; + if (!userResponse || userResponse._getType() !== "human") { + userResponse = undefined; + } + + newArray.push({ + chunkedMessages: tempArray, + userResponse: userResponse + ? new HumanMessage(userResponse) + : undefined, + }); + } + }); + + return newArray; + } + + async _call( + values: ChainValues, + _runManager?: CallbackManagerForChainRun + ): Promise { + if (!(this.chatHistoryKey in values)) { + throw new Error(`Chat history key ${this.chatHistoryKey} not found`); + } + + const chatHistory: unknown[] = values[this.chatHistoryKey]; + + const isEveryMessageBaseMessage = chatHistory.every((message) => + isBaseMessage(message) + ); + if (!isEveryMessageBaseMessage) { + throw new Error("Chat history must be an array of BaseMessages"); + } + + const messageChunks = this.getMessageChunks(chatHistory as BaseMessage[]); + + const userPredictions = await Promise.all( + messageChunks.map(async (chatHistoryChunk) => ({ + userPredictions: await this.predictNextUserMessage( + chatHistoryChunk.chunkedMessages, + this.llm + ), + userResponse: chatHistoryChunk.userResponse, + })) + ); + + const predictionViolations = await Promise.all( + userPredictions.map((prediction) => + this.getPredictionViolations({ + userPredictions: prediction.userPredictions, + userResponse: prediction.userResponse, + llm: this.llm, + }) + ) + ); + + return predictionViolations; + } + + private async predictNextUserMessage( + chatHistory: BaseMessage[], + llm: ChatOpenAI + ): Promise { + const messageString = this.getChatHistoryString(chatHistory); + + const outputParser = new JsonOutputFunctionsParser(); + + const llmWithFunctions = llm.bind({ + functions: [PREDICT_NEXT_USER_MESSAGE_FUNCTION], + function_call: { name: PREDICT_NEXT_USER_MESSAGE_FUNCTION.name }, + }); + + const chain = + PREDICT_NEXT_USER_MESSAGE_PROMPT.pipe(llmWithFunctions).pipe( + outputParser + ); + + const res = await chain.invoke({ + chat_history: messageString, + }); + + if ( + "userState" in res && + "predictedUserMessage" in res && + "insights" in res + ) { + return res as PredictNextUserMessageResponse; + } + throw new Error(`Invalid response from LLM: ${JSON.stringify(res)}`); + } + + private async retrieveRelevantInsights( + insights: Array + ): Promise> { + const relevantInsightsDocuments = ( + await Promise.all( + insights.map(async (insight) => { + const relevantInsight = await this.retriever.getRelevantDocuments( + insight + ); + return relevantInsight; + }) + ) + ).flat(); + + const relevantInsightsContent = relevantInsightsDocuments.map( + (document) => document.pageContent + ); + + return relevantInsightsContent; + } + + private async getPredictionViolations({ + userPredictions, + userResponse, + llm, + }: { + userPredictions: PredictNextUserMessageResponse; + userResponse?: BaseMessage; + llm: ChatOpenAI; + }) { + const outputParser = new JsonOutputFunctionsParser(); + + const llmWithFunctions = llm.bind({ + functions: [PREDICTION_VIOLATIONS_FUNCTION], + function_call: { name: PREDICTION_VIOLATIONS_FUNCTION.name }, + }); + + const chain = + PREDICTION_VIOLATIONS_PROMPT.pipe(llmWithFunctions).pipe(outputParser); + + const res = await chain.invoke({ + predicted_output: userPredictions.predictedUserMessage, + actual_output: userResponse?.content ?? "", + user_insights: userPredictions.insights.join("\n"), + }); + + return res; + } + + private async generateFacts({ + llm, + userMessage, + predictionViolations: { violationExplanation, explainedPredictionErrors }, + }: { + llm: ChatOpenAI; + userMessage: BaseMessage; + /** + * Optional if the prediction was accurate. + */ + predictionViolations?: { + violationExplanation: string; + explainedPredictionErrors: Array; + }; + }) { + throw new Error("Not implemented"); + } +} diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts new file mode 100644 index 000000000000..41aa198971d0 --- /dev/null +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts @@ -0,0 +1,28 @@ +import { PromptTemplate } from "../../prompts/prompt.js"; + +export const PREDICT_NEXT_USER_MESSAGE_PROMPT = PromptTemplate.fromTemplate(` +You have been tasked with coming up with insights and data-points based on a chat history between a human and an AI. +Given the user's chat history provide the following: +- Concise reasoning about the users internal mental state. +- Likely possibility for the next user response. +- A concise list of any additional data that would be useful to improve prediction. +-------- +Chat History: {chat_history}`); + +export const PREDICTION_VIOLATIONS_PROMPT = + PromptTemplate.fromTemplate(`You are analyzing some data points generated by an LLM and are tasked to derive any violations between expected outputs and actual outputs. +If there are no violations, respond with accuratePrediction = true. +You are provided with the following data-points: +- The predicted output the LLM generated based on what it believed the user would say. +- The actual output the user said. +- Key data about the user, their mental state, and any other insights derived from the chat history. +-------- +Predicted Output: {predicted_output} +-------- +Actual Output: {actual_output} +-------- +User Insights: {user_insights} +-------- +Respond with: +- How exactly was the expectation violated? Which parts were wrong? +- If there were errors with the prediction, what were they and why?`); From cd57077bff410f68b890c4157737cc44c26b84ca Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 13:37:11 -0700 Subject: [PATCH 02/23] fix: final fact generation chain --- .../violation_of_expectation_chain.test.ts | 9 +-- .../chains/violation_of_expectation/types.ts | 2 - .../violation_of_expectation_chain.ts | 73 +++++++++++++++++-- .../violation_of_expectation_prompt.ts | 26 ++++++- 4 files changed, 93 insertions(+), 17 deletions(-) diff --git a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts b/langchain/src/chains/tests/violation_of_expectation_chain.test.ts index 56ee658bb925..cb7d6cef037f 100644 --- a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts +++ b/langchain/src/chains/tests/violation_of_expectation_chain.test.ts @@ -24,10 +24,7 @@ test("should respond with the proper schema", async () => { chat_history: dummyMessages, }); - console.log( - res.map((i) => ({ - explainedPredictionErrors: i.explainedPredictionErrors, - violationExplanation: i.violationExplanation, - })) - ); + console.log({ + res, + }); }); diff --git a/langchain/src/chains/violation_of_expectation/types.ts b/langchain/src/chains/violation_of_expectation/types.ts index ea3cf3bf4b0f..11d61beb104d 100644 --- a/langchain/src/chains/violation_of_expectation/types.ts +++ b/langchain/src/chains/violation_of_expectation/types.ts @@ -45,11 +45,9 @@ export const PREDICT_NEXT_USER_MESSAGE_FUNCTION = { export const PREDICTION_VIOLATIONS_ZOD_SCHEMA = z.object({ violationExplanation: z .string() - .optional() .describe("How was the predication violated?"), explainedPredictionErrors: z .string() - .optional() .array() .describe("Explanations of how the prediction was violated and why"), accuratePrediction: z diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index 1ad419d51376..e70087970081 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -7,6 +7,7 @@ import { HumanMessage, isBaseMessage, } from "../../schema/index.js"; +import { StringOutputParser } from "../../schema/output_parser.js"; import { BaseRetriever } from "../../schema/retriever.js"; import { BaseChain, ChainInputs } from "../base.js"; import { @@ -16,6 +17,8 @@ import { PredictNextUserMessageResponse, } from "./types.js"; import { + GENERATE_FACTS_PROMPT, + GENERATE_REVISED_PREDICTION_PROMPT, PREDICTION_VIOLATIONS_PROMPT, PREDICT_NEXT_USER_MESSAGE_PROMPT, } from "./violation_of_expectation_prompt.js"; @@ -143,7 +146,18 @@ export class ViolationOfExpectationChain ) ); - return predictionViolations; + const insights = await Promise.all(predictionViolations.map((violation) => + this.generateFacts({ + llm: this.llm, + userMessage: violation.userMessage, + predictions: { + revisedPrediction: violation.revisedPrediction, + explainedPredictionErrors: violation.explainedPredictionErrors, + }, + }) + )); + + return insights; } private async predictNextUserMessage( @@ -224,24 +238,69 @@ export class ViolationOfExpectationChain user_insights: userPredictions.insights.join("\n"), }); - return res; + if ( + !( + "violationExplanation" in res && + "explainedPredictionErrors" in res && + "accuratePrediction" in res + ) + ) { + throw new Error( + "Predictions violations response is missing required fields" + ); + } + + const violations = res as { + violationExplanation: string; + explainedPredictionErrors: Array; + accuratePrediction: boolean; + }; + + // make one more call to regenerate the prediction. use the retrieved facts and generated facts. + + const revisedPredictionChain = GENERATE_REVISED_PREDICTION_PROMPT.pipe( + llm + ).pipe(new StringOutputParser()); + + const revisedPredictionRes = await revisedPredictionChain.invoke({ + prediction: userPredictions.predictedUserMessage, + explained_prediction_errors: + violations.explainedPredictionErrors.join("\n"), + user_insights: userPredictions.insights.join("\n"), + }); + + return { + userMessage: userResponse, + revisedPrediction: revisedPredictionRes, + explainedPredictionErrors: violations.explainedPredictionErrors, + }; } private async generateFacts({ llm, userMessage, - predictionViolations: { violationExplanation, explainedPredictionErrors }, + predictions, }: { llm: ChatOpenAI; - userMessage: BaseMessage; + userMessage?: BaseMessage; /** * Optional if the prediction was accurate. */ - predictionViolations?: { - violationExplanation: string; + predictions: { + revisedPrediction: string; explainedPredictionErrors: Array; }; }) { - throw new Error("Not implemented"); + const chain = GENERATE_FACTS_PROMPT.pipe(llm).pipe( + new StringOutputParser() + ); + + const res = await chain.invoke({ + prediction_violations: predictions.explainedPredictionErrors.join("\n"), + prediction: predictions.revisedPrediction, + user_message: userMessage?.content ?? "", + }); + + return res; } } diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts index 41aa198971d0..7995d6a1986e 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts @@ -10,8 +10,7 @@ Given the user's chat history provide the following: Chat History: {chat_history}`); export const PREDICTION_VIOLATIONS_PROMPT = - PromptTemplate.fromTemplate(`You are analyzing some data points generated by an LLM and are tasked to derive any violations between expected outputs and actual outputs. -If there are no violations, respond with accuratePrediction = true. + PromptTemplate.fromTemplate(`You are analyzing some data points generated by an LLM and are tasked to derive any violations between expected outputs and actual output. You are provided with the following data-points: - The predicted output the LLM generated based on what it believed the user would say. - The actual output the user said. @@ -26,3 +25,26 @@ User Insights: {user_insights} Respond with: - How exactly was the expectation violated? Which parts were wrong? - If there were errors with the prediction, what were they and why?`); + +export const GENERATE_REVISED_PREDICTION_PROMPT = PromptTemplate.fromTemplate(` +You have been tasked with revising a prediction on what a user might say in a chat conversation. +-------- +Your previous prediction: {prediction} +-------- +Ways in which your prediction was off: {explained_prediction_errors} +-------- +Key insights to the user: {user_insights} +-------- +Given the above, revise your prediction to be more accurate. +Revised Prediction:`); + +export const GENERATE_FACTS_PROMPT = PromptTemplate.fromTemplate(` +Given a user message, an LLM generated prediction of what that message might be, and a list of violations which the prediction made compared to the actual message, generate a fact about the user, relevant to the users message. +-------- +Prediction violations: {prediction_violations} +-------- +Revised prediction: {prediction} +-------- +Actual user message: {user_message} +-------- +Relevant fact:`); From 3b8d4e809875595211b138f996a445ddd6436939 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 14:27:11 -0700 Subject: [PATCH 03/23] fix: improve prompting --- .../violation_of_expectation_chain.test.ts | 2 +- .../chains/violation_of_expectation/types.ts | 33 +++++++++++-------- .../violation_of_expectation_chain.ts | 22 +++++++------ .../violation_of_expectation_prompt.ts | 17 ++++------ 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts b/langchain/src/chains/tests/violation_of_expectation_chain.test.ts index cb7d6cef037f..97069bd6c99e 100644 --- a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts +++ b/langchain/src/chains/tests/violation_of_expectation_chain.test.ts @@ -13,7 +13,7 @@ test("should respond with the proper schema", async () => { const retriever = vectorStore.asRetriever(); const llm = new ChatOpenAI({ - modelName: "gpt-3.5-turbo", + modelName: "gpt-4", }); const chain = new ViolationOfExpectationChain({ llm, diff --git a/langchain/src/chains/violation_of_expectation/types.ts b/langchain/src/chains/violation_of_expectation/types.ts index 11d61beb104d..8c788dc9666f 100644 --- a/langchain/src/chains/violation_of_expectation/types.ts +++ b/langchain/src/chains/violation_of_expectation/types.ts @@ -24,15 +24,17 @@ export type PredictNextUserMessageResponse = { export const PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA = z.object({ userState: z .string() - .describe("Reasoning about the users internal mental state"), + .describe("Concise reasoning about the users internal mental state."), predictedUserMessage: z .string() - .describe("Likely possibility for the next user response"), + .describe( + "Your prediction on how they will respond to the AI's most recent message." + ), insights: z .string() .array() .describe( - "A list of any additional data, relevant to the conversation that would be useful to improve prediction" + "A concise list of any additional insights that would be useful to improve prediction." ), }); @@ -64,30 +66,33 @@ export const PREDICTION_VIOLATIONS_FUNCTION = { export const dummyMessages = [ new HumanMessage( - "Hey, have you read the entry I wrote about college choices in my journal?" + "I've been thinking about the importance of time with myself to discover my voice. I feel like 1-2 hours is never enough." ), new AIMessage( - "Ah, the infamous journal of yours. Yeah, you left it open on the page. MIT, huh?" + "The concept of 'adequate time' varies. Have you tried different formats of introspection, such as morning pages or long-form writing, to see if they make the process more efficient?" ), new HumanMessage( - "Yeah, but it's not just the prestige. I wrote about the bio-engineering program there, it's groundbreaking. Plus, the campus is near the Charles River." + "I have tried journaling but never consistently. Sometimes it feels like writing doesn't capture everything." + ), + new AIMessage( + "Writing has its limits. What about other mediums like digital art, or interactive journal apps with dynamic prompts that dig deeper? Even coding a personal project can be a form of self-discovery." ), - new AIMessage("Haha, the river! Planning to join the rowing team?"), new HumanMessage( - "Nah, I was actually looking at their Rocket Team. Imagine, designing real rockets!" + "That's an interesting idea. I've never thought about coding as a form of self-discovery." ), new AIMessage( - "Well, space is the final frontier. I, on the other hand, wrote about Stanford in my journal. The whole Silicon Valley ecosystem is exciting." + "Since you're comfortable with code, consider building a tool to log and analyze your emotional state, thoughts, or personal growth metrics. It merges skill with introspection, makes the data quantifiable." ), new HumanMessage( - "True. Your love for entrepreneurship would fit well there. And it’s near Palo Alto, so your startup dreams could actually take off." + "The idea of quantifying emotions and personal growth is fascinating. But I wonder how much it can really capture the 'dark zone' within us." ), new AIMessage( - "Yeah, but it’s also because the campus has those Rodin sculptures. Art amidst tech, you know?" + "Good point. The 'dark zone' isn't fully quantifiable. But a tool could serve as a scaffold to explore those areas. It gives a structured approach to an unstructured problem." ), new HumanMessage( - "Haha, I bet you also love that it's a bike-friendly campus. You and your vintage Schwinn." + "You might be onto something. A structured approach could help unearth patterns or triggers I hadn't noticed." + ), + new AIMessage( + "Exactly. It's about creating a framework to understand what can't easily be understood. Then you can allocate those 5+ hours more effectively, targeting areas that your data flags." ), - new AIMessage("Guilty as charged. So, East Coast or West Coast next year?"), - // new HumanMessage("Let’s see whose journal predicts the future better."), ]; diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index e70087970081..9354c387fce3 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -146,16 +146,18 @@ export class ViolationOfExpectationChain ) ); - const insights = await Promise.all(predictionViolations.map((violation) => - this.generateFacts({ - llm: this.llm, - userMessage: violation.userMessage, - predictions: { - revisedPrediction: violation.revisedPrediction, - explainedPredictionErrors: violation.explainedPredictionErrors, - }, - }) - )); + const insights = await Promise.all( + predictionViolations.map((violation) => + this.generateFacts({ + llm: this.llm, + userMessage: violation.userMessage, + predictions: { + revisedPrediction: violation.revisedPrediction, + explainedPredictionErrors: violation.explainedPredictionErrors, + }, + }) + ) + ); return insights; } diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts index 7995d6a1986e..4602d06abcac 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts @@ -4,17 +4,16 @@ export const PREDICT_NEXT_USER_MESSAGE_PROMPT = PromptTemplate.fromTemplate(` You have been tasked with coming up with insights and data-points based on a chat history between a human and an AI. Given the user's chat history provide the following: - Concise reasoning about the users internal mental state. -- Likely possibility for the next user response. -- A concise list of any additional data that would be useful to improve prediction. +- Your prediction on how they will respond to the AI's most recent message. +- A concise list of any additional insights that would be useful to improve prediction. -------- Chat History: {chat_history}`); export const PREDICTION_VIOLATIONS_PROMPT = - PromptTemplate.fromTemplate(`You are analyzing some data points generated by an LLM and are tasked to derive any violations between expected outputs and actual output. -You are provided with the following data-points: -- The predicted output the LLM generated based on what it believed the user would say. -- The actual output the user said. -- Key data about the user, their mental state, and any other insights derived from the chat history. + PromptTemplate.fromTemplate(`You have been given a prediction and an actual message from a human and AI conversation. +Using the prediction, actual message, and additional user insights, generate the following: +- How exactly was the original prediction violated? Which parts were wrong? State the exact differences. +- If there were errors with the prediction, what were they and why? -------- Predicted Output: {predicted_output} -------- @@ -22,9 +21,7 @@ Actual Output: {actual_output} -------- User Insights: {user_insights} -------- -Respond with: -- How exactly was the expectation violated? Which parts were wrong? -- If there were errors with the prediction, what were they and why?`); +`); export const GENERATE_REVISED_PREDICTION_PROMPT = PromptTemplate.fromTemplate(` You have been tasked with revising a prediction on what a user might say in a chat conversation. From 630539c4bebdb27cffb9cb9b8762579eb7eb48de Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 14:32:22 -0700 Subject: [PATCH 04/23] refactor: generateRevisedPrediction to its own method --- .../violation_of_expectation_chain.ts | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index 9354c387fce3..bce5cadeeb88 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -150,7 +150,7 @@ export class ViolationOfExpectationChain predictionViolations.map((violation) => this.generateFacts({ llm: this.llm, - userMessage: violation.userMessage, + userResponse: violation.userResponse, predictions: { revisedPrediction: violation.revisedPrediction, explainedPredictionErrors: violation.explainedPredictionErrors, @@ -238,53 +238,58 @@ export class ViolationOfExpectationChain predicted_output: userPredictions.predictedUserMessage, actual_output: userResponse?.content ?? "", user_insights: userPredictions.insights.join("\n"), - }); - - if ( - !( - "violationExplanation" in res && - "explainedPredictionErrors" in res && - "accuratePrediction" in res - ) - ) { - throw new Error( - "Predictions violations response is missing required fields" - ); - } - - const violations = res as { + }) as Awaited<{ violationExplanation: string; explainedPredictionErrors: Array; accuratePrediction: boolean; - }; + }>; + + // Generate a revised prediction based on violations. + const revisedPrediction = await this.generateRevisedPrediction({ + llm, + originalPrediction: userPredictions.predictedUserMessage, + explainedPredictionErrors: res.explainedPredictionErrors, + userInsights: userPredictions.insights, + }); - // make one more call to regenerate the prediction. use the retrieved facts and generated facts. + return { + userResponse, + revisedPrediction, + explainedPredictionErrors: res.explainedPredictionErrors, + }; + } + private async generateRevisedPrediction({ + llm, + originalPrediction, + explainedPredictionErrors, + userInsights, + }: { + llm: ChatOpenAI; + originalPrediction: string; + explainedPredictionErrors: Array; + userInsights: Array; + }): Promise { const revisedPredictionChain = GENERATE_REVISED_PREDICTION_PROMPT.pipe( llm ).pipe(new StringOutputParser()); const revisedPredictionRes = await revisedPredictionChain.invoke({ - prediction: userPredictions.predictedUserMessage, - explained_prediction_errors: - violations.explainedPredictionErrors.join("\n"), - user_insights: userPredictions.insights.join("\n"), + prediction: originalPrediction, + explained_prediction_errors: explainedPredictionErrors.join("\n"), + user_insights: userInsights.join("\n"), }); - return { - userMessage: userResponse, - revisedPrediction: revisedPredictionRes, - explainedPredictionErrors: violations.explainedPredictionErrors, - }; + return revisedPredictionRes; } private async generateFacts({ llm, - userMessage, + userResponse, predictions, }: { llm: ChatOpenAI; - userMessage?: BaseMessage; + userResponse?: BaseMessage; /** * Optional if the prediction was accurate. */ @@ -300,7 +305,7 @@ export class ViolationOfExpectationChain const res = await chain.invoke({ prediction_violations: predictions.explainedPredictionErrors.join("\n"), prediction: predictions.revisedPrediction, - user_message: userMessage?.content ?? "", + user_message: userResponse?.content ?? "", }); return res; From e4e7008c6743dae3b61e8b5d6644acffcd67b624 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 14:35:41 -0700 Subject: [PATCH 05/23] chore: lint files --- .../violation_of_expectation_chain.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index bce5cadeeb88..a285bab3dc4a 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -234,11 +234,11 @@ export class ViolationOfExpectationChain const chain = PREDICTION_VIOLATIONS_PROMPT.pipe(llmWithFunctions).pipe(outputParser); - const res = await chain.invoke({ + const res = (await chain.invoke({ predicted_output: userPredictions.predictedUserMessage, actual_output: userResponse?.content ?? "", user_insights: userPredictions.insights.join("\n"), - }) as Awaited<{ + })) as Awaited<{ violationExplanation: string; explainedPredictionErrors: Array; accuratePrediction: boolean; From f5299bd878fb010ae79d35a188bee4f9bf38d52b Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 14:41:10 -0700 Subject: [PATCH 06/23] feat: added run manager for tracing --- .../violation_of_expectation_chain.ts | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index a285bab3dc4a..1d594660392f 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -109,7 +109,7 @@ export class ViolationOfExpectationChain async _call( values: ChainValues, - _runManager?: CallbackManagerForChainRun + runManager?: CallbackManagerForChainRun ): Promise { if (!(this.chatHistoryKey in values)) { throw new Error(`Chat history key ${this.chatHistoryKey} not found`); @@ -133,6 +133,7 @@ export class ViolationOfExpectationChain this.llm ), userResponse: chatHistoryChunk.userResponse, + runManager, })) ); @@ -142,6 +143,7 @@ export class ViolationOfExpectationChain userPredictions: prediction.userPredictions, userResponse: prediction.userResponse, llm: this.llm, + runManager, }) ) ); @@ -164,7 +166,8 @@ export class ViolationOfExpectationChain private async predictNextUserMessage( chatHistory: BaseMessage[], - llm: ChatOpenAI + llm: ChatOpenAI, + runManager?: CallbackManagerForChainRun ): Promise { const messageString = this.getChatHistoryString(chatHistory); @@ -180,9 +183,12 @@ export class ViolationOfExpectationChain outputParser ); - const res = await chain.invoke({ - chat_history: messageString, - }); + const res = await chain.invoke( + { + chat_history: messageString, + }, + runManager?.getChild("prediction") + ); if ( "userState" in res && @@ -191,6 +197,7 @@ export class ViolationOfExpectationChain ) { return res as PredictNextUserMessageResponse; } + throw new Error(`Invalid response from LLM: ${JSON.stringify(res)}`); } @@ -219,10 +226,12 @@ export class ViolationOfExpectationChain userPredictions, userResponse, llm, + runManager, }: { userPredictions: PredictNextUserMessageResponse; userResponse?: BaseMessage; llm: ChatOpenAI; + runManager?: CallbackManagerForChainRun; }) { const outputParser = new JsonOutputFunctionsParser(); @@ -234,11 +243,14 @@ export class ViolationOfExpectationChain const chain = PREDICTION_VIOLATIONS_PROMPT.pipe(llmWithFunctions).pipe(outputParser); - const res = (await chain.invoke({ - predicted_output: userPredictions.predictedUserMessage, - actual_output: userResponse?.content ?? "", - user_insights: userPredictions.insights.join("\n"), - })) as Awaited<{ + const res = (await chain.invoke( + { + predicted_output: userPredictions.predictedUserMessage, + actual_output: userResponse?.content ?? "", + user_insights: userPredictions.insights.join("\n"), + }, + runManager?.getChild("prediction_violations") + )) as Awaited<{ violationExplanation: string; explainedPredictionErrors: Array; accuratePrediction: boolean; @@ -250,6 +262,7 @@ export class ViolationOfExpectationChain originalPrediction: userPredictions.predictedUserMessage, explainedPredictionErrors: res.explainedPredictionErrors, userInsights: userPredictions.insights, + runManager, }); return { @@ -264,21 +277,26 @@ export class ViolationOfExpectationChain originalPrediction, explainedPredictionErrors, userInsights, + runManager, }: { llm: ChatOpenAI; originalPrediction: string; explainedPredictionErrors: Array; userInsights: Array; + runManager?: CallbackManagerForChainRun; }): Promise { const revisedPredictionChain = GENERATE_REVISED_PREDICTION_PROMPT.pipe( llm ).pipe(new StringOutputParser()); - const revisedPredictionRes = await revisedPredictionChain.invoke({ - prediction: originalPrediction, - explained_prediction_errors: explainedPredictionErrors.join("\n"), - user_insights: userInsights.join("\n"), - }); + const revisedPredictionRes = await revisedPredictionChain.invoke( + { + prediction: originalPrediction, + explained_prediction_errors: explainedPredictionErrors.join("\n"), + user_insights: userInsights.join("\n"), + }, + runManager?.getChild("prediction_revision") + ); return revisedPredictionRes; } @@ -287,6 +305,7 @@ export class ViolationOfExpectationChain llm, userResponse, predictions, + runManager, }: { llm: ChatOpenAI; userResponse?: BaseMessage; @@ -297,16 +316,20 @@ export class ViolationOfExpectationChain revisedPrediction: string; explainedPredictionErrors: Array; }; + runManager?: CallbackManagerForChainRun; }) { const chain = GENERATE_FACTS_PROMPT.pipe(llm).pipe( new StringOutputParser() ); - const res = await chain.invoke({ - prediction_violations: predictions.explainedPredictionErrors.join("\n"), - prediction: predictions.revisedPrediction, - user_message: userResponse?.content ?? "", - }); + const res = await chain.invoke( + { + prediction_violations: predictions.explainedPredictionErrors.join("\n"), + prediction: predictions.revisedPrediction, + user_message: userResponse?.content ?? "", + }, + runManager?.getChild("generate_facts") + ); return res; } From 848500e7f9c0f9d8baa49b047486468631cfc1d6 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 14:52:51 -0700 Subject: [PATCH 07/23] nit: use class llm instead of passing through --- .../violation_of_expectation_chain.ts | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index 1d594660392f..adc26c76e1d4 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -63,10 +63,16 @@ export class ViolationOfExpectationChain llm: ChatOpenAI; + jsonOutputParser: JsonOutputFunctionsParser; + + stringOutputParser: StringOutputParser; + constructor(fields: ViolationOfExpectationChainInput) { super(fields); this.retriever = fields.retriever; this.llm = fields.llm; + this.jsonOutputParser = new JsonOutputFunctionsParser(); + this.stringOutputParser = new StringOutputParser(); } getChatHistoryString(chatHistory: BaseMessage[]): string { @@ -126,32 +132,33 @@ export class ViolationOfExpectationChain const messageChunks = this.getMessageChunks(chatHistory as BaseMessage[]); + // Generate the initial prediction for every user message. const userPredictions = await Promise.all( messageChunks.map(async (chatHistoryChunk) => ({ userPredictions: await this.predictNextUserMessage( - chatHistoryChunk.chunkedMessages, - this.llm + chatHistoryChunk.chunkedMessages ), userResponse: chatHistoryChunk.userResponse, runManager, })) ); + // Generate insights, and prediction violations for every user message. + // This call also regenerates the prediction based on the violations. const predictionViolations = await Promise.all( userPredictions.map((prediction) => this.getPredictionViolations({ userPredictions: prediction.userPredictions, userResponse: prediction.userResponse, - llm: this.llm, runManager, }) ) ); + // Generate a fact/insight about the user for every set of messages. const insights = await Promise.all( predictionViolations.map((violation) => this.generateFacts({ - llm: this.llm, userResponse: violation.userResponse, predictions: { revisedPrediction: violation.revisedPrediction, @@ -166,22 +173,18 @@ export class ViolationOfExpectationChain private async predictNextUserMessage( chatHistory: BaseMessage[], - llm: ChatOpenAI, runManager?: CallbackManagerForChainRun ): Promise { const messageString = this.getChatHistoryString(chatHistory); - const outputParser = new JsonOutputFunctionsParser(); - - const llmWithFunctions = llm.bind({ + const llmWithFunctions = this.llm.bind({ functions: [PREDICT_NEXT_USER_MESSAGE_FUNCTION], function_call: { name: PREDICT_NEXT_USER_MESSAGE_FUNCTION.name }, }); - const chain = - PREDICT_NEXT_USER_MESSAGE_PROMPT.pipe(llmWithFunctions).pipe( - outputParser - ); + const chain = PREDICT_NEXT_USER_MESSAGE_PROMPT.pipe(llmWithFunctions).pipe( + this.jsonOutputParser + ); const res = await chain.invoke( { @@ -225,23 +228,20 @@ export class ViolationOfExpectationChain private async getPredictionViolations({ userPredictions, userResponse, - llm, runManager, }: { userPredictions: PredictNextUserMessageResponse; userResponse?: BaseMessage; - llm: ChatOpenAI; runManager?: CallbackManagerForChainRun; }) { - const outputParser = new JsonOutputFunctionsParser(); - - const llmWithFunctions = llm.bind({ + const llmWithFunctions = this.llm.bind({ functions: [PREDICTION_VIOLATIONS_FUNCTION], function_call: { name: PREDICTION_VIOLATIONS_FUNCTION.name }, }); - const chain = - PREDICTION_VIOLATIONS_PROMPT.pipe(llmWithFunctions).pipe(outputParser); + const chain = PREDICTION_VIOLATIONS_PROMPT.pipe(llmWithFunctions).pipe( + this.jsonOutputParser + ); const res = (await chain.invoke( { @@ -258,7 +258,6 @@ export class ViolationOfExpectationChain // Generate a revised prediction based on violations. const revisedPrediction = await this.generateRevisedPrediction({ - llm, originalPrediction: userPredictions.predictedUserMessage, explainedPredictionErrors: res.explainedPredictionErrors, userInsights: userPredictions.insights, @@ -273,21 +272,19 @@ export class ViolationOfExpectationChain } private async generateRevisedPrediction({ - llm, originalPrediction, explainedPredictionErrors, userInsights, runManager, }: { - llm: ChatOpenAI; originalPrediction: string; explainedPredictionErrors: Array; userInsights: Array; runManager?: CallbackManagerForChainRun; }): Promise { const revisedPredictionChain = GENERATE_REVISED_PREDICTION_PROMPT.pipe( - llm - ).pipe(new StringOutputParser()); + this.llm + ).pipe(this.stringOutputParser); const revisedPredictionRes = await revisedPredictionChain.invoke( { @@ -302,12 +299,10 @@ export class ViolationOfExpectationChain } private async generateFacts({ - llm, userResponse, predictions, runManager, }: { - llm: ChatOpenAI; userResponse?: BaseMessage; /** * Optional if the prediction was accurate. @@ -318,8 +313,8 @@ export class ViolationOfExpectationChain }; runManager?: CallbackManagerForChainRun; }) { - const chain = GENERATE_FACTS_PROMPT.pipe(llm).pipe( - new StringOutputParser() + const chain = GENERATE_FACTS_PROMPT.pipe(this.llm).pipe( + this.stringOutputParser ); const res = await chain.invoke( From 83d4019eb291174648926d9f20c7bb79ecc0a137 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 14:59:48 -0700 Subject: [PATCH 08/23] chore: added jsdoc --- .../violation_of_expectation_chain.ts | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index adc26c76e1d4..4f9eedc9dc9a 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -113,6 +113,25 @@ export class ViolationOfExpectationChain return newArray; } + /** + * This method processes a chat history to generate insights about the user. + * + * @param {ChainValues} values - The input values for the chain. It should contain a key for chat history. + * @param {CallbackManagerForChainRun} [runManager] - Optional callback manager for the chain run. + * + * @returns {Promise} A promise that resolves to a list of insights about the user. + * + * @throws {Error} If the chat history key is not found in the input values or if the chat history is not an array of BaseMessages. + * + * @description + * The method performs the following steps: + * 1. Checks if the chat history key is present in the input values and if the chat history is an array of BaseMessages. + * 2. Breaks the chat history into chunks of messages. + * 3. For each chunk, it generates an initial prediction for the user's next message. + * 4. For each prediction, it generates insights and prediction violations, and regenerates the prediction based on the violations. + * 5. For each set of messages, it generates a fact/insight about the user. + * The method returns a list of these insights. + */ async _call( values: ChainValues, runManager?: CallbackManagerForChainRun @@ -171,6 +190,16 @@ export class ViolationOfExpectationChain return insights; } + /** + * This method predicts the next user message based on the chat history. + * + * @param {BaseMessage[]} chatHistory - The chat history based on which the next user message is predicted. + * @param {CallbackManagerForChainRun} [runManager] - Optional callback manager for the chain run. + * + * @returns {Promise} A promise that resolves to the predicted next user message, the user state, and any insights. + * + * @throws {Error} If the response from the language model does not contain the expected keys: 'userState', 'predictedUserMessage', and 'insights'. + */ private async predictNextUserMessage( chatHistory: BaseMessage[], runManager?: CallbackManagerForChainRun @@ -204,6 +233,13 @@ export class ViolationOfExpectationChain throw new Error(`Invalid response from LLM: ${JSON.stringify(res)}`); } + /** + * Retrieves relevant insights based on the provided insights. + * + * @param {Array} insights - An array of insights to be used for retrieving relevant documents. + * + * @returns {Promise>} A promise that resolves to an array of relevant insights content. + */ private async retrieveRelevantInsights( insights: Array ): Promise> { @@ -225,6 +261,19 @@ export class ViolationOfExpectationChain return relevantInsightsContent; } + /** + * This method generates prediction violations based on the predicted and actual user responses. + * It also generates a revised prediction based on the identified violations. + * + * @param {Object} params - The parameters for the method. + * @param {PredictNextUserMessageResponse} params.userPredictions - The predicted user message, user state, and insights. + * @param {BaseMessage} [params.userResponse] - The actual user response. + * @param {CallbackManagerForChainRun} [params.runManager] - Optional callback manager for the chain run. + * + * @returns {Promise<{ userResponse: BaseMessage | undefined; revisedPrediction: string; explainedPredictionErrors: Array; }>} A promise that resolves to an object containing the actual user response, the revised prediction, and the explained prediction errors. + * + * @throws {Error} If the response from the language model does not contain the expected keys: 'violationExplanation', 'explainedPredictionErrors', and 'accuratePrediction'. + */ private async getPredictionViolations({ userPredictions, userResponse, @@ -271,6 +320,17 @@ export class ViolationOfExpectationChain }; } + /** + * This method generates a revised prediction based on the original prediction, explained prediction errors, and user insights. + * + * @param {Object} params - The parameters for the method. + * @param {string} params.originalPrediction - The original prediction made by the model. + * @param {Array} params.explainedPredictionErrors - An array of explained prediction errors. + * @param {Array} params.userInsights - An array of insights about the user. + * @param {CallbackManagerForChainRun} [params.runManager] - Optional callback manager for the chain run. + * + * @returns {Promise} A promise that resolves to a revised prediction. + */ private async generateRevisedPrediction({ originalPrediction, explainedPredictionErrors, @@ -298,6 +358,18 @@ export class ViolationOfExpectationChain return revisedPredictionRes; } + /** + * This method generates facts or insights about the user based on the revised prediction, explained prediction errors, and the user's response. + * + * @param {Object} params - The parameters for the method. + * @param {BaseMessage} [params.userResponse] - The actual user response. + * @param {Object} params.predictions - The revised prediction and explained prediction errors. + * @param {string} params.predictions.revisedPrediction - The revised prediction made by the model. + * @param {Array} params.predictions.explainedPredictionErrors - An array of explained prediction errors. + * @param {CallbackManagerForChainRun} [params.runManager] - Optional callback manager for the chain run. + * + * @returns {Promise} A promise that resolves to a string containing the generated facts or insights about the user. + */ private async generateFacts({ userResponse, predictions, From e0cb6087a60c553d2afb0fcd7b3663698afb716c Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 15:06:04 -0700 Subject: [PATCH 09/23] chore: more jsdoc --- .../violation_of_expectation_chain.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index 4f9eedc9dc9a..7d9a008f469d 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -23,6 +23,9 @@ import { PREDICT_NEXT_USER_MESSAGE_PROMPT, } from "./violation_of_expectation_prompt.js"; +/** + * Interface for the input parameters of the ViolationOfExpectationChain class. + */ export interface ViolationOfExpectationChainInput extends ChainInputs { /** * The retriever to use for retrieving stored @@ -35,6 +38,10 @@ export interface ViolationOfExpectationChainInput extends ChainInputs { llm: ChatOpenAI; } +/** + * Chain that generates key insights/facts of a user based on a + * a chat conversation with an AI. + */ export class ViolationOfExpectationChain extends BaseChain implements ViolationOfExpectationChainInput @@ -89,6 +96,22 @@ export class ViolationOfExpectationChain .join("\n"); } + /** + * This method breaks down the chat history into chunks of messages. + * Each chunk consists of a sequence of messages ending with an AI message and the subsequent user response, if any. + * + * @param {BaseMessage[]} chatHistory - The chat history to be chunked. + * + * @returns {MessageChunkResult[]} An array of message chunks. Each chunk includes a sequence of messages and the subsequent user response. + * + * @description + * The method iterates over the chat history and pushes each message into a temporary array. + * When it encounters an AI message, it checks for a subsequent user message. + * If a user message is found, it is considered as the user response to the AI message. + * If no user message is found after the AI message, the user response is undefined. + * The method then pushes the chunk (sequence of messages and user response) into the result array. + * This process continues until all messages in the chat history have been processed. + */ getMessageChunks(chatHistory: BaseMessage[]): MessageChunkResult[] { const newArray: MessageChunkResult[] = []; const tempArray: BaseMessage[] = []; From 68a4ba9e54ad27e69782f8225646a142865e9c5c Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 15:07:05 -0700 Subject: [PATCH 10/23] chore: lint files --- .../violation_of_expectation_chain.ts | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index 7d9a008f469d..e7738e418912 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -99,11 +99,11 @@ export class ViolationOfExpectationChain /** * This method breaks down the chat history into chunks of messages. * Each chunk consists of a sequence of messages ending with an AI message and the subsequent user response, if any. - * + * * @param {BaseMessage[]} chatHistory - The chat history to be chunked. - * + * * @returns {MessageChunkResult[]} An array of message chunks. Each chunk includes a sequence of messages and the subsequent user response. - * + * * @description * The method iterates over the chat history and pushes each message into a temporary array. * When it encounters an AI message, it checks for a subsequent user message. @@ -138,14 +138,14 @@ export class ViolationOfExpectationChain /** * This method processes a chat history to generate insights about the user. - * + * * @param {ChainValues} values - The input values for the chain. It should contain a key for chat history. * @param {CallbackManagerForChainRun} [runManager] - Optional callback manager for the chain run. - * + * * @returns {Promise} A promise that resolves to a list of insights about the user. - * + * * @throws {Error} If the chat history key is not found in the input values or if the chat history is not an array of BaseMessages. - * + * * @description * The method performs the following steps: * 1. Checks if the chat history key is present in the input values and if the chat history is an array of BaseMessages. @@ -215,12 +215,12 @@ export class ViolationOfExpectationChain /** * This method predicts the next user message based on the chat history. - * + * * @param {BaseMessage[]} chatHistory - The chat history based on which the next user message is predicted. * @param {CallbackManagerForChainRun} [runManager] - Optional callback manager for the chain run. - * + * * @returns {Promise} A promise that resolves to the predicted next user message, the user state, and any insights. - * + * * @throws {Error} If the response from the language model does not contain the expected keys: 'userState', 'predictedUserMessage', and 'insights'. */ private async predictNextUserMessage( @@ -258,9 +258,9 @@ export class ViolationOfExpectationChain /** * Retrieves relevant insights based on the provided insights. - * + * * @param {Array} insights - An array of insights to be used for retrieving relevant documents. - * + * * @returns {Promise>} A promise that resolves to an array of relevant insights content. */ private async retrieveRelevantInsights( @@ -287,14 +287,14 @@ export class ViolationOfExpectationChain /** * This method generates prediction violations based on the predicted and actual user responses. * It also generates a revised prediction based on the identified violations. - * + * * @param {Object} params - The parameters for the method. * @param {PredictNextUserMessageResponse} params.userPredictions - The predicted user message, user state, and insights. * @param {BaseMessage} [params.userResponse] - The actual user response. * @param {CallbackManagerForChainRun} [params.runManager] - Optional callback manager for the chain run. - * + * * @returns {Promise<{ userResponse: BaseMessage | undefined; revisedPrediction: string; explainedPredictionErrors: Array; }>} A promise that resolves to an object containing the actual user response, the revised prediction, and the explained prediction errors. - * + * * @throws {Error} If the response from the language model does not contain the expected keys: 'violationExplanation', 'explainedPredictionErrors', and 'accuratePrediction'. */ private async getPredictionViolations({ @@ -345,13 +345,13 @@ export class ViolationOfExpectationChain /** * This method generates a revised prediction based on the original prediction, explained prediction errors, and user insights. - * + * * @param {Object} params - The parameters for the method. * @param {string} params.originalPrediction - The original prediction made by the model. * @param {Array} params.explainedPredictionErrors - An array of explained prediction errors. * @param {Array} params.userInsights - An array of insights about the user. * @param {CallbackManagerForChainRun} [params.runManager] - Optional callback manager for the chain run. - * + * * @returns {Promise} A promise that resolves to a revised prediction. */ private async generateRevisedPrediction({ @@ -383,14 +383,14 @@ export class ViolationOfExpectationChain /** * This method generates facts or insights about the user based on the revised prediction, explained prediction errors, and the user's response. - * + * * @param {Object} params - The parameters for the method. * @param {BaseMessage} [params.userResponse] - The actual user response. * @param {Object} params.predictions - The revised prediction and explained prediction errors. * @param {string} params.predictions.revisedPrediction - The revised prediction made by the model. * @param {Array} params.predictions.explainedPredictionErrors - An array of explained prediction errors. * @param {CallbackManagerForChainRun} [params.runManager] - Optional callback manager for the chain run. - * + * * @returns {Promise} A promise that resolves to a string containing the generated facts or insights about the user. */ private async generateFacts({ From a5da8c3d44d25e73fbd40fd5a4673d45f25aed6d Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 15:21:13 -0700 Subject: [PATCH 11/23] feat: connect retriever to chain --- .../violation_of_expectation_chain.ts | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index e7738e418912..1edb68100700 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -246,14 +246,27 @@ export class ViolationOfExpectationChain ); if ( - "userState" in res && - "predictedUserMessage" in res && - "insights" in res + !( + "userState" in res && + "predictedUserMessage" in res && + "insights" in res + ) ) { - return res as PredictNextUserMessageResponse; + throw new Error(`Invalid response from LLM: ${JSON.stringify(res)}`); } - throw new Error(`Invalid response from LLM: ${JSON.stringify(res)}`); + const predictionResponse = res as PredictNextUserMessageResponse; + + // Query the retriever for relevant insights. Use the generates insights as a query. + const retrievedDocs = await this.retrieveRelevantInsights( + predictionResponse.insights + ); + const relevantDocs = [...predictionResponse.insights, ...retrievedDocs]; + + return { + ...predictionResponse, + insights: relevantDocs, + }; } /** @@ -266,16 +279,15 @@ export class ViolationOfExpectationChain private async retrieveRelevantInsights( insights: Array ): Promise> { - const relevantInsightsDocuments = ( - await Promise.all( - insights.map(async (insight) => { - const relevantInsight = await this.retriever.getRelevantDocuments( - insight - ); - return relevantInsight; - }) - ) - ).flat(); + // Only extract the first relevant doc from the retriever. We don't need more than one. + const relevantInsightsDocuments = await Promise.all( + insights.map(async (insight) => { + const relevantInsight = await this.retriever.getRelevantDocuments( + insight + ); + return relevantInsight[0]; + }) + ); const relevantInsightsContent = relevantInsightsDocuments.map( (document) => document.pageContent From 5b3db0e30bad87462a4eb3b73151b305604c82e6 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 15:25:25 -0700 Subject: [PATCH 12/23] feat: add .fromLLM static method --- ...olation_of_expectation_chain.init.test.ts} | 4 +- .../violation_of_expectation_chain.ts | 37 ++++++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) rename langchain/src/chains/tests/{violation_of_expectation_chain.test.ts => violation_of_expectation_chain.init.test.ts} (81%) diff --git a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts b/langchain/src/chains/tests/violation_of_expectation_chain.init.test.ts similarity index 81% rename from langchain/src/chains/tests/violation_of_expectation_chain.test.ts rename to langchain/src/chains/tests/violation_of_expectation_chain.init.test.ts index 97069bd6c99e..463830c11fcd 100644 --- a/langchain/src/chains/tests/violation_of_expectation_chain.test.ts +++ b/langchain/src/chains/tests/violation_of_expectation_chain.init.test.ts @@ -2,7 +2,7 @@ import { ChatOpenAI } from "../../chat_models/openai.js"; import { OpenAIEmbeddings } from "../../embeddings/openai.js"; import { HNSWLib } from "../../vectorstores/hnswlib.js"; import { dummyMessages } from "../violation_of_expectation/types.js"; -import { ViolationOfExpectationChain } from "../violation_of_expectation/violation_of_expectation_chain.js"; +import { ViolationOfExpectationsChain } from "../violation_of_expectation/violation_of_expectation_chain.js"; test("should respond with the proper schema", async () => { const vectorStore = await HNSWLib.fromTexts( @@ -15,7 +15,7 @@ test("should respond with the proper schema", async () => { const llm = new ChatOpenAI({ modelName: "gpt-4", }); - const chain = new ViolationOfExpectationChain({ + const chain = new ViolationOfExpectationsChain({ llm, retriever, }); diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts index 1edb68100700..d4470daebb1d 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts @@ -24,9 +24,9 @@ import { } from "./violation_of_expectation_prompt.js"; /** - * Interface for the input parameters of the ViolationOfExpectationChain class. + * Interface for the input parameters of the ViolationOfExpectationsChain class. */ -export interface ViolationOfExpectationChainInput extends ChainInputs { +export interface ViolationOfExpectationsChainInput extends ChainInputs { /** * The retriever to use for retrieving stored * thoughts and insights. @@ -42,12 +42,12 @@ export interface ViolationOfExpectationChainInput extends ChainInputs { * Chain that generates key insights/facts of a user based on a * a chat conversation with an AI. */ -export class ViolationOfExpectationChain +export class ViolationOfExpectationsChain extends BaseChain - implements ViolationOfExpectationChainInput + implements ViolationOfExpectationsChainInput { static lc_name() { - return "ViolationOfExpectationChain"; + return "ViolationOfExpectationsChain"; } _chainType(): string { @@ -74,7 +74,7 @@ export class ViolationOfExpectationChain stringOutputParser: StringOutputParser; - constructor(fields: ViolationOfExpectationChainInput) { + constructor(fields: ViolationOfExpectationsChainInput) { super(fields); this.retriever = fields.retriever; this.llm = fields.llm; @@ -435,4 +435,29 @@ export class ViolationOfExpectationChain return res; } + + /** + * Static method that creates a ViolationOfExpectationsChain instance from a + * ChatOpenAI and retriever. It also accepts optional options + * to customize the chain. + * + * @param llm The ChatOpenAI instance. + * @param retriever The retriever used for similarity search. + * @param options Optional options to customize the chain. + * + * @returns A new instance of ViolationOfExpectationsChain. + */ + static fromLLM( + llm: ChatOpenAI, + retriever: BaseRetriever, + options?: Partial< + Omit + > + ): ViolationOfExpectationsChain { + return new this({ + retriever, + llm, + ...options, + }); + } } From 5d01fd81cb2e5d40d4a820a404bc9fefdb9dea03 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 16:08:42 -0700 Subject: [PATCH 13/23] chore: added pure comment to pure exports, renamed to be plural --- ...s => violation_of_expectations_chain.init.test.ts} | 4 ++-- .../types.ts | 4 ++-- .../violation_of_expectations_chain.ts} | 2 +- .../violation_of_expectations_prompt.ts} | 11 +++++++---- 4 files changed, 12 insertions(+), 9 deletions(-) rename langchain/src/chains/tests/{violation_of_expectation_chain.init.test.ts => violation_of_expectations_chain.init.test.ts} (86%) rename langchain/src/chains/{violation_of_expectation => violation_of_expectations}/types.ts (96%) rename langchain/src/chains/{violation_of_expectation/violation_of_expectation_chain.ts => violation_of_expectations/violation_of_expectations_chain.ts} (99%) rename langchain/src/chains/{violation_of_expectation/violation_of_expectation_prompt.ts => violation_of_expectations/violation_of_expectations_prompt.ts} (80%) diff --git a/langchain/src/chains/tests/violation_of_expectation_chain.init.test.ts b/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts similarity index 86% rename from langchain/src/chains/tests/violation_of_expectation_chain.init.test.ts rename to langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts index 463830c11fcd..c3224e8b8068 100644 --- a/langchain/src/chains/tests/violation_of_expectation_chain.init.test.ts +++ b/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts @@ -1,8 +1,8 @@ import { ChatOpenAI } from "../../chat_models/openai.js"; import { OpenAIEmbeddings } from "../../embeddings/openai.js"; import { HNSWLib } from "../../vectorstores/hnswlib.js"; -import { dummyMessages } from "../violation_of_expectation/types.js"; -import { ViolationOfExpectationsChain } from "../violation_of_expectation/violation_of_expectation_chain.js"; +import { dummyMessages } from "../violation_of_expectations/types.js"; +import { ViolationOfExpectationsChain } from "../violation_of_expectations/violation_of_expectations_chain.js"; test("should respond with the proper schema", async () => { const vectorStore = await HNSWLib.fromTexts( diff --git a/langchain/src/chains/violation_of_expectation/types.ts b/langchain/src/chains/violation_of_expectations/types.ts similarity index 96% rename from langchain/src/chains/violation_of_expectation/types.ts rename to langchain/src/chains/violation_of_expectations/types.ts index 8c788dc9666f..78132325ad62 100644 --- a/langchain/src/chains/violation_of_expectation/types.ts +++ b/langchain/src/chains/violation_of_expectations/types.ts @@ -21,7 +21,7 @@ export type PredictNextUserMessageResponse = { insights: Array; }; -export const PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA = z.object({ +export const PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA = /* #__PURE__ */ z.object({ userState: z .string() .describe("Concise reasoning about the users internal mental state."), @@ -44,7 +44,7 @@ export const PREDICT_NEXT_USER_MESSAGE_FUNCTION = { parameters: zodToJsonSchema(PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA), }; -export const PREDICTION_VIOLATIONS_ZOD_SCHEMA = z.object({ +export const PREDICTION_VIOLATIONS_ZOD_SCHEMA = /* #__PURE__ */ z.object({ violationExplanation: z .string() .describe("How was the predication violated?"), diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts b/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts similarity index 99% rename from langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts rename to langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts index d4470daebb1d..f6effb865dd0 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_chain.ts +++ b/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts @@ -21,7 +21,7 @@ import { GENERATE_REVISED_PREDICTION_PROMPT, PREDICTION_VIOLATIONS_PROMPT, PREDICT_NEXT_USER_MESSAGE_PROMPT, -} from "./violation_of_expectation_prompt.js"; +} from "./violation_of_expectations_prompt.js"; /** * Interface for the input parameters of the ViolationOfExpectationsChain class. diff --git a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts b/langchain/src/chains/violation_of_expectations/violation_of_expectations_prompt.ts similarity index 80% rename from langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts rename to langchain/src/chains/violation_of_expectations/violation_of_expectations_prompt.ts index 4602d06abcac..78544b2a3dc2 100644 --- a/langchain/src/chains/violation_of_expectation/violation_of_expectation_prompt.ts +++ b/langchain/src/chains/violation_of_expectations/violation_of_expectations_prompt.ts @@ -1,6 +1,7 @@ import { PromptTemplate } from "../../prompts/prompt.js"; -export const PREDICT_NEXT_USER_MESSAGE_PROMPT = PromptTemplate.fromTemplate(` +export const PREDICT_NEXT_USER_MESSAGE_PROMPT = + /* #__PURE__ */ PromptTemplate.fromTemplate(` You have been tasked with coming up with insights and data-points based on a chat history between a human and an AI. Given the user's chat history provide the following: - Concise reasoning about the users internal mental state. @@ -10,7 +11,7 @@ Given the user's chat history provide the following: Chat History: {chat_history}`); export const PREDICTION_VIOLATIONS_PROMPT = - PromptTemplate.fromTemplate(`You have been given a prediction and an actual message from a human and AI conversation. + /* #__PURE__ */ PromptTemplate.fromTemplate(`You have been given a prediction and an actual message from a human and AI conversation. Using the prediction, actual message, and additional user insights, generate the following: - How exactly was the original prediction violated? Which parts were wrong? State the exact differences. - If there were errors with the prediction, what were they and why? @@ -23,7 +24,8 @@ User Insights: {user_insights} -------- `); -export const GENERATE_REVISED_PREDICTION_PROMPT = PromptTemplate.fromTemplate(` +export const GENERATE_REVISED_PREDICTION_PROMPT = + /* #__PURE__ */ PromptTemplate.fromTemplate(` You have been tasked with revising a prediction on what a user might say in a chat conversation. -------- Your previous prediction: {prediction} @@ -35,7 +37,8 @@ Key insights to the user: {user_insights} Given the above, revise your prediction to be more accurate. Revised Prediction:`); -export const GENERATE_FACTS_PROMPT = PromptTemplate.fromTemplate(` +export const GENERATE_FACTS_PROMPT = + /* #__PURE__ */ PromptTemplate.fromTemplate(` Given a user message, an LLM generated prediction of what that message might be, and a list of violations which the prediction made compared to the actual message, generate a fact about the user, relevant to the users message. -------- Prediction violations: {prediction_violations} From af0a23bcd43fab0cbbd4edab8a14a6d542e3076a Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 16:24:58 -0700 Subject: [PATCH 14/23] chore: fix side effect build errors, properly export chain --- .../test-exports-bun/src/entrypoints.js | 1 + .../test-exports-cf/src/entrypoints.js | 1 + .../test-exports-cjs/src/entrypoints.js | 1 + .../test-exports-esbuild/src/entrypoints.js | 1 + .../test-exports-esm/src/entrypoints.js | 1 + .../test-exports-vercel/src/entrypoints.js | 1 + .../test-exports-vite/src/entrypoints.js | 1 + langchain/.gitignore | 3 + langchain/package.json | 8 ++ langchain/scripts/create-entrypoints.js | 1 + ...olation_of_expectations_chain.init.test.ts | 35 +++++- .../chains/violation_of_expectations/types.ts | 111 +++++++----------- langchain/src/load/import_map.ts | 1 + langchain/tsconfig.json | 1 + 14 files changed, 97 insertions(+), 70 deletions(-) diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 6e08cbe691fc..4876e5795cc8 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -6,6 +6,7 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; +export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 6e08cbe691fc..4876e5795cc8 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -6,6 +6,7 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; +export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index 4391d69fb0e0..75609c2269f3 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -6,6 +6,7 @@ const base_language = require("langchain/base_language"); const tools = require("langchain/tools"); const chains = require("langchain/chains"); const chains_openai_functions = require("langchain/chains/openai_functions"); +const chains_violation_of_expectations = require("langchain/chains/violation_of_expectations"); const embeddings_base = require("langchain/embeddings/base"); const embeddings_cache_backed = require("langchain/embeddings/cache_backed"); const embeddings_fake = require("langchain/embeddings/fake"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index d430748fbcd4..052936b44c6f 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -6,6 +6,7 @@ import * as base_language from "langchain/base_language"; import * as tools from "langchain/tools"; import * as chains from "langchain/chains"; import * as chains_openai_functions from "langchain/chains/openai_functions"; +import * as chains_violation_of_expectations from "langchain/chains/violation_of_expectations"; import * as embeddings_base from "langchain/embeddings/base"; import * as embeddings_cache_backed from "langchain/embeddings/cache_backed"; import * as embeddings_fake from "langchain/embeddings/fake"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index d430748fbcd4..052936b44c6f 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -6,6 +6,7 @@ import * as base_language from "langchain/base_language"; import * as tools from "langchain/tools"; import * as chains from "langchain/chains"; import * as chains_openai_functions from "langchain/chains/openai_functions"; +import * as chains_violation_of_expectations from "langchain/chains/violation_of_expectations"; import * as embeddings_base from "langchain/embeddings/base"; import * as embeddings_cache_backed from "langchain/embeddings/cache_backed"; import * as embeddings_fake from "langchain/embeddings/fake"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 6e08cbe691fc..4876e5795cc8 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -6,6 +6,7 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; +export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 6e08cbe691fc..4876e5795cc8 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -6,6 +6,7 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; +export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; diff --git a/langchain/.gitignore b/langchain/.gitignore index dcb989e1bc32..c1199962c704 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -64,6 +64,9 @@ chains/sql_db.d.ts chains/graph_qa/cypher.cjs chains/graph_qa/cypher.js chains/graph_qa/cypher.d.ts +chains/violation_of_expectations.cjs +chains/violation_of_expectations.js +chains/violation_of_expectations.d.ts embeddings/base.cjs embeddings/base.js embeddings/base.d.ts diff --git a/langchain/package.json b/langchain/package.json index 77e8164f036e..8cc77b2837e3 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -76,6 +76,9 @@ "chains/graph_qa/cypher.cjs", "chains/graph_qa/cypher.js", "chains/graph_qa/cypher.d.ts", + "chains/violation_of_expectations.cjs", + "chains/violation_of_expectations.js", + "chains/violation_of_expectations.d.ts", "embeddings/base.cjs", "embeddings/base.js", "embeddings/base.d.ts", @@ -1370,6 +1373,11 @@ "import": "./chains/graph_qa/cypher.js", "require": "./chains/graph_qa/cypher.cjs" }, + "./chains/violation_of_expectations": { + "types": "./chains/violation_of_expectations.d.ts", + "import": "./chains/violation_of_expectations.js", + "require": "./chains/violation_of_expectations.cjs" + }, "./embeddings/base": { "types": "./embeddings/base.d.ts", "import": "./embeddings/base.js", diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index ad1361e4f6b2..8f90ab72c575 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -34,6 +34,7 @@ const entrypoints = { "chains/query_constructor/ir": "chains/query_constructor/ir", "chains/sql_db": "chains/sql_db/index", "chains/graph_qa/cypher": "chains/graph_qa/cypher", + "chains/violation_of_expectations": "chains/violation_of_expectations/violation_of_expectations_chain", // embeddings "embeddings/base": "embeddings/base", "embeddings/bedrock": "embeddings/bedrock", diff --git a/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts b/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts index c3224e8b8068..cc311de0d325 100644 --- a/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts +++ b/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts @@ -1,9 +1,42 @@ import { ChatOpenAI } from "../../chat_models/openai.js"; import { OpenAIEmbeddings } from "../../embeddings/openai.js"; +import { AIMessage, HumanMessage } from "../../schema/index.js"; import { HNSWLib } from "../../vectorstores/hnswlib.js"; -import { dummyMessages } from "../violation_of_expectations/types.js"; import { ViolationOfExpectationsChain } from "../violation_of_expectations/violation_of_expectations_chain.js"; +const dummyMessages = [ + new HumanMessage( + "I've been thinking about the importance of time with myself to discover my voice. I feel like 1-2 hours is never enough." + ), + new AIMessage( + "The concept of 'adequate time' varies. Have you tried different formats of introspection, such as morning pages or long-form writing, to see if they make the process more efficient?" + ), + new HumanMessage( + "I have tried journaling but never consistently. Sometimes it feels like writing doesn't capture everything." + ), + new AIMessage( + "Writing has its limits. What about other mediums like digital art, or interactive journal apps with dynamic prompts that dig deeper? Even coding a personal project can be a form of self-discovery." + ), + new HumanMessage( + "That's an interesting idea. I've never thought about coding as a form of self-discovery." + ), + new AIMessage( + "Since you're comfortable with code, consider building a tool to log and analyze your emotional state, thoughts, or personal growth metrics. It merges skill with introspection, makes the data quantifiable." + ), + new HumanMessage( + "The idea of quantifying emotions and personal growth is fascinating. But I wonder how much it can really capture the 'dark zone' within us." + ), + new AIMessage( + "Good point. The 'dark zone' isn't fully quantifiable. But a tool could serve as a scaffold to explore those areas. It gives a structured approach to an unstructured problem." + ), + new HumanMessage( + "You might be onto something. A structured approach could help unearth patterns or triggers I hadn't noticed." + ), + new AIMessage( + "Exactly. It's about creating a framework to understand what can't easily be understood. Then you can allocate those 5+ hours more effectively, targeting areas that your data flags." + ), +]; + test("should respond with the proper schema", async () => { const vectorStore = await HNSWLib.fromTexts( ["Mitochondria are the powerhouse of the cell", "Foo is red"], diff --git a/langchain/src/chains/violation_of_expectations/types.ts b/langchain/src/chains/violation_of_expectations/types.ts index 78132325ad62..3db2be311f45 100644 --- a/langchain/src/chains/violation_of_expectations/types.ts +++ b/langchain/src/chains/violation_of_expectations/types.ts @@ -1,6 +1,4 @@ -import { z } from "zod"; -import { zodToJsonSchema } from "zod-to-json-schema"; -import { AIMessage, BaseMessage, HumanMessage } from "../../schema/index.js"; +import { BaseMessage, HumanMessage } from "../../schema/index.js"; /** * Contains the chunk of messages, along with the @@ -21,78 +19,53 @@ export type PredictNextUserMessageResponse = { insights: Array; }; -export const PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA = /* #__PURE__ */ z.object({ - userState: z - .string() - .describe("Concise reasoning about the users internal mental state."), - predictedUserMessage: z - .string() - .describe( - "Your prediction on how they will respond to the AI's most recent message." - ), - insights: z - .string() - .array() - .describe( - "A concise list of any additional insights that would be useful to improve prediction." - ), -}); - export const PREDICT_NEXT_USER_MESSAGE_FUNCTION = { name: "predictNextUserMessage", description: "Predicts the next user message, along with insights.", - parameters: zodToJsonSchema(PREDICT_NEXT_USER_MESSAGE_ZOD_SCHEMA), + parameters: { + type: "object", + properties: { + userState: { + type: "string", + description: "Concise reasoning about the users internal mental state.", + }, + predictedUserMessage: { + type: "string", + description: + "Your prediction on how they will respond to the AI's most recent message.", + }, + insights: { + type: "array", + items: { + type: "string", + }, + description: + "A concise list of any additional insights that would be useful to improve prediction.", + }, + }, + required: ["userState", "predictedUserMessage", "insights"], + }, }; -export const PREDICTION_VIOLATIONS_ZOD_SCHEMA = /* #__PURE__ */ z.object({ - violationExplanation: z - .string() - .describe("How was the predication violated?"), - explainedPredictionErrors: z - .string() - .array() - .describe("Explanations of how the prediction was violated and why"), - accuratePrediction: z - .boolean() - .describe("Whether or not there was a violation."), -}); - export const PREDICTION_VIOLATIONS_FUNCTION = { name: "predictionViolations", description: - "Generates violations, errors and differences between the predicted user response, and the actual response, if any.", - parameters: zodToJsonSchema(PREDICTION_VIOLATIONS_ZOD_SCHEMA), + "Generates violations, errors and differences between the predicted user response, and the actual response.", + parameters: { + type: "object", + properties: { + violationExplanation: { + type: "string", + description: "How was the predication violated?", + }, + explainedPredictionErrors: { + type: "array", + items: { + type: "string", + }, + description: "Explanations of how the prediction was violated and why", + }, + }, + required: ["violationExplanation", "explainedPredictionErrors"], + }, }; - -export const dummyMessages = [ - new HumanMessage( - "I've been thinking about the importance of time with myself to discover my voice. I feel like 1-2 hours is never enough." - ), - new AIMessage( - "The concept of 'adequate time' varies. Have you tried different formats of introspection, such as morning pages or long-form writing, to see if they make the process more efficient?" - ), - new HumanMessage( - "I have tried journaling but never consistently. Sometimes it feels like writing doesn't capture everything." - ), - new AIMessage( - "Writing has its limits. What about other mediums like digital art, or interactive journal apps with dynamic prompts that dig deeper? Even coding a personal project can be a form of self-discovery." - ), - new HumanMessage( - "That's an interesting idea. I've never thought about coding as a form of self-discovery." - ), - new AIMessage( - "Since you're comfortable with code, consider building a tool to log and analyze your emotional state, thoughts, or personal growth metrics. It merges skill with introspection, makes the data quantifiable." - ), - new HumanMessage( - "The idea of quantifying emotions and personal growth is fascinating. But I wonder how much it can really capture the 'dark zone' within us." - ), - new AIMessage( - "Good point. The 'dark zone' isn't fully quantifiable. But a tool could serve as a scaffold to explore those areas. It gives a structured approach to an unstructured problem." - ), - new HumanMessage( - "You might be onto something. A structured approach could help unearth patterns or triggers I hadn't noticed." - ), - new AIMessage( - "Exactly. It's about creating a framework to understand what can't easily be understood. Then you can allocate those 5+ hours more effectively, targeting areas that your data flags." - ), -]; diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index 3ad007a2c345..1a16ade673cb 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -7,6 +7,7 @@ export * as base_language from "../base_language/index.js"; export * as tools from "../tools/index.js"; export * as chains from "../chains/index.js"; export * as chains__openai_functions from "../chains/openai_functions/index.js"; +export * as chains__violation_of_expectations from "../chains/violation_of_expectations/violation_of_expectations_chain.js"; export * as embeddings__base from "../embeddings/base.js"; export * as embeddings__cache_backed from "../embeddings/cache_backed.js"; export * as embeddings__fake from "../embeddings/fake.js"; diff --git a/langchain/tsconfig.json b/langchain/tsconfig.json index 684232876b79..300881ebfa64 100644 --- a/langchain/tsconfig.json +++ b/langchain/tsconfig.json @@ -54,6 +54,7 @@ "src/chains/query_constructor/ir.ts", "src/chains/sql_db/index.ts", "src/chains/graph_qa/cypher.ts", + "src/chains/violation_of_expectations/violation_of_expectations_chain.ts", "src/embeddings/base.ts", "src/embeddings/bedrock.ts", "src/embeddings/cache_backed.ts", From 16e9a835a8194d3c36cb95fae826ce90a1f4a4a0 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 16:38:07 -0700 Subject: [PATCH 15/23] fix: remove dup strings from relevant docs, nit on testing data --- ...olation_of_expectations_chain.init.test.ts | 25 ++----------------- .../violation_of_expectations_chain.ts | 13 +++++++--- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts b/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts index cc311de0d325..931310ae39a1 100644 --- a/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts +++ b/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts @@ -14,33 +14,12 @@ const dummyMessages = [ new HumanMessage( "I have tried journaling but never consistently. Sometimes it feels like writing doesn't capture everything." ), - new AIMessage( - "Writing has its limits. What about other mediums like digital art, or interactive journal apps with dynamic prompts that dig deeper? Even coding a personal project can be a form of self-discovery." - ), - new HumanMessage( - "That's an interesting idea. I've never thought about coding as a form of self-discovery." - ), - new AIMessage( - "Since you're comfortable with code, consider building a tool to log and analyze your emotional state, thoughts, or personal growth metrics. It merges skill with introspection, makes the data quantifiable." - ), - new HumanMessage( - "The idea of quantifying emotions and personal growth is fascinating. But I wonder how much it can really capture the 'dark zone' within us." - ), - new AIMessage( - "Good point. The 'dark zone' isn't fully quantifiable. But a tool could serve as a scaffold to explore those areas. It gives a structured approach to an unstructured problem." - ), - new HumanMessage( - "You might be onto something. A structured approach could help unearth patterns or triggers I hadn't noticed." - ), - new AIMessage( - "Exactly. It's about creating a framework to understand what can't easily be understood. Then you can allocate those 5+ hours more effectively, targeting areas that your data flags." - ), ]; test("should respond with the proper schema", async () => { const vectorStore = await HNSWLib.fromTexts( - ["Mitochondria are the powerhouse of the cell", "Foo is red"], - [{ id: 2 }, { id: 1 }], + [" "], + [{ id: 1 }], new OpenAIEmbeddings() ); const retriever = vectorStore.asRetriever(); diff --git a/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts b/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts index f6effb865dd0..97992ea9d5b0 100644 --- a/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts +++ b/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts @@ -96,6 +96,10 @@ export class ViolationOfExpectationsChain .join("\n"); } + removeDuplicateStrings(strings: Array): Array { + return [...new Set(strings)]; + } + /** * This method breaks down the chat history into chunks of messages. * Each chunk consists of a sequence of messages ending with an AI message and the subsequent user response, if any. @@ -261,7 +265,10 @@ export class ViolationOfExpectationsChain const retrievedDocs = await this.retrieveRelevantInsights( predictionResponse.insights ); - const relevantDocs = [...predictionResponse.insights, ...retrievedDocs]; + const relevantDocs = this.removeDuplicateStrings([ + ...predictionResponse.insights, + ...retrievedDocs, + ]); return { ...predictionResponse, @@ -440,11 +447,11 @@ export class ViolationOfExpectationsChain * Static method that creates a ViolationOfExpectationsChain instance from a * ChatOpenAI and retriever. It also accepts optional options * to customize the chain. - * + * * @param llm The ChatOpenAI instance. * @param retriever The retriever used for similarity search. * @param options Optional options to customize the chain. - * + * * @returns A new instance of ViolationOfExpectationsChain. */ static fromLLM( From 8b73975c08be9e097699d8e5e6b32c2f6db0585d Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Tue, 17 Oct 2023 16:45:40 -0700 Subject: [PATCH 16/23] nit: rename test .init -> .int --- ...n.init.test.ts => violation_of_expectations_chain.int.test.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename langchain/src/chains/tests/{violation_of_expectations_chain.init.test.ts => violation_of_expectations_chain.int.test.ts} (100%) diff --git a/langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts b/langchain/src/chains/tests/violation_of_expectations_chain.int.test.ts similarity index 100% rename from langchain/src/chains/tests/violation_of_expectations_chain.init.test.ts rename to langchain/src/chains/tests/violation_of_expectations_chain.int.test.ts From 8c0a677477870f725b88dbb7f43c06b4dab2950a Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 18 Oct 2023 12:04:58 -0700 Subject: [PATCH 17/23] chore: type return values --- langchain/src/chains/violation_of_expectations/types.ts | 6 ++++++ .../violation_of_expectations_chain.ts | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/langchain/src/chains/violation_of_expectations/types.ts b/langchain/src/chains/violation_of_expectations/types.ts index 3db2be311f45..d8d519f71fa4 100644 --- a/langchain/src/chains/violation_of_expectations/types.ts +++ b/langchain/src/chains/violation_of_expectations/types.ts @@ -19,6 +19,12 @@ export type PredictNextUserMessageResponse = { insights: Array; }; +export type GetPredictionViolationsResponse = { + userResponse?: HumanMessage; + revisedPrediction: string; + explainedPredictionErrors: Array; +} + export const PREDICT_NEXT_USER_MESSAGE_FUNCTION = { name: "predictNextUserMessage", description: "Predicts the next user message, along with insights.", diff --git a/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts b/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts index 97992ea9d5b0..8ef9ce12844e 100644 --- a/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts +++ b/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts @@ -11,6 +11,7 @@ import { StringOutputParser } from "../../schema/output_parser.js"; import { BaseRetriever } from "../../schema/retriever.js"; import { BaseChain, ChainInputs } from "../base.js"; import { + GetPredictionViolationsResponse, MessageChunkResult, PREDICTION_VIOLATIONS_FUNCTION, PREDICT_NEXT_USER_MESSAGE_FUNCTION, @@ -214,7 +215,9 @@ export class ViolationOfExpectationsChain ) ); - return insights; + return { + insights, + }; } /** @@ -324,7 +327,7 @@ export class ViolationOfExpectationsChain userPredictions: PredictNextUserMessageResponse; userResponse?: BaseMessage; runManager?: CallbackManagerForChainRun; - }) { + }): Promise { const llmWithFunctions = this.llm.bind({ functions: [PREDICTION_VIOLATIONS_FUNCTION], function_call: { name: PREDICTION_VIOLATIONS_FUNCTION.name }, @@ -426,7 +429,7 @@ export class ViolationOfExpectationsChain explainedPredictionErrors: Array; }; runManager?: CallbackManagerForChainRun; - }) { + }): Promise { const chain = GENERATE_FACTS_PROMPT.pipe(this.llm).pipe( this.stringOutputParser ); From 7026316fde57ec48bb61a38d9ab37b9e8395887e Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 18 Oct 2023 12:06:18 -0700 Subject: [PATCH 18/23] chore: lint files --- langchain/src/chains/violation_of_expectations/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain/src/chains/violation_of_expectations/types.ts b/langchain/src/chains/violation_of_expectations/types.ts index d8d519f71fa4..92e72bd43335 100644 --- a/langchain/src/chains/violation_of_expectations/types.ts +++ b/langchain/src/chains/violation_of_expectations/types.ts @@ -23,7 +23,7 @@ export type GetPredictionViolationsResponse = { userResponse?: HumanMessage; revisedPrediction: string; explainedPredictionErrors: Array; -} +}; export const PREDICT_NEXT_USER_MESSAGE_FUNCTION = { name: "predictNextUserMessage", From 2fbd1e1726ea069d5349247f1ade1602db4e55a7 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 18 Oct 2023 16:57:31 -0700 Subject: [PATCH 19/23] refactor: move to experimental --- .../test-exports-bun/src/entrypoints.js | 2 +- .../test-exports-cf/src/entrypoints.js | 2 +- .../test-exports-cjs/src/entrypoints.js | 2 +- .../test-exports-esbuild/src/entrypoints.js | 2 +- .../test-exports-esm/src/entrypoints.js | 2 +- .../test-exports-vercel/src/entrypoints.js | 2 +- .../test-exports-vite/src/entrypoints.js | 2 +- langchain/.gitignore | 6 +++--- langchain/package.json | 16 ++++++++-------- langchain/scripts/create-entrypoints.js | 2 +- .../violation_of_expectations_chain.int.test.ts | 8 ++++---- .../chains/violation_of_expectations/index.ts | 4 ++++ .../chains/violation_of_expectations/types.ts | 2 +- .../violation_of_expectations_chain.ts | 14 +++++++------- .../violation_of_expectations_prompt.ts | 2 +- langchain/src/load/import_map.ts | 2 +- langchain/tsconfig.json | 2 +- 17 files changed, 38 insertions(+), 34 deletions(-) rename langchain/src/{ => experimental}/chains/tests/violation_of_expectations_chain.int.test.ts (81%) create mode 100644 langchain/src/experimental/chains/violation_of_expectations/index.ts rename langchain/src/{ => experimental}/chains/violation_of_expectations/types.ts (96%) rename langchain/src/{ => experimental}/chains/violation_of_expectations/violation_of_expectations_chain.ts (97%) rename langchain/src/{ => experimental}/chains/violation_of_expectations/violation_of_expectations_prompt.ts (97%) diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 4104ad67729c..9582bc8e0af1 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -6,7 +6,6 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; -export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; @@ -82,5 +81,6 @@ export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; export * from "langchain/experimental/chat_models/bittensor"; +export * from "langchain/experimental/chains/violation_of_expectations"; export * from "langchain/evaluation"; export * from "langchain/runnables/remote"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 4104ad67729c..9582bc8e0af1 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -6,7 +6,6 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; -export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; @@ -82,5 +81,6 @@ export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; export * from "langchain/experimental/chat_models/bittensor"; +export * from "langchain/experimental/chains/violation_of_expectations"; export * from "langchain/evaluation"; export * from "langchain/runnables/remote"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index 4d476e6a4124..5b0368a809c0 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -6,7 +6,6 @@ const base_language = require("langchain/base_language"); const tools = require("langchain/tools"); const chains = require("langchain/chains"); const chains_openai_functions = require("langchain/chains/openai_functions"); -const chains_violation_of_expectations = require("langchain/chains/violation_of_expectations"); const embeddings_base = require("langchain/embeddings/base"); const embeddings_cache_backed = require("langchain/embeddings/cache_backed"); const embeddings_fake = require("langchain/embeddings/fake"); @@ -82,5 +81,6 @@ const experimental_babyagi = require("langchain/experimental/babyagi"); const experimental_generative_agents = require("langchain/experimental/generative_agents"); const experimental_plan_and_execute = require("langchain/experimental/plan_and_execute"); const experimental_chat_models_bittensor = require("langchain/experimental/chat_models/bittensor"); +const experimental_chains_violation_of_expectations = require("langchain/experimental/chains/violation_of_expectations"); const evaluation = require("langchain/evaluation"); const runnables_remote = require("langchain/runnables/remote"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index 953f2a262a98..5c228b2a898e 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -6,7 +6,6 @@ import * as base_language from "langchain/base_language"; import * as tools from "langchain/tools"; import * as chains from "langchain/chains"; import * as chains_openai_functions from "langchain/chains/openai_functions"; -import * as chains_violation_of_expectations from "langchain/chains/violation_of_expectations"; import * as embeddings_base from "langchain/embeddings/base"; import * as embeddings_cache_backed from "langchain/embeddings/cache_backed"; import * as embeddings_fake from "langchain/embeddings/fake"; @@ -82,5 +81,6 @@ import * as experimental_babyagi from "langchain/experimental/babyagi"; import * as experimental_generative_agents from "langchain/experimental/generative_agents"; import * as experimental_plan_and_execute from "langchain/experimental/plan_and_execute"; import * as experimental_chat_models_bittensor from "langchain/experimental/chat_models/bittensor"; +import * as experimental_chains_violation_of_expectations from "langchain/experimental/chains/violation_of_expectations"; import * as evaluation from "langchain/evaluation"; import * as runnables_remote from "langchain/runnables/remote"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index 953f2a262a98..5c228b2a898e 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -6,7 +6,6 @@ import * as base_language from "langchain/base_language"; import * as tools from "langchain/tools"; import * as chains from "langchain/chains"; import * as chains_openai_functions from "langchain/chains/openai_functions"; -import * as chains_violation_of_expectations from "langchain/chains/violation_of_expectations"; import * as embeddings_base from "langchain/embeddings/base"; import * as embeddings_cache_backed from "langchain/embeddings/cache_backed"; import * as embeddings_fake from "langchain/embeddings/fake"; @@ -82,5 +81,6 @@ import * as experimental_babyagi from "langchain/experimental/babyagi"; import * as experimental_generative_agents from "langchain/experimental/generative_agents"; import * as experimental_plan_and_execute from "langchain/experimental/plan_and_execute"; import * as experimental_chat_models_bittensor from "langchain/experimental/chat_models/bittensor"; +import * as experimental_chains_violation_of_expectations from "langchain/experimental/chains/violation_of_expectations"; import * as evaluation from "langchain/evaluation"; import * as runnables_remote from "langchain/runnables/remote"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 4104ad67729c..9582bc8e0af1 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -6,7 +6,6 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; -export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; @@ -82,5 +81,6 @@ export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; export * from "langchain/experimental/chat_models/bittensor"; +export * from "langchain/experimental/chains/violation_of_expectations"; export * from "langchain/evaluation"; export * from "langchain/runnables/remote"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 4104ad67729c..9582bc8e0af1 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -6,7 +6,6 @@ export * from "langchain/base_language"; export * from "langchain/tools"; export * from "langchain/chains"; export * from "langchain/chains/openai_functions"; -export * from "langchain/chains/violation_of_expectations"; export * from "langchain/embeddings/base"; export * from "langchain/embeddings/cache_backed"; export * from "langchain/embeddings/fake"; @@ -82,5 +81,6 @@ export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; export * from "langchain/experimental/chat_models/bittensor"; +export * from "langchain/experimental/chains/violation_of_expectations"; export * from "langchain/evaluation"; export * from "langchain/runnables/remote"; diff --git a/langchain/.gitignore b/langchain/.gitignore index 36edf0c0ccaa..761f453d4e7f 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -64,9 +64,6 @@ chains/sql_db.d.ts chains/graph_qa/cypher.cjs chains/graph_qa/cypher.js chains/graph_qa/cypher.d.ts -chains/violation_of_expectations.cjs -chains/violation_of_expectations.js -chains/violation_of_expectations.d.ts embeddings/base.cjs embeddings/base.js embeddings/base.d.ts @@ -691,6 +688,9 @@ experimental/llms/bittensor.d.ts experimental/hubs/makersuite/googlemakersuitehub.cjs experimental/hubs/makersuite/googlemakersuitehub.js experimental/hubs/makersuite/googlemakersuitehub.d.ts +experimental/chains/violation_of_expectations.cjs +experimental/chains/violation_of_expectations.js +experimental/chains/violation_of_expectations.d.ts evaluation.cjs evaluation.js evaluation.d.ts diff --git a/langchain/package.json b/langchain/package.json index ed83ae7fe956..a2b4af5950e5 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -76,9 +76,6 @@ "chains/graph_qa/cypher.cjs", "chains/graph_qa/cypher.js", "chains/graph_qa/cypher.d.ts", - "chains/violation_of_expectations.cjs", - "chains/violation_of_expectations.js", - "chains/violation_of_expectations.d.ts", "embeddings/base.cjs", "embeddings/base.js", "embeddings/base.d.ts", @@ -703,6 +700,9 @@ "experimental/hubs/makersuite/googlemakersuitehub.cjs", "experimental/hubs/makersuite/googlemakersuitehub.js", "experimental/hubs/makersuite/googlemakersuitehub.d.ts", + "experimental/chains/violation_of_expectations.cjs", + "experimental/chains/violation_of_expectations.js", + "experimental/chains/violation_of_expectations.d.ts", "evaluation.cjs", "evaluation.js", "evaluation.d.ts", @@ -1406,11 +1406,6 @@ "import": "./chains/graph_qa/cypher.js", "require": "./chains/graph_qa/cypher.cjs" }, - "./chains/violation_of_expectations": { - "types": "./chains/violation_of_expectations.d.ts", - "import": "./chains/violation_of_expectations.js", - "require": "./chains/violation_of_expectations.cjs" - }, "./embeddings/base": { "types": "./embeddings/base.d.ts", "import": "./embeddings/base.js", @@ -2451,6 +2446,11 @@ "import": "./experimental/hubs/makersuite/googlemakersuitehub.js", "require": "./experimental/hubs/makersuite/googlemakersuitehub.cjs" }, + "./experimental/chains/violation_of_expectations": { + "types": "./experimental/chains/violation_of_expectations.d.ts", + "import": "./experimental/chains/violation_of_expectations.js", + "require": "./experimental/chains/violation_of_expectations.cjs" + }, "./evaluation": { "types": "./evaluation.d.ts", "import": "./evaluation.js", diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index a8ce9554d6a2..c463f1bba84f 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -34,7 +34,6 @@ const entrypoints = { "chains/query_constructor/ir": "chains/query_constructor/ir", "chains/sql_db": "chains/sql_db/index", "chains/graph_qa/cypher": "chains/graph_qa/cypher", - "chains/violation_of_expectations": "chains/violation_of_expectations/violation_of_expectations_chain", // embeddings "embeddings/base": "embeddings/base", "embeddings/bedrock": "embeddings/bedrock", @@ -276,6 +275,7 @@ const entrypoints = { "experimental/llms/bittensor": "experimental/llms/bittensor", "experimental/hubs/makersuite/googlemakersuitehub": "experimental/hubs/makersuite/googlemakersuitehub", + "experimental/chains/violation_of_expectations": "experimental/chains/violation_of_expectations/index", // evaluation evaluation: "evaluation/index", // runnables diff --git a/langchain/src/chains/tests/violation_of_expectations_chain.int.test.ts b/langchain/src/experimental/chains/tests/violation_of_expectations_chain.int.test.ts similarity index 81% rename from langchain/src/chains/tests/violation_of_expectations_chain.int.test.ts rename to langchain/src/experimental/chains/tests/violation_of_expectations_chain.int.test.ts index 931310ae39a1..0ece5872b07e 100644 --- a/langchain/src/chains/tests/violation_of_expectations_chain.int.test.ts +++ b/langchain/src/experimental/chains/tests/violation_of_expectations_chain.int.test.ts @@ -1,7 +1,7 @@ -import { ChatOpenAI } from "../../chat_models/openai.js"; -import { OpenAIEmbeddings } from "../../embeddings/openai.js"; -import { AIMessage, HumanMessage } from "../../schema/index.js"; -import { HNSWLib } from "../../vectorstores/hnswlib.js"; +import { ChatOpenAI } from "../../../chat_models/openai.js"; +import { OpenAIEmbeddings } from "../../../embeddings/openai.js"; +import { AIMessage, HumanMessage } from "../../../schema/index.js"; +import { HNSWLib } from "../../../vectorstores/hnswlib.js"; import { ViolationOfExpectationsChain } from "../violation_of_expectations/violation_of_expectations_chain.js"; const dummyMessages = [ diff --git a/langchain/src/experimental/chains/violation_of_expectations/index.ts b/langchain/src/experimental/chains/violation_of_expectations/index.ts new file mode 100644 index 000000000000..2c5b18343fb5 --- /dev/null +++ b/langchain/src/experimental/chains/violation_of_expectations/index.ts @@ -0,0 +1,4 @@ +export { + type ViolationOfExpectationsChainInput, + ViolationOfExpectationsChain, +} from "./violation_of_expectations_chain.js"; diff --git a/langchain/src/chains/violation_of_expectations/types.ts b/langchain/src/experimental/chains/violation_of_expectations/types.ts similarity index 96% rename from langchain/src/chains/violation_of_expectations/types.ts rename to langchain/src/experimental/chains/violation_of_expectations/types.ts index 92e72bd43335..1dffcb5d6274 100644 --- a/langchain/src/chains/violation_of_expectations/types.ts +++ b/langchain/src/experimental/chains/violation_of_expectations/types.ts @@ -1,4 +1,4 @@ -import { BaseMessage, HumanMessage } from "../../schema/index.js"; +import { BaseMessage, HumanMessage } from "../../../schema/index.js"; /** * Contains the chunk of messages, along with the diff --git a/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts similarity index 97% rename from langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts rename to langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts index 8ef9ce12844e..d761d4f82adc 100644 --- a/langchain/src/chains/violation_of_expectations/violation_of_expectations_chain.ts +++ b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts @@ -1,15 +1,15 @@ -import { CallbackManagerForChainRun } from "../../callbacks/manager.js"; -import { ChatOpenAI } from "../../chat_models/openai.js"; -import { JsonOutputFunctionsParser } from "../../output_parsers/openai_functions.js"; +import { CallbackManagerForChainRun } from "../../../callbacks/manager.js"; +import { ChatOpenAI } from "../../../chat_models/openai.js"; +import { JsonOutputFunctionsParser } from "../../../output_parsers/openai_functions.js"; import { BaseMessage, ChainValues, HumanMessage, isBaseMessage, -} from "../../schema/index.js"; -import { StringOutputParser } from "../../schema/output_parser.js"; -import { BaseRetriever } from "../../schema/retriever.js"; -import { BaseChain, ChainInputs } from "../base.js"; +} from "../../../schema/index.js"; +import { StringOutputParser } from "../../../schema/output_parser.js"; +import { BaseRetriever } from "../../../schema/retriever.js"; +import { BaseChain, ChainInputs } from "../../../chains/base.js"; import { GetPredictionViolationsResponse, MessageChunkResult, diff --git a/langchain/src/chains/violation_of_expectations/violation_of_expectations_prompt.ts b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_prompt.ts similarity index 97% rename from langchain/src/chains/violation_of_expectations/violation_of_expectations_prompt.ts rename to langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_prompt.ts index 78544b2a3dc2..b5957c6c538d 100644 --- a/langchain/src/chains/violation_of_expectations/violation_of_expectations_prompt.ts +++ b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_prompt.ts @@ -1,4 +1,4 @@ -import { PromptTemplate } from "../../prompts/prompt.js"; +import { PromptTemplate } from "../../../prompts/prompt.js"; export const PREDICT_NEXT_USER_MESSAGE_PROMPT = /* #__PURE__ */ PromptTemplate.fromTemplate(` diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index b679410d36fe..cfc440b134cc 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -7,7 +7,6 @@ export * as base_language from "../base_language/index.js"; export * as tools from "../tools/index.js"; export * as chains from "../chains/index.js"; export * as chains__openai_functions from "../chains/openai_functions/index.js"; -export * as chains__violation_of_expectations from "../chains/violation_of_expectations/violation_of_expectations_chain.js"; export * as embeddings__base from "../embeddings/base.js"; export * as embeddings__cache_backed from "../embeddings/cache_backed.js"; export * as embeddings__fake from "../embeddings/fake.js"; @@ -83,5 +82,6 @@ export * as experimental__babyagi from "../experimental/babyagi/index.js"; export * as experimental__generative_agents from "../experimental/generative_agents/index.js"; export * as experimental__plan_and_execute from "../experimental/plan_and_execute/index.js"; export * as experimental__chat_models__bittensor from "../experimental/chat_models/bittensor.js"; +export * as experimental__chains__violation_of_expectations from "../experimental/chains/violation_of_expectations/index.js"; export * as evaluation from "../evaluation/index.js"; export * as runnables__remote from "../runnables/remote.js"; diff --git a/langchain/tsconfig.json b/langchain/tsconfig.json index 302e0ef364c3..656c10a8eb24 100644 --- a/langchain/tsconfig.json +++ b/langchain/tsconfig.json @@ -54,7 +54,6 @@ "src/chains/query_constructor/ir.ts", "src/chains/sql_db/index.ts", "src/chains/graph_qa/cypher.ts", - "src/chains/violation_of_expectations/violation_of_expectations_chain.ts", "src/embeddings/base.ts", "src/embeddings/bedrock.ts", "src/embeddings/cache_backed.ts", @@ -263,6 +262,7 @@ "src/experimental/chat_models/bittensor.ts", "src/experimental/llms/bittensor.ts", "src/experimental/hubs/makersuite/googlemakersuitehub.ts", + "src/experimental/chains/violation_of_expectations/index.ts", "src/evaluation/index.ts", "src/runnables/remote.ts" ], From 1df10332130fce8bc5cd74afc5e78960edae3ae5 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 18 Oct 2023 17:02:36 -0700 Subject: [PATCH 20/23] feat: documentation --- .../violation_of_expectations_chain.mdx | 60 +++++++++++++++ .../violation_of_expectations_chain.ts | 77 +++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 docs/docs/use_cases/violation_of_expectations_chain.mdx create mode 100644 examples/src/use_cases/advanced/violation_of_expectations_chain.ts diff --git a/docs/docs/use_cases/violation_of_expectations_chain.mdx b/docs/docs/use_cases/violation_of_expectations_chain.mdx new file mode 100644 index 000000000000..76a81ddc1c61 --- /dev/null +++ b/docs/docs/use_cases/violation_of_expectations_chain.mdx @@ -0,0 +1,60 @@ +# Violation of Expectations Chain + +This page demonstrates how to use the `ViolationOfExpectationsChain`, and scenarios where it can be extremely powerful for extracting insights on chat conversations. +The example provided will use a chat between a human and an AI, talking about a journal entry the user made. + +The `ViolationOfExpectationsChain` was built using the findings from a paper by [Plastic Labs](https://plasticlabs.ai/). Their paper `Violation of Expectation via Metacognitive Prompting Reduces +Theory of Mind Prediction Error in Large Language Models` can be found [here](https://arxiv.org/abs/2310.06983). + +import CodeBlock from "@theme/CodeBlock"; +import ViolationOfExpectationsChainExample from "@examples/use_cases/advanced/violation_of_expectations_chain.ts"; + + + {ViolationOfExpectationsChainExample} + + +Now let's go over everything the chaim is doing, step by step. + +Under the hood, the `ViolationOfExpectationsChain` performs four main steps: + +## Step 1. Predict the user's next message using only the chat history. + +The LLM is tasked with generating three key pieces of information: + +- Concise reasoning about the users internal mental state. +- A prediction on how they will respond to the AI's most recent message. +- A concise list of any additional insights that would be useful to improve prediction. + Once the LLM response is returned, we query our retriever with the insights, mapping over all. + From each result we extract the first retrieved document, and return it. + Then, all retrieved documents and generated insights are sorted to remove duplicates, and returned. + +## Step 2. Generate prediction violations. + +Using the results from step 1, we query the LLM to generate the following: + +- How exactly was the original prediction violated? Which parts were wrong? State the exact differences. +- If there were errors with the prediction, what were they and why? + We pass the LLM our predicted response and generated (along with any retrieved) insights from step 1, and the actual response from the user. + +Once we have the difference between the predicted and actual response, we can move on to step 3. + +## Step 3. Regenerate the prediction. + +Using the original prediction, key insights and the differences between the actual response and our prediction, we can generate a new more accurate prediction. +These predictions will help us in the next step to generate an insight that isn't just parts of the user's conversation verbatim. + +## Step 4. Generate an insight. + +Lastly, we prompt the LLM to generate one concise insight given the following context: + +- Ways in which our original prediction was violated. +- Our generated revised prediction (step 3) +- The actual response from the user. + Given these three data points, we prompt the LLM to return one fact relevant to the specific user response. + A key point here is giving it the ways in which our original prediction was violated. This list contains the exact differences --and often specific facts themselves-- between the predicted and actual response. + +We perform these steps on every human message, so if you have a conversation with 10 messages (5 human 5 AI), you'll get 5 insights. +The list of messages are chunked by iterating over the entire chat history, stopping at an AI message and returning it, along with all messages that preceded it. + +Once our `.call({...})` method returns the array of insights, we can save them to our vector store. +Later, we can retrieve them in future insight generations, or for other reasons like insightful context in a chat bot. \ No newline at end of file diff --git a/examples/src/use_cases/advanced/violation_of_expectations_chain.ts b/examples/src/use_cases/advanced/violation_of_expectations_chain.ts new file mode 100644 index 000000000000..6cb9eee3357e --- /dev/null +++ b/examples/src/use_cases/advanced/violation_of_expectations_chain.ts @@ -0,0 +1,77 @@ +import { ViolationOfExpectationsChain } from "langchain/experimental/chains/violation_of_expectations"; +import { ChatOpenAI } from "langchain/chat_models/openai"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { AIMessage, HumanMessage } from "langchain/schema"; +import { HNSWLib } from "langchain/vectorstores/hnswlib"; + +// Short GPT generated conversation between a human and an AI. +const dummyMessages = [ + new HumanMessage( + "I've been thinking about the importance of time with myself to discover my voice. I feel like 1-2 hours is never enough." + ), + new AIMessage( + "The concept of 'adequate time' varies. Have you tried different formats of introspection, such as morning pages or long-form writing, to see if they make the process more efficient?" + ), + new HumanMessage( + "I have tried journaling but never consistently. Sometimes it feels like writing doesn't capture everything." + ), + new AIMessage( + "Writing has its limits. What about other mediums like digital art, or interactive journal apps with dynamic prompts that dig deeper? Even coding a personal project can be a form of self-discovery." + ), + new HumanMessage( + "That's an interesting idea. I've never thought about coding as a form of self-discovery." + ), + new AIMessage( + "Since you're comfortable with code, consider building a tool to log and analyze your emotional state, thoughts, or personal growth metrics. It merges skill with introspection, makes the data quantifiable." + ), + new HumanMessage( + "The idea of quantifying emotions and personal growth is fascinating. But I wonder how much it can really capture the 'dark zone' within us." + ), + new AIMessage( + "Good point. The 'dark zone' isn't fully quantifiable. But a tool could serve as a scaffold to explore those areas. It gives a structured approach to an unstructured problem." + ), + new HumanMessage( + "You might be onto something. A structured approach could help unearth patterns or triggers I hadn't noticed." + ), + new AIMessage( + "Exactly. It's about creating a framework to understand what can't easily be understood. Then you can allocate those 5+ hours more effectively, targeting areas that your data flags." + ), +]; + +// Instantiate with an empty string to start, since we have no data yet. +const vectorStore = await HNSWLib.fromTexts( + [" "], + [{ id: 1 }], + new OpenAIEmbeddings() +); +const retriever = vectorStore.asRetriever(); + +// Instantiate the LLM, +const llm = new ChatOpenAI({ + modelName: "gpt-4", +}); + +// And the chain. +const voeChain = ViolationOfExpectationsChain.fromLLM(llm, retriever); + +// Requires an input key of "chat_history" with an array of messages. +const result = await voeChain.call({ + chat_history: dummyMessages, +}); + +console.log({ + result, +}); + +/** + * Output: +{ + result: [ + 'The user has experience with coding and has tried journaling before, but struggles with maintaining consistency and fully expressing their thoughts and feelings through writing.', + 'The user shows a thoughtful approach towards new concepts and is willing to engage with and contemplate novel ideas before making a decision. They also consider time effectiveness as a crucial factor in their decision-making process.', + 'The user is curious and open-minded about new concepts, but also values introspection and self-discovery in understanding emotions and personal growth.', + 'The user is open to new ideas and strategies, specifically those that involve a structured approach to identifying patterns or triggers.', + 'The user may not always respond or engage with prompts, indicating a need for varied and adaptable communication strategies.' + ] +} + */ From 36acea6098659d30d13ac5ca211edf84efdb5c62 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 18 Oct 2023 17:25:58 -0700 Subject: [PATCH 21/23] lint --- docs/docs/use_cases/violation_of_expectations_chain.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/use_cases/violation_of_expectations_chain.mdx b/docs/docs/use_cases/violation_of_expectations_chain.mdx index 76a81ddc1c61..d22a70a9132b 100644 --- a/docs/docs/use_cases/violation_of_expectations_chain.mdx +++ b/docs/docs/use_cases/violation_of_expectations_chain.mdx @@ -57,4 +57,4 @@ We perform these steps on every human message, so if you have a conversation wit The list of messages are chunked by iterating over the entire chat history, stopping at an AI message and returning it, along with all messages that preceded it. Once our `.call({...})` method returns the array of insights, we can save them to our vector store. -Later, we can retrieve them in future insight generations, or for other reasons like insightful context in a chat bot. \ No newline at end of file +Later, we can retrieve them in future insight generations, or for other reasons like insightful context in a chat bot. From 823bdb942fa3b25607d405e9353c7075d0be21ef Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 18 Oct 2023 17:27:35 -0700 Subject: [PATCH 22/23] nit: method rename -> chunkMessagesByAIResponse --- .../violation_of_expectations_chain.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts index d761d4f82adc..a3f4d0d8f1ce 100644 --- a/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts +++ b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts @@ -117,7 +117,7 @@ export class ViolationOfExpectationsChain * The method then pushes the chunk (sequence of messages and user response) into the result array. * This process continues until all messages in the chat history have been processed. */ - getMessageChunks(chatHistory: BaseMessage[]): MessageChunkResult[] { + chunkMessagesByAIResponse(chatHistory: BaseMessage[]): MessageChunkResult[] { const newArray: MessageChunkResult[] = []; const tempArray: BaseMessage[] = []; @@ -177,7 +177,7 @@ export class ViolationOfExpectationsChain throw new Error("Chat history must be an array of BaseMessages"); } - const messageChunks = this.getMessageChunks(chatHistory as BaseMessage[]); + const messageChunks = this.chunkMessagesByAIResponse(chatHistory as BaseMessage[]); // Generate the initial prediction for every user message. const userPredictions = await Promise.all( From 180aca6808c2419258f7bacd57ccab1f2967391b Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Wed, 18 Oct 2023 20:52:58 -0700 Subject: [PATCH 23/23] Small docs updates, run format --- .../violation_of_expectations_chain.mdx | 23 ++++++++++++------- .../violation_of_expectations_chain.ts | 4 +++- package.json | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) rename docs/docs/use_cases/{ => agent_simulations}/violation_of_expectations_chain.mdx (77%) diff --git a/docs/docs/use_cases/violation_of_expectations_chain.mdx b/docs/docs/use_cases/agent_simulations/violation_of_expectations_chain.mdx similarity index 77% rename from docs/docs/use_cases/violation_of_expectations_chain.mdx rename to docs/docs/use_cases/agent_simulations/violation_of_expectations_chain.mdx index d22a70a9132b..31226a9b7d66 100644 --- a/docs/docs/use_cases/violation_of_expectations_chain.mdx +++ b/docs/docs/use_cases/agent_simulations/violation_of_expectations_chain.mdx @@ -1,11 +1,16 @@ # Violation of Expectations Chain -This page demonstrates how to use the `ViolationOfExpectationsChain`, and scenarios where it can be extremely powerful for extracting insights on chat conversations. -The example provided will use a chat between a human and an AI, talking about a journal entry the user made. +This page demonstrates how to use the `ViolationOfExpectationsChain`. This chain extracts insights from chat conversations +by comparing the differences between an LLM's prediction of the next message in a conversation and the user's mental state against the actual next message, +and is intended to provide a form of reflection for long-term memory. -The `ViolationOfExpectationsChain` was built using the findings from a paper by [Plastic Labs](https://plasticlabs.ai/). Their paper `Violation of Expectation via Metacognitive Prompting Reduces +The `ViolationOfExpectationsChain` was implemented using the results of a paper by [Plastic Labs](https://plasticlabs.ai/). Their paper, `Violation of Expectation via Metacognitive Prompting Reduces Theory of Mind Prediction Error in Large Language Models` can be found [here](https://arxiv.org/abs/2310.06983). +## Usage + +The below example features a chat between a human and an AI, talking about a journal entry the user made. + import CodeBlock from "@theme/CodeBlock"; import ViolationOfExpectationsChainExample from "@examples/use_cases/advanced/violation_of_expectations_chain.ts"; @@ -13,11 +18,13 @@ import ViolationOfExpectationsChainExample from "@examples/use_cases/advanced/vi {ViolationOfExpectationsChainExample} -Now let's go over everything the chaim is doing, step by step. +## Explanation + +Now let's go over everything the chain is doing, step by step. Under the hood, the `ViolationOfExpectationsChain` performs four main steps: -## Step 1. Predict the user's next message using only the chat history. +### Step 1. Predict the user's next message using only the chat history. The LLM is tasked with generating three key pieces of information: @@ -28,7 +35,7 @@ The LLM is tasked with generating three key pieces of information: From each result we extract the first retrieved document, and return it. Then, all retrieved documents and generated insights are sorted to remove duplicates, and returned. -## Step 2. Generate prediction violations. +### Step 2. Generate prediction violations. Using the results from step 1, we query the LLM to generate the following: @@ -38,12 +45,12 @@ Using the results from step 1, we query the LLM to generate the following: Once we have the difference between the predicted and actual response, we can move on to step 3. -## Step 3. Regenerate the prediction. +### Step 3. Regenerate the prediction. Using the original prediction, key insights and the differences between the actual response and our prediction, we can generate a new more accurate prediction. These predictions will help us in the next step to generate an insight that isn't just parts of the user's conversation verbatim. -## Step 4. Generate an insight. +### Step 4. Generate an insight. Lastly, we prompt the LLM to generate one concise insight given the following context: diff --git a/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts index a3f4d0d8f1ce..a4777e06c244 100644 --- a/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts +++ b/langchain/src/experimental/chains/violation_of_expectations/violation_of_expectations_chain.ts @@ -177,7 +177,9 @@ export class ViolationOfExpectationsChain throw new Error("Chat history must be an array of BaseMessages"); } - const messageChunks = this.chunkMessagesByAIResponse(chatHistory as BaseMessage[]); + const messageChunks = this.chunkMessagesByAIResponse( + chatHistory as BaseMessage[] + ); // Generate the initial prediction for every user message. const userPredictions = await Promise.all( diff --git a/package.json b/package.json index 6da3b1490609..46fcb548bfba 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "publish": "bash langchain/scripts/release-branch.sh && turbo run --filter langchain build lint test && yarn run test:exports:docker && yarn workspace langchain run release && echo '🔗 Open https://github.com/hwchase17/langchainjs/compare/release?expand=1 and merge the release PR'", "example": "yarn workspace examples start", "precommit": "turbo run precommit", - "docs": "yarn workspace docs dev", + "docs": "yarn workspace docs start", "postinstall": "husky install" }, "author": "LangChain",