From 62275b1f83b49f24ffe764d6198fa7e76d4a3d1c Mon Sep 17 00:00:00 2001 From: Jamsheed Mistri Date: Mon, 20 May 2024 17:28:35 -0700 Subject: [PATCH] community[minor]: feat: Layerup Security integration (#4929) * feat: Layerup Security integration * nit: remove environment variable reference * docs: Layerup Security docs * fix: package.json, fix docs location * feat: bump layerup-security version, support untrusted_input * nit: LLM -> BaseLLM * Update lock * Build and lint fixes * fix: bump layerup-security SDK version to 1.5.9 * Update lockfile * feat: bump SDK to 1.5.10 to match OpenAI requirement * Bump version * fix: type errors * Bump dep * Rename test --------- Co-authored-by: jacoblee93 --- .../integrations/llms/layerup_security.mdx | 31 ++++ examples/package.json | 1 + examples/src/llms/layerup_security.ts | 59 ++++++ libs/langchain-community/langchain.config.js | 2 + libs/langchain-community/package.json | 5 + .../src/llms/layerup_security.ts | 169 ++++++++++++++++++ .../src/llms/tests/layerup_security.test.ts | 48 +++++ .../src/load/import_constants.ts | 1 + yarn.lock | 33 ++++ 9 files changed, 349 insertions(+) create mode 100644 docs/core_docs/docs/integrations/llms/layerup_security.mdx create mode 100644 examples/src/llms/layerup_security.ts create mode 100644 libs/langchain-community/src/llms/layerup_security.ts create mode 100644 libs/langchain-community/src/llms/tests/layerup_security.test.ts diff --git a/docs/core_docs/docs/integrations/llms/layerup_security.mdx b/docs/core_docs/docs/integrations/llms/layerup_security.mdx new file mode 100644 index 000000000000..983509419177 --- /dev/null +++ b/docs/core_docs/docs/integrations/llms/layerup_security.mdx @@ -0,0 +1,31 @@ +import CodeBlock from "@theme/CodeBlock"; + +# Layerup Security + +The [Layerup Security](https://uselayerup.com) integration allows you to secure your calls to any LangChain LLM, LLM chain or LLM agent. The LLM object wraps around any existing LLM object, allowing for a secure layer between your users and your LLMs. + +While the Layerup Security object is designed as an LLM, it is not actually an LLM itself, it simply wraps around an LLM, allowing it to adapt the same functionality as the underlying LLM. + +## Setup + +First, you'll need a Layerup Security account from the Layerup [website](https://uselayerup.com). + +Next, create a project via the [dashboard](https://dashboard.uselayerup.com), and copy your API key. We recommend putting your API key in your project's environment. + +Install the Layerup Security SDK: + +```bash npm2yarn +npm install @layerup/layerup-security +``` + +And install LangChain Community: + +```bash npm2yarn +npm install @langchain/community +``` + +And now you're ready to start protecting your LLM calls with Layerup Security! + +import LayerupSecurityExampleCode from "@examples/llms/layerup_security.ts"; + +{LayerupSecurityExampleCode} diff --git a/examples/package.json b/examples/package.json index 1965d68e12f8..4943b290fe74 100644 --- a/examples/package.json +++ b/examples/package.json @@ -54,6 +54,7 @@ "@langchain/textsplitters": "workspace:*", "@langchain/weaviate": "workspace:*", "@langchain/yandex": "workspace:*", + "@layerup/layerup-security": "^1.5.12", "@opensearch-project/opensearch": "^2.2.0", "@pinecone-database/pinecone": "^2.2.0", "@planetscale/database": "^1.8.0", diff --git a/examples/src/llms/layerup_security.ts b/examples/src/llms/layerup_security.ts new file mode 100644 index 000000000000..b77cf72883b1 --- /dev/null +++ b/examples/src/llms/layerup_security.ts @@ -0,0 +1,59 @@ +import { + LayerupSecurity, + LayerupSecurityOptions, +} from "@langchain/community/llms/layerup_security"; +import { GuardrailResponse } from "@layerup/layerup-security"; +import { OpenAI } from "@langchain/openai"; + +// Create an instance of your favorite LLM +const openai = new OpenAI({ + modelName: "gpt-3.5-turbo", + openAIApiKey: process.env.OPENAI_API_KEY, +}); + +// Configure Layerup Security +const layerupSecurityOptions: LayerupSecurityOptions = { + // Specify a LLM that Layerup Security will wrap around + llm: openai, + + // Layerup API key, from the Layerup dashboard + layerupApiKey: process.env.LAYERUP_API_KEY, + + // Custom base URL, if self hosting + layerupApiBaseUrl: "https://api.uselayerup.com/v1", + + // List of guardrails to run on prompts before the LLM is invoked + promptGuardrails: [], + + // List of guardrails to run on responses from the LLM + responseGuardrails: ["layerup.hallucination"], + + // Whether or not to mask the prompt for PII & sensitive data before it is sent to the LLM + mask: false, + + // Metadata for abuse tracking, customer tracking, and scope tracking. + metadata: { customer: "example@uselayerup.com" }, + + // Handler for guardrail violations on the response guardrails + handlePromptGuardrailViolation: (violation: GuardrailResponse) => { + if (violation.offending_guardrail === "layerup.sensitive_data") { + // Custom logic goes here + } + + return { + role: "assistant", + content: `There was sensitive data! I cannot respond. Here's a dynamic canned response. Current date: ${Date.now()}`, + }; + }, + + // Handler for guardrail violations on the response guardrails + handleResponseGuardrailViolation: (violation: GuardrailResponse) => ({ + role: "assistant", + content: `Custom canned response with dynamic data! The violation rule was ${violation.offending_guardrail}.`, + }), +}; + +const layerupSecurity = new LayerupSecurity(layerupSecurityOptions); +const response = await layerupSecurity.invoke( + "Summarize this message: my name is Bob Dylan. My SSN is 123-45-6789." +); diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index d87dc490eed8..c6a2bb7d488f 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -106,6 +106,7 @@ export const config = { "llms/watsonx_ai": "llms/watsonx_ai", "llms/writer": "llms/writer", "llms/yandex": "llms/yandex", + "llms/layerup_security": "llms/layerup_security", // vectorstores "vectorstores/analyticdb": "vectorstores/analyticdb", "vectorstores/astradb": "vectorstores/astradb", @@ -340,6 +341,7 @@ export const config = { "llms/llama_cpp", "llms/writer", "llms/portkey", + "llms/layerup_security", "vectorstores/analyticdb", "vectorstores/astradb", "vectorstores/azure_aisearch", diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 8c9fc51c1c76..d5b0dd9a26e7 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -82,6 +82,7 @@ "@huggingface/inference": "^2.6.4", "@jest/globals": "^29.5.0", "@langchain/scripts": "~0.0", + "@layerup/layerup-security": "^1.5.12", "@mendable/firecrawl-js": "^0.0.13", "@mlc-ai/web-llm": "^0.2.35", "@mozilla/readability": "^0.4.4", @@ -236,6 +237,7 @@ "@google-cloud/storage": "^6.10.1 || ^7.7.0", "@gradientai/nodejs-sdk": "^1.2.0", "@huggingface/inference": "^2.6.4", + "@layerup/layerup-security": "^1.5.12", "@mendable/firecrawl-js": "^0.0.13", "@mlc-ai/web-llm": "^0.2.35", "@mozilla/readability": "*", @@ -404,6 +406,9 @@ "@huggingface/inference": { "optional": true }, + "@layerup/layerup-security": { + "optional": true + }, "@mendable/firecrawl-js": { "optional": true }, diff --git a/libs/langchain-community/src/llms/layerup_security.ts b/libs/langchain-community/src/llms/layerup_security.ts new file mode 100644 index 000000000000..e60676094892 --- /dev/null +++ b/libs/langchain-community/src/llms/layerup_security.ts @@ -0,0 +1,169 @@ +import { + LLM, + BaseLLM, + type BaseLLMParams, +} from "@langchain/core/language_models/llms"; +import { + GuardrailResponse, + LayerupSecurity as LayerupSecuritySDK, + LLMMessage, +} from "@layerup/layerup-security"; + +export interface LayerupSecurityOptions extends BaseLLMParams { + llm: BaseLLM; + layerupApiKey?: string; + layerupApiBaseUrl?: string; + promptGuardrails?: string[]; + responseGuardrails?: string[]; + mask?: boolean; + metadata?: Record; + handlePromptGuardrailViolation?: (violation: GuardrailResponse) => LLMMessage; + handleResponseGuardrailViolation?: ( + violation: GuardrailResponse + ) => LLMMessage; +} + +function defaultGuardrailViolationHandler( + violation: GuardrailResponse +): LLMMessage { + if (violation.canned_response) return violation.canned_response; + + const guardrailName = violation.offending_guardrail + ? `Guardrail ${violation.offending_guardrail}` + : "A guardrail"; + throw new Error( + `${guardrailName} was violated without a proper guardrail violation handler.` + ); +} + +export class LayerupSecurity extends LLM { + static lc_name() { + return "LayerupSecurity"; + } + + lc_serializable = true; + + llm: BaseLLM; + + layerupApiKey: string; + + layerupApiBaseUrl = "https://api.uselayerup.com/v1"; + + promptGuardrails: string[] = []; + + responseGuardrails: string[] = []; + + mask = false; + + metadata: Record = {}; + + handlePromptGuardrailViolation: (violation: GuardrailResponse) => LLMMessage = + defaultGuardrailViolationHandler; + + handleResponseGuardrailViolation: ( + violation: GuardrailResponse + ) => LLMMessage = defaultGuardrailViolationHandler; + + private layerup: LayerupSecuritySDK; + + constructor(options: LayerupSecurityOptions) { + super(options); + + if (!options.llm) { + throw new Error("Layerup Security requires an LLM to be provided."); + } else if (!options.layerupApiKey) { + throw new Error("Layerup Security requires an API key to be provided."); + } + + this.llm = options.llm; + this.layerupApiKey = options.layerupApiKey; + this.layerupApiBaseUrl = + options.layerupApiBaseUrl || this.layerupApiBaseUrl; + this.promptGuardrails = options.promptGuardrails || this.promptGuardrails; + this.responseGuardrails = + options.responseGuardrails || this.responseGuardrails; + this.mask = options.mask || this.mask; + this.metadata = options.metadata || this.metadata; + this.handlePromptGuardrailViolation = + options.handlePromptGuardrailViolation || + this.handlePromptGuardrailViolation; + this.handleResponseGuardrailViolation = + options.handleResponseGuardrailViolation || + this.handleResponseGuardrailViolation; + + this.layerup = new LayerupSecuritySDK({ + apiKey: this.layerupApiKey, + baseURL: this.layerupApiBaseUrl, + }); + } + + _llmType() { + return "layerup_security"; + } + + async _call(input: string, options?: BaseLLMParams): Promise { + // Since LangChain LLMs only support string inputs, we will wrap each call to Layerup in a single-message + // array of messages, then extract the string element when we need to access it. + let messages: LLMMessage[] = [ + { + role: "user", + content: input, + }, + ]; + let unmaskResponse; + + if (this.mask) { + [messages, unmaskResponse] = await this.layerup.maskPrompt( + messages, + this.metadata + ); + } + + if (this.promptGuardrails.length > 0) { + const securityResponse = await this.layerup.executeGuardrails( + this.promptGuardrails, + messages, + input, + this.metadata + ); + + // If there is a guardrail violation, extract the canned response and reply with that instead + if (!securityResponse.all_safe) { + const replacedResponse: LLMMessage = + this.handlePromptGuardrailViolation(securityResponse); + return replacedResponse.content as string; + } + } + + // Invoke the underlying LLM with the prompt and options + let result = await this.llm.invoke(messages[0].content as string, options); + + if (this.mask && unmaskResponse) { + result = unmaskResponse(result); + } + + // Add to messages array for response guardrail handler + messages.push({ + role: "assistant", + content: result, + }); + + if (this.responseGuardrails.length > 0) { + const securityResponse = await this.layerup.executeGuardrails( + this.responseGuardrails, + messages, + result, + this.metadata + ); + + // If there is a guardrail violation, extract the canned response and reply with that instead + if (!securityResponse.all_safe) { + const replacedResponse: LLMMessage = + this.handleResponseGuardrailViolation(securityResponse); + return replacedResponse.content as string; + } + } + + return result; + } +} diff --git a/libs/langchain-community/src/llms/tests/layerup_security.test.ts b/libs/langchain-community/src/llms/tests/layerup_security.test.ts new file mode 100644 index 000000000000..670a56ca200b --- /dev/null +++ b/libs/langchain-community/src/llms/tests/layerup_security.test.ts @@ -0,0 +1,48 @@ +import { test } from "@jest/globals"; +import { LLM, type BaseLLMParams } from "@langchain/core/language_models/llms"; +import { GuardrailResponse } from "@layerup/layerup-security/types.js"; +import { + LayerupSecurity, + LayerupSecurityOptions, +} from "../layerup_security.js"; + +// Mock LLM for testing purposes +export class MockLLM extends LLM { + static lc_name() { + return "MockLLM"; + } + + lc_serializable = true; + + _llmType() { + return "mock_llm"; + } + + async _call(_input: string, _options?: BaseLLMParams): Promise { + return "Hi Bob! How are you?"; + } +} + +test("Test LayerupSecurity with invalid API key", async () => { + const mockLLM = new MockLLM({}); + const layerupSecurityOptions: LayerupSecurityOptions = { + llm: mockLLM, + layerupApiKey: "-- invalid API key --", + layerupApiBaseUrl: "https://api.uselayerup.com/v1", + promptGuardrails: [], + responseGuardrails: ["layerup.hallucination"], + mask: false, + metadata: { customer: "example@uselayerup.com" }, + handleResponseGuardrailViolation: (violation: GuardrailResponse) => ({ + role: "assistant", + content: `Custom canned response with dynamic data! The violation rule was ${violation.offending_guardrail}.`, + }), + }; + + await expect(async () => { + const layerupSecurity = new LayerupSecurity(layerupSecurityOptions); + await layerupSecurity.invoke( + "My name is Bob Dylan. My SSN is 123-45-6789." + ); + }).rejects.toThrowError(); +}, 50000); diff --git a/libs/langchain-community/src/load/import_constants.ts b/libs/langchain-community/src/load/import_constants.ts index 5e3aba60a9ce..6154ae56146b 100644 --- a/libs/langchain-community/src/load/import_constants.ts +++ b/libs/langchain-community/src/load/import_constants.ts @@ -35,6 +35,7 @@ export const optionalImportEntrypoints: string[] = [ "langchain_community/llms/sagemaker_endpoint", "langchain_community/llms/watsonx_ai", "langchain_community/llms/writer", + "langchain_community/llms/layerup_security", "langchain_community/vectorstores/analyticdb", "langchain_community/vectorstores/astradb", "langchain_community/vectorstores/azure_aisearch", diff --git a/yarn.lock b/yarn.lock index 8ab2db295ee6..f974355c9e11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9035,6 +9035,7 @@ __metadata: "@langchain/core": ~0.2.0 "@langchain/openai": ~0.0.28 "@langchain/scripts": ~0.0 + "@layerup/layerup-security": ^1.5.12 "@mendable/firecrawl-js": ^0.0.13 "@mlc-ai/web-llm": ^0.2.35 "@mozilla/readability": ^0.4.4 @@ -9197,6 +9198,7 @@ __metadata: "@google-cloud/storage": ^6.10.1 || ^7.7.0 "@gradientai/nodejs-sdk": ^1.2.0 "@huggingface/inference": ^2.6.4 + "@layerup/layerup-security": ^1.5.12 "@mendable/firecrawl-js": ^0.0.13 "@mlc-ai/web-llm": ^0.2.35 "@mozilla/readability": "*" @@ -9339,6 +9341,8 @@ __metadata: optional: true "@huggingface/inference": optional: true + "@layerup/layerup-security": + optional: true "@mendable/firecrawl-js": optional: true "@mlc-ai/web-llm": @@ -10209,6 +10213,16 @@ __metadata: languageName: unknown linkType: soft +"@layerup/layerup-security@npm:^1.5.12": + version: 1.5.12 + resolution: "@layerup/layerup-security@npm:1.5.12" + dependencies: + axios: ^1.6.8 + openai: ^4.32.1 + checksum: f506b7aa266dcf7e062e3eebdba99468363f45f4d87419f108538f13d5cae70b6ac96f1c263d614de4ff7d742cf0ee3bd67b44124de717680bf02c69a458b5e2 + languageName: node + linkType: hard + "@leichtgewicht/ip-codec@npm:^2.0.1": version: 2.0.4 resolution: "@leichtgewicht/ip-codec@npm:2.0.4" @@ -22451,6 +22465,7 @@ __metadata: "@langchain/textsplitters": "workspace:*" "@langchain/weaviate": "workspace:*" "@langchain/yandex": "workspace:*" + "@layerup/layerup-security": ^1.5.12 "@opensearch-project/opensearch": ^2.2.0 "@pinecone-database/pinecone": ^2.2.0 "@planetscale/database": ^1.8.0 @@ -30239,6 +30254,24 @@ __metadata: languageName: node linkType: hard +"openai@npm:^4.32.1": + version: 4.47.1 + resolution: "openai@npm:4.47.1" + dependencies: + "@types/node": ^18.11.18 + "@types/node-fetch": ^2.6.4 + abort-controller: ^3.0.0 + agentkeepalive: ^4.2.1 + form-data-encoder: 1.7.2 + formdata-node: ^4.3.2 + node-fetch: ^2.6.7 + web-streams-polyfill: ^3.2.1 + bin: + openai: bin/cli + checksum: 746aa39479f7bc62b1536582bd2f068870715a3a1ae372a187ec5fd1a6cf70c6e49b94a5a8b408186ec52aca886ed47b52fccf6010993fcf16915ab11f96b3b9 + languageName: node + linkType: hard + "openai@npm:^4.41.1": version: 4.42.0 resolution: "openai@npm:4.42.0"