From ba39dbcda3566d09216736e96537835a7ee7e951 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Sun, 3 Dec 2023 15:44:26 -0800 Subject: [PATCH 1/2] Adds npx create-langchain-integration command --- libs/create-langchain-integration/LICENSE.md | 21 ++ .../create-app.ts | 54 +++++ .../helpers/copy.ts | 50 +++++ .../helpers/git.ts | 58 +++++ .../helpers/is-folder-empty.ts | 62 ++++++ .../helpers/is-url.ts | 8 + .../helpers/is-writeable.ts | 10 + .../helpers/make-dir.ts | 8 + .../helpers/templates.ts | 43 ++++ .../helpers/validate-pkg.ts | 20 ++ libs/create-langchain-integration/index.ts | 168 +++++++++++++++ .../create-langchain-integration/package.json | 29 +++ .../template/.eslintrc.cjs | 66 ++++++ .../template/.gitignore | 3 + .../template/LICENSE | 21 ++ .../template/README.md | 42 ++++ .../template/jest.config.cjs | 19 ++ .../template/jest.env.cjs | 12 ++ .../template/package.json | 76 +++++++ .../template/scripts/check-tree-shaking.js | 80 +++++++ .../template/scripts/create-entrypoints.js | 101 +++++++++ .../template/scripts/identify-secrets.js | 77 +++++++ .../template/scripts/move-cjs-to-dist.js | 38 ++++ .../template/scripts/release-branch.sh | 6 + .../template/src/chat_models.ts | 88 ++++++++ .../template/src/index.ts | 3 + .../template/src/llms.ts | 73 +++++++ .../template/src/tests/chat_models.test.ts | 5 + .../src/tests/integration.int.test.ts | 5 + .../template/src/tests/llms.test.ts | 5 + .../template/src/tests/vectorstores.test.ts | 5 + .../template/src/vectorstores.ts | 80 +++++++ .../template/tsconfig.cjs.json | 13 ++ .../template/tsconfig.json | 33 +++ .../tsconfig.json | 31 +++ yarn.lock | 201 +++++++++++++++--- 36 files changed, 1589 insertions(+), 25 deletions(-) create mode 100644 libs/create-langchain-integration/LICENSE.md create mode 100644 libs/create-langchain-integration/create-app.ts create mode 100644 libs/create-langchain-integration/helpers/copy.ts create mode 100644 libs/create-langchain-integration/helpers/git.ts create mode 100644 libs/create-langchain-integration/helpers/is-folder-empty.ts create mode 100644 libs/create-langchain-integration/helpers/is-url.ts create mode 100644 libs/create-langchain-integration/helpers/is-writeable.ts create mode 100644 libs/create-langchain-integration/helpers/make-dir.ts create mode 100644 libs/create-langchain-integration/helpers/templates.ts create mode 100644 libs/create-langchain-integration/helpers/validate-pkg.ts create mode 100644 libs/create-langchain-integration/index.ts create mode 100644 libs/create-langchain-integration/package.json create mode 100644 libs/create-langchain-integration/template/.eslintrc.cjs create mode 100644 libs/create-langchain-integration/template/.gitignore create mode 100644 libs/create-langchain-integration/template/LICENSE create mode 100644 libs/create-langchain-integration/template/README.md create mode 100644 libs/create-langchain-integration/template/jest.config.cjs create mode 100644 libs/create-langchain-integration/template/jest.env.cjs create mode 100644 libs/create-langchain-integration/template/package.json create mode 100644 libs/create-langchain-integration/template/scripts/check-tree-shaking.js create mode 100644 libs/create-langchain-integration/template/scripts/create-entrypoints.js create mode 100644 libs/create-langchain-integration/template/scripts/identify-secrets.js create mode 100644 libs/create-langchain-integration/template/scripts/move-cjs-to-dist.js create mode 100644 libs/create-langchain-integration/template/scripts/release-branch.sh create mode 100644 libs/create-langchain-integration/template/src/chat_models.ts create mode 100644 libs/create-langchain-integration/template/src/index.ts create mode 100644 libs/create-langchain-integration/template/src/llms.ts create mode 100644 libs/create-langchain-integration/template/src/tests/chat_models.test.ts create mode 100644 libs/create-langchain-integration/template/src/tests/integration.int.test.ts create mode 100644 libs/create-langchain-integration/template/src/tests/llms.test.ts create mode 100644 libs/create-langchain-integration/template/src/tests/vectorstores.test.ts create mode 100644 libs/create-langchain-integration/template/src/vectorstores.ts create mode 100644 libs/create-langchain-integration/template/tsconfig.cjs.json create mode 100644 libs/create-langchain-integration/template/tsconfig.json create mode 100644 libs/create-langchain-integration/tsconfig.json diff --git a/libs/create-langchain-integration/LICENSE.md b/libs/create-langchain-integration/LICENSE.md new file mode 100644 index 000000000000..d5c9d8189aa9 --- /dev/null +++ b/libs/create-langchain-integration/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) Harrison Chase + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/libs/create-langchain-integration/create-app.ts b/libs/create-langchain-integration/create-app.ts new file mode 100644 index 000000000000..c5456433633c --- /dev/null +++ b/libs/create-langchain-integration/create-app.ts @@ -0,0 +1,54 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import path from "path"; +import { green } from "picocolors"; +import { tryGitInit } from "./helpers/git"; +import { isFolderEmpty } from "./helpers/is-folder-empty"; +import { isWriteable } from "./helpers/is-writeable"; +import { makeDir } from "./helpers/make-dir"; + +import { installTemplate } from "./helpers/templates"; + +export type InstallAppArgs = { + appPath: string; +}; + +export async function createApp({ + appPath, +}: InstallAppArgs): Promise { + const root = path.resolve(appPath); + + if (!(await isWriteable(path.dirname(root)))) { + console.error( + "The application path is not writable, please check folder permissions and try again.", + ); + console.error( + "It is likely you do not have write permissions for this folder.", + ); + process.exit(1); + } + + const appName = path.basename(root); + + await makeDir(root); + if (!isFolderEmpty(root, appName)) { + process.exit(1); + } + + console.log(`Creating a new LangChain integration in ${green(root)}.`); + console.log(); + + await installTemplate({ root, appName }); + + process.chdir(root); + if (tryGitInit(root)) { + console.log("Initialized a git repository."); + console.log(); + } + + console.log(`${green("Success!")} Created ${appName} at ${appPath}`); + console.log(); + console.log( + `Run "cd ${appPath} to see your new integration.`, + ); + console.log(); +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/copy.ts b/libs/create-langchain-integration/helpers/copy.ts new file mode 100644 index 000000000000..a887b53d5881 --- /dev/null +++ b/libs/create-langchain-integration/helpers/copy.ts @@ -0,0 +1,50 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import { async as glob } from "fast-glob"; +import fs from "fs"; +import path from "path"; + +interface CopyOption { + cwd?: string; + rename?: (basename: string) => string; + parents?: boolean; +} + +const identity = (x: string) => x; + +export const copy = async ( + src: string | string[], + dest: string, + { cwd, rename = identity, parents = true }: CopyOption = {}, +) => { + const source = typeof src === "string" ? [src] : src; + + if (source.length === 0 || !dest) { + throw new TypeError("`src` and `dest` are required"); + } + + const sourceFiles = await glob(source, { + cwd, + dot: true, + absolute: false, + stats: false, + }); + + const destRelativeToCwd = cwd ? path.resolve(cwd, dest) : dest; + + return Promise.all( + sourceFiles.map(async (p) => { + const dirname = path.dirname(p); + const basename = rename(path.basename(p)); + + const from = cwd ? path.resolve(cwd, p) : p; + const to = parents + ? path.join(destRelativeToCwd, dirname, basename) + : path.join(destRelativeToCwd, basename); + + // Ensure the destination directory exists + await fs.promises.mkdir(path.dirname(to), { recursive: true }); + + return fs.promises.copyFile(from, to); + }), + ); +}; \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/git.ts b/libs/create-langchain-integration/helpers/git.ts new file mode 100644 index 000000000000..9d663afddcec --- /dev/null +++ b/libs/create-langchain-integration/helpers/git.ts @@ -0,0 +1,58 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import { execSync } from "child_process"; +import fs from "fs"; +import path from "path"; + +function isInGitRepository(): boolean { + try { + execSync("git rev-parse --is-inside-work-tree", { stdio: "ignore" }); + return true; + } catch (_) {} + return false; +} + +function isInMercurialRepository(): boolean { + try { + execSync("hg --cwd . root", { stdio: "ignore" }); + return true; + } catch (_) {} + return false; +} + +function isDefaultBranchSet(): boolean { + try { + execSync("git config init.defaultBranch", { stdio: "ignore" }); + return true; + } catch (_) {} + return false; +} + +export function tryGitInit(root: string): boolean { + let didInit = false; + try { + execSync("git --version", { stdio: "ignore" }); + if (isInGitRepository() || isInMercurialRepository()) { + return false; + } + + execSync("git init", { stdio: "ignore" }); + didInit = true; + + if (!isDefaultBranchSet()) { + execSync("git checkout -b main", { stdio: "ignore" }); + } + + execSync("git add -A", { stdio: "ignore" }); + execSync('git commit -m "Initial commit from create-langchain-integration"', { + stdio: "ignore", + }); + return true; + } catch (e) { + if (didInit) { + try { + fs.rmSync(path.join(root, ".git"), { recursive: true, force: true }); + } catch (_) {} + } + return false; + } +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/is-folder-empty.ts b/libs/create-langchain-integration/helpers/is-folder-empty.ts new file mode 100644 index 000000000000..3cd93954e38b --- /dev/null +++ b/libs/create-langchain-integration/helpers/is-folder-empty.ts @@ -0,0 +1,62 @@ +/* eslint-disable import/no-extraneous-dependencies */ +import fs from "fs"; +import path from "path"; +import { blue, green } from "picocolors"; + +export function isFolderEmpty(root: string, name: string): boolean { + const validFiles = [ + ".DS_Store", + ".git", + ".gitattributes", + ".gitignore", + ".gitlab-ci.yml", + ".hg", + ".hgcheck", + ".hgignore", + ".idea", + ".npmignore", + ".travis.yml", + "LICENSE", + "Thumbs.db", + "docs", + "mkdocs.yml", + "npm-debug.log", + "yarn-debug.log", + "yarn-error.log", + "yarnrc.yml", + ".yarn", + ]; + + const conflicts = fs + .readdirSync(root) + .filter((file) => !validFiles.includes(file)) + // Support IntelliJ IDEA-based editors + .filter((file) => !/\.iml$/.test(file)); + + if (conflicts.length > 0) { + console.log( + `The directory ${green(name)} contains files that could conflict:`, + ); + console.log(); + for (const file of conflicts) { + try { + const stats = fs.lstatSync(path.join(root, file)); + if (stats.isDirectory()) { + console.log(` ${blue(file)}/`); + } else { + console.log(` ${file}`); + } + } catch { + console.log(` ${file}`); + } + } + console.log(); + console.log( + "Either try using a new directory name, or remove the files listed above.", + ); + console.log(); + return false; + } + + return true; +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/is-url.ts b/libs/create-langchain-integration/helpers/is-url.ts new file mode 100644 index 000000000000..2d262f99964f --- /dev/null +++ b/libs/create-langchain-integration/helpers/is-url.ts @@ -0,0 +1,8 @@ +export function isUrl(url: string): boolean { + try { + new URL(url); + return true; + } catch (error) { + return false; + } +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/is-writeable.ts b/libs/create-langchain-integration/helpers/is-writeable.ts new file mode 100644 index 000000000000..4bdb9aa84bbb --- /dev/null +++ b/libs/create-langchain-integration/helpers/is-writeable.ts @@ -0,0 +1,10 @@ +import fs from "fs"; + +export async function isWriteable(directory: string): Promise { + try { + await fs.promises.access(directory, (fs.constants || fs).W_OK); + return true; + } catch (err) { + return false; + } +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/make-dir.ts b/libs/create-langchain-integration/helpers/make-dir.ts new file mode 100644 index 000000000000..29b89053619e --- /dev/null +++ b/libs/create-langchain-integration/helpers/make-dir.ts @@ -0,0 +1,8 @@ +import fs from "fs"; + +export function makeDir( + root: string, + options = { recursive: true }, +): Promise { + return fs.promises.mkdir(root, options); +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/templates.ts b/libs/create-langchain-integration/helpers/templates.ts new file mode 100644 index 000000000000..a5660b993f01 --- /dev/null +++ b/libs/create-langchain-integration/helpers/templates.ts @@ -0,0 +1,43 @@ +import path from "path"; +import fs from "fs/promises"; +import os from "os"; + +import { copy } from "./copy"; + +/** + * Install a internal template to a given `root` directory. + */ +export async function installTemplate({ + appName, + root, +}: any) { + + /** + * Copy the template files to the target directory. + */ + const templatePath = path.join(__dirname, "..", "template"); + const copySource = ["**"]; + + console.log(`Initializing project...`); + + await copy(copySource, root, { + parents: true, + cwd: templatePath, + }); + + /** + * Update the package.json scripts. + */ + const packageJsonFile = path.join(root, "package.json"); + const packageJson: any = JSON.parse( + await fs.readFile(packageJsonFile, "utf8"), + ); + packageJson.name = appName; + + await fs.writeFile( + packageJsonFile, + JSON.stringify(packageJson, null, 2) + os.EOL, + ); + + console.log("\nDone!\n"); +} \ No newline at end of file diff --git a/libs/create-langchain-integration/helpers/validate-pkg.ts b/libs/create-langchain-integration/helpers/validate-pkg.ts new file mode 100644 index 000000000000..3c001bda8efe --- /dev/null +++ b/libs/create-langchain-integration/helpers/validate-pkg.ts @@ -0,0 +1,20 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import validateProjectName from "validate-npm-package-name"; + +export function validateNpmName(name: string): { + valid: boolean; + problems?: string[]; +} { + const nameValidation = validateProjectName(name); + if (nameValidation.validForNewPackages) { + return { valid: true }; + } + + return { + valid: false, + problems: [ + ...(nameValidation.errors || []), + ...(nameValidation.warnings || []), + ], + }; +} \ No newline at end of file diff --git a/libs/create-langchain-integration/index.ts b/libs/create-langchain-integration/index.ts new file mode 100644 index 000000000000..b8adf56c817e --- /dev/null +++ b/libs/create-langchain-integration/index.ts @@ -0,0 +1,168 @@ +#!/usr/bin/env node +/* eslint-disable import/no-extraneous-dependencies */ +import Commander from "commander"; +import Conf from "conf"; +import fs from "fs"; +import path from "path"; +import { bold, cyan, green, red, yellow } from "picocolors"; +import prompts from "prompts"; +import checkForUpdate from "update-check"; +import { createApp } from "./create-app"; +import { isFolderEmpty } from "./helpers/is-folder-empty"; +import { validateNpmName } from "./helpers/validate-pkg"; +import packageJson from "./package.json"; + +let projectPath: string = ""; + +const handleSigTerm = () => process.exit(0); + +process.on("SIGINT", handleSigTerm); +process.on("SIGTERM", handleSigTerm); + +const onPromptState = (state: any) => { + if (state.aborted) { + // If we don't re-enable the terminal cursor before exiting + // the program, the cursor will remain hidden + process.stdout.write("\x1B[?25h"); + process.stdout.write("\n"); + process.exit(1); + } +}; + +const program = new Commander.Command(packageJson.name) + .version((packageJson as any).version) + .arguments("") + .usage(`${green("")} [options]`) + .action((name) => { + projectPath = name; + }) + .allowUnknownOption() + .parse(process.argv); + +const packageManager = "yarn"; + +async function run(): Promise { + const conf = new Conf({ projectName: "create-langchain-integration" }); + + if (program.resetPreferences) { + conf.clear(); + console.log(`Preferences reset successfully`); + return; + } + + if (typeof projectPath === "string") { + projectPath = projectPath.trim(); + } + + if (!projectPath) { + const res = await prompts({ + onState: onPromptState, + type: "text", + name: "path", + message: "What is your project named?", + initial: "my-langchain-integration", + validate: (name) => { + const validation = validateNpmName(path.basename(path.resolve(name))); + if (validation.valid) { + return true; + } + return "Invalid project name: " + validation.problems![0]; + }, + }); + + if (typeof res.path === "string") { + projectPath = res.path.trim(); + } + } + + if (!projectPath) { + console.log( + "\nPlease specify the project directory:\n" + + ` ${cyan(program.name())} ${green("")}\n` + + "For example:\n" + + ` ${cyan(program.name())} ${green("my-langchain-integration")}\n\n` + + `Run ${cyan(`${program.name()} --help`)} to see all options.`, + ); + process.exit(1); + } + + const resolvedProjectPath = path.resolve(projectPath); + const projectName = path.basename(resolvedProjectPath); + + const { valid, problems } = validateNpmName(projectName); + if (!valid) { + console.error( + `Could not create a project called ${red( + `"${projectName}"`, + )} because of npm naming restrictions:`, + ); + + problems!.forEach((p) => console.error(` ${red(bold("*"))} ${p}`)); + process.exit(1); + } + + /** + * Verify the project dir is empty or doesn't exist + */ + const root = path.resolve(resolvedProjectPath); + const appName = path.basename(root); + const folderExists = fs.existsSync(root); + + if (folderExists && !isFolderEmpty(root, appName)) { + process.exit(1); + } + + const preferences = (conf.get("preferences") || {}); + + await createApp({ + appPath: resolvedProjectPath, + }); + conf.set("preferences", preferences); +} + +const update = checkForUpdate(packageJson).catch(() => null); + +async function notifyUpdate(): Promise { + try { + const res = await update; + if (res?.latest) { + const updateMessage = + packageManager === "yarn" + ? "yarn global add create-langchain-integration@latest" + : packageManager === "pnpm" + ? "pnpm add -g create-langchain-integration@latest" + : "npm i -g create-langchain-integration@latest"; + + console.log( + yellow(bold("A new version of `create-langchain-integration` is available!")) + + "\n" + + "You can update by running: " + + cyan(updateMessage) + + "\n", + ); + } + process.exit(); + } catch { + // ignore error + } +} + +run() + .then(notifyUpdate) + .catch(async (reason) => { + console.log(); + console.log("Aborting installation."); + if (reason.command) { + console.log(` ${cyan(reason.command)} has failed.`); + } else { + console.log( + red("Unexpected error. Please report it as a bug:") + "\n", + reason, + ); + } + console.log(); + + await notifyUpdate(); + + process.exit(1); + }); \ No newline at end of file diff --git a/libs/create-langchain-integration/package.json b/libs/create-langchain-integration/package.json new file mode 100644 index 000000000000..ca82f5cafb3f --- /dev/null +++ b/libs/create-langchain-integration/package.json @@ -0,0 +1,29 @@ +{ + "name": "create-langchain-integration", + "version": "0.0.2", + "repository": { + "type": "git", + "url": "https://github.com/langchain-ai/langchainjs", + "directory": "libs/create-langchain-integration" + }, + "bin": "./dist/index.js", + "scripts": { + "dev": "ncc build ./index.ts -w -o dist/", + "build": "ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register", + "lint": "eslint . --ignore-pattern dist" + }, + "devDependencies": { + "@types/prompts": "^2", + "@types/validate-npm-package-name": "3.0.0", + "@vercel/ncc": "^0.34.0", + "commander": "^2.20.0", + "conf": "^10.2.0", + "dotenv": "^16.3.1", + "fast-glob": "^3.3.2", + "picocolors": "^1.0.0", + "prompts": "^2.4.2", + "typescript": "<5.2.0", + "update-check": "^1.5.4", + "validate-npm-package-name": "^5.0.0" + } +} diff --git a/libs/create-langchain-integration/template/.eslintrc.cjs b/libs/create-langchain-integration/template/.eslintrc.cjs new file mode 100644 index 000000000000..344f8a9d6cd9 --- /dev/null +++ b/libs/create-langchain-integration/template/.eslintrc.cjs @@ -0,0 +1,66 @@ +module.exports = { + extends: [ + "airbnb-base", + "eslint:recommended", + "prettier", + "plugin:@typescript-eslint/recommended", + ], + parserOptions: { + ecmaVersion: 12, + parser: "@typescript-eslint/parser", + project: "./tsconfig.json", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "no-instanceof"], + ignorePatterns: [ + ".eslintrc.cjs", + "scripts", + "node_modules", + "dist", + "dist-cjs", + "*.js", + "*.cjs", + "*.d.ts", + ], + rules: { + "no-process-env": 2, + "no-instanceof/no-instanceof": 2, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-shadow": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/no-use-before-define": ["error", "nofunc"], + "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-floating-promises": "error", + "@typescript-eslint/no-misused-promises": "error", + camelcase: 0, + "class-methods-use-this": 0, + "import/extensions": [2, "ignorePackages"], + "import/no-extraneous-dependencies": [ + "error", + { devDependencies: ["**/*.test.ts"] }, + ], + "import/no-unresolved": 0, + "import/prefer-default-export": 0, + "keyword-spacing": "error", + "max-classes-per-file": 0, + "max-len": 0, + "no-await-in-loop": 0, + "no-bitwise": 0, + "no-console": 0, + "no-restricted-syntax": 0, + "no-shadow": 0, + "no-continue": 0, + "no-void": 0, + "no-underscore-dangle": 0, + "no-use-before-define": 0, + "no-useless-constructor": 0, + "no-return-await": 0, + "consistent-return": 0, + "no-else-return": 0, + "func-names": 0, + "no-lonely-if": 0, + "prefer-rest-params": 0, + "new-cap": ["error", { properties: false, capIsNew: false }], + }, +}; diff --git a/libs/create-langchain-integration/template/.gitignore b/libs/create-langchain-integration/template/.gitignore new file mode 100644 index 000000000000..e399d336f991 --- /dev/null +++ b/libs/create-langchain-integration/template/.gitignore @@ -0,0 +1,3 @@ +node_modules +dist +.yarn \ No newline at end of file diff --git a/libs/create-langchain-integration/template/LICENSE b/libs/create-langchain-integration/template/LICENSE new file mode 100644 index 000000000000..8cd8f501eb49 --- /dev/null +++ b/libs/create-langchain-integration/template/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2023 LangChain + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/libs/create-langchain-integration/template/README.md b/libs/create-langchain-integration/template/README.md new file mode 100644 index 000000000000..4c35f7533d21 --- /dev/null +++ b/libs/create-langchain-integration/template/README.md @@ -0,0 +1,42 @@ +# langchainjs-integration-template + +Template repo for [LangChain.js](https://github.com/langchain-ai/langchainjs) integration packages. + +## Usage + +Source code lives in the `src/` directory. + +By default, there is a single entrypoint under `langchain_integration/src/index.ts` that you should re-export all relevant classes and functions from. +If you want to add more entrypoints, add them in `langchain_integration/scripts/create-entrypoints.js`. + +To build your source code, run the following commands: + +```bash +$ yarn build +``` + +The build process will automatically create build artifacts for both ESM and CJS. + +Note that because of the way compiled files are created and ignored, the `.gitignore` will be dynamically generated by `scripts/create-entrypoints.js` whenever you run `yarn build`. +This means that if you want to add more ignored files, you'll need to add them under `DEFAULT_GITIGNORE_PATHS` in that script. + +After running `yarn build`, publish a new version with: + +```bash +$ npm publish +``` + +Prettier and ESLint are also configured: + +```bash +$ yarn format +$ yarn lint +``` + +As well as Jest for testing. Test files should live within a `tests/` file in the `src/` folder. Unit tests should end in `.test.ts` and integration tests should +end in `.int.test.ts`: + +```bash +$ yarn test +$ yarn test:int +``` \ No newline at end of file diff --git a/libs/create-langchain-integration/template/jest.config.cjs b/libs/create-langchain-integration/template/jest.config.cjs new file mode 100644 index 000000000000..65bec1efdaf8 --- /dev/null +++ b/libs/create-langchain-integration/template/jest.config.cjs @@ -0,0 +1,19 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest/presets/default-esm", + testEnvironment: "./jest.env.cjs", + modulePathIgnorePatterns: ["dist/", "docs/"], + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, + transform: { + '^.+\\.tsx?$': ['@swc/jest'], + }, + transformIgnorePatterns: [ + "/node_modules/", + "\\.pnp\\.[^\\/]+$", + "./scripts/jest-setup-after-env.js", + ], + setupFiles: ["dotenv/config"], + testTimeout: 20_000, +}; diff --git a/libs/create-langchain-integration/template/jest.env.cjs b/libs/create-langchain-integration/template/jest.env.cjs new file mode 100644 index 000000000000..2ccedccb8672 --- /dev/null +++ b/libs/create-langchain-integration/template/jest.env.cjs @@ -0,0 +1,12 @@ +const { TestEnvironment } = require("jest-environment-node"); + +class AdjustedTestEnvironmentToSupportFloat32Array extends TestEnvironment { + constructor(config, context) { + // Make `instanceof Float32Array` return true in tests + // to avoid https://github.com/xenova/transformers.js/issues/57 and https://github.com/jestjs/jest/issues/2549 + super(config, context); + this.global.Float32Array = Float32Array; + } +} + +module.exports = AdjustedTestEnvironmentToSupportFloat32Array; diff --git a/libs/create-langchain-integration/template/package.json b/libs/create-langchain-integration/template/package.json new file mode 100644 index 000000000000..cafe065c68ac --- /dev/null +++ b/libs/create-langchain-integration/template/package.json @@ -0,0 +1,76 @@ +{ + "name": "langchain-integration", + "version": "0.0.0", + "description": "Sample integration for LangChain.js", + "type": "module", + "engines": { + "node": ">=18" + }, + "main": "./index.js", + "types": "./index.d.ts", + "repository": { + "type": "git", + "url": "git@github.com:langchain-ai/langchainjs.git" + }, + "scripts": { + "build": "yarn clean && yarn build:esm && yarn build:cjs && yarn build:scripts", + "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", + "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && node scripts/move-cjs-to-dist.js && rm -rf dist-cjs", + "build:watch": "node scripts/create-entrypoints.js && tsc --outDir dist/ --watch", + "build:scripts": "node scripts/create-entrypoints.js && node scripts/check-tree-shaking.js", + "lint": "NODE_OPTIONS=--max-old-space-size=4096 eslint src && dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", + "lint:fix": "yarn lint --fix", + "clean": "rm -rf dist/ && NODE_OPTIONS=--max-old-space-size=4096 node scripts/create-entrypoints.js pre", + "prepack": "yarn build", + "release": "release-it --only-version --config .release-it.json", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "format": "prettier --write \"src\"", + "format:check": "prettier --check \"src\"" + }, + "author": "LangChain", + "license": "MIT", + "dependencies": { + "@langchain/core": "~0.0.6" + }, + "devDependencies": { + "@jest/globals": "^29.5.0", + "@swc/core": "^1.3.90", + "@swc/jest": "^0.2.29", + "@tsconfig/recommended": "^1.0.3", + "@typescript-eslint/eslint-plugin": "^6.12.0", + "@typescript-eslint/parser": "^6.12.0", + "dpdm": "^3.12.0", + "eslint": "^8.33.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-config-prettier": "^8.6.0", + "eslint-plugin-import": "^2.27.5", + "eslint-plugin-no-instanceof": "^1.0.1", + "eslint-plugin-prettier": "^4.2.1", + "jest": "^29.5.0", + "jest-environment-node": "^29.6.4", + "prettier": "^2.8.3", + "rollup": "^4.5.2", + "ts-jest": "^29.1.0", + "typescript": "<5.2.0" + }, + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./index.js", + "require": "./index.cjs" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/", + "index.cjs", + "index.js", + "index.d.ts" + ] +} diff --git a/libs/create-langchain-integration/template/scripts/check-tree-shaking.js b/libs/create-langchain-integration/template/scripts/check-tree-shaking.js new file mode 100644 index 000000000000..8073e3d5507b --- /dev/null +++ b/libs/create-langchain-integration/template/scripts/check-tree-shaking.js @@ -0,0 +1,80 @@ +import fs from "fs/promises"; +import { rollup } from "rollup"; + +const packageJson = JSON.parse(await fs.readFile("package.json", "utf-8")); + +export function listEntrypoints() { + const exports = packageJson.exports; + const entrypoints = []; + + for (const [key, value] of Object.entries(exports)) { + if (key === "./package.json") { + continue; + } + if (typeof value === "string") { + entrypoints.push(value); + } else if (typeof value === "object" && value.import) { + entrypoints.push(value.import); + } + } + + return entrypoints; +} + +export function listExternals() { + return [ + ...Object.keys(packageJson.dependencies), + ...Object.keys(packageJson.peerDependencies ?? {}), + /node\:/, + /@langchain\/core\//, + ]; +} + +export async function checkTreeShaking() { + const externals = listExternals(); + const entrypoints = listEntrypoints(); + const consoleLog = console.log; + const reportMap = new Map(); + + for (const entrypoint of entrypoints) { + let sideEffects = ""; + + console.log = function (...args) { + const line = args.length ? args.join(" ") : ""; + if (line.trim().startsWith("First side effect in")) { + sideEffects += line + "\n"; + } + }; + + await rollup({ + external: externals, + input: entrypoint, + experimentalLogSideEffects: true, + }); + + reportMap.set(entrypoint, { + log: sideEffects, + hasSideEffects: sideEffects.length > 0, + }); + } + + console.log = consoleLog; + + let failed = false; + for (const [entrypoint, report] of reportMap) { + if (report.hasSideEffects) { + failed = true; + console.log("---------------------------------"); + console.log(`Tree shaking failed for ${entrypoint}`); + console.log(report.log); + } + } + + if (failed) { + process.exit(1); + } else { + console.log("Tree shaking checks passed!"); + } +} + +checkTreeShaking(); diff --git a/libs/create-langchain-integration/template/scripts/create-entrypoints.js b/libs/create-langchain-integration/template/scripts/create-entrypoints.js new file mode 100644 index 000000000000..634b63f9c342 --- /dev/null +++ b/libs/create-langchain-integration/template/scripts/create-entrypoints.js @@ -0,0 +1,101 @@ +import * as fs from "fs"; +import * as path from "path"; + +// .gitignore +const DEFAULT_GITIGNORE_PATHS = [ + "node_modules", + "dist", + ".yarn", +] + +// This lists all the entrypoints for the library. Each key corresponds to an +// importable path, eg. `import { AgentExecutor } from "langchain/agents"`. +// The value is the path to the file in `src/` that exports the entrypoint. +// This is used to generate the `exports` field in package.json. +// Order is not important. +const entrypoints = { + "index": "index" +}; + +// Entrypoints in this list require an optional dependency to be installed. +// Therefore they are not tested in the generated test-exports-* packages. +const requiresOptionalDependency = []; + +const updateJsonFile = (relativePath, updateFunction) => { + const contents = fs.readFileSync(relativePath).toString(); + const res = updateFunction(JSON.parse(contents)); + fs.writeFileSync(relativePath, JSON.stringify(res, null, 2) + "\n"); +}; + +const generateFiles = () => { + const files = [...Object.entries(entrypoints), ["index", "index"]].flatMap( + ([key, value]) => { + const nrOfDots = key.split("/").length - 1; + const relativePath = "../".repeat(nrOfDots) || "./"; + const compiledPath = `${relativePath}dist/${value}.js`; + return [ + [ + `${key}.cjs`, + `module.exports = require('${relativePath}dist/${value}.cjs');`, + ], + [`${key}.js`, `export * from '${compiledPath}'`], + [`${key}.d.ts`, `export * from '${compiledPath}'`], + ]; + } + ); + + return Object.fromEntries(files); +}; + +const updateConfig = () => { + const generatedFiles = generateFiles(); + const filenames = Object.keys(generatedFiles); + + // Update package.json `exports` and `files` fields + updateJsonFile("./package.json", (json) => ({ + ...json, + exports: Object.assign( + Object.fromEntries( + [...Object.keys(entrypoints)].map((key) => { + let entryPoint = { + types: `./${key}.d.ts`, + import: `./${key}.js`, + require: `./${key}.cjs`, + }; + + return [key === "index" ? "." : `./${key}`, entryPoint]; + }) + ), + { "./package.json": "./package.json" } + ), + files: ["dist/", ...filenames], + })); + + // Write generated files + Object.entries(generatedFiles).forEach(([filename, content]) => { + fs.mkdirSync(path.dirname(filename), { recursive: true }); + fs.writeFileSync(filename, content); + }); + + // Update .gitignore + fs.writeFileSync("./.gitignore", filenames.join("\n") + "\n" + DEFAULT_GITIGNORE_PATHS.join("\n") + "\n"); +}; + +const cleanGenerated = () => { + const filenames = Object.keys(generateFiles()); + filenames.forEach((fname) => { + try { + fs.unlinkSync(fname); + } catch { + // ignore error + } + }); +}; + +const command = process.argv[2]; + +if (command === "pre") { + cleanGenerated(); +} else { + updateConfig(); +} diff --git a/libs/create-langchain-integration/template/scripts/identify-secrets.js b/libs/create-langchain-integration/template/scripts/identify-secrets.js new file mode 100644 index 000000000000..c54bdd97c870 --- /dev/null +++ b/libs/create-langchain-integration/template/scripts/identify-secrets.js @@ -0,0 +1,77 @@ +import ts from "typescript"; +import * as fs from "fs"; + +export function identifySecrets() { + const secrets = new Set(); + + const tsConfig = ts.parseJsonConfigFileContent( + ts.readJsonConfigFile("./tsconfig.json", (p) => + fs.readFileSync(p, "utf-8") + ), + ts.sys, + "./src/" + ); + + for (const fileName of tsConfig.fileNames.filter( + (fn) => !fn.endsWith("test.ts") + )) { + const sourceFile = ts.createSourceFile( + fileName, + fs.readFileSync(fileName, "utf-8"), + tsConfig.options.target, + true + ); + sourceFile.forEachChild((node) => { + switch (node.kind) { + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.ClassExpression: { + node.forEachChild((node) => { + // look for get lc_secrets() + switch (node.kind) { + case ts.SyntaxKind.GetAccessor: { + const property = node; + if (property.name.getText() === "lc_secrets") { + // look for return { ... } + property.body.statements.forEach((stmt) => { + if ( + stmt.kind === ts.SyntaxKind.ReturnStatement && + stmt.expression.kind === + ts.SyntaxKind.ObjectLiteralExpression + ) { + // collect secret identifier + stmt.expression.properties.forEach((element) => { + if ( + element.initializer.kind === + ts.SyntaxKind.StringLiteral + ) { + const secret = element.initializer.text; + + if (secret.toUpperCase() !== secret) { + throw new Error( + `Secret identifier must be uppercase: ${secret} at ${fileName}` + ); + } + if (/\s/.test(secret)) { + throw new Error( + `Secret identifier must not contain whitespace: ${secret} at ${fileName}` + ); + } + + secrets.add(secret); + } + }); + } + }); + } + break; + } + } + }); + break; + } + } + }); + } + + return secrets; +} diff --git a/libs/create-langchain-integration/template/scripts/move-cjs-to-dist.js b/libs/create-langchain-integration/template/scripts/move-cjs-to-dist.js new file mode 100644 index 000000000000..1e89ccca88e9 --- /dev/null +++ b/libs/create-langchain-integration/template/scripts/move-cjs-to-dist.js @@ -0,0 +1,38 @@ +import { resolve, dirname, parse, format } from "node:path"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +function abs(relativePath) { + return resolve(dirname(fileURLToPath(import.meta.url)), relativePath); +} + +async function moveAndRename(source, dest) { + for (const file of await readdir(abs(source), { withFileTypes: true })) { + if (file.isDirectory()) { + await moveAndRename(`${source}/${file.name}`, `${dest}/${file.name}`); + } else if (file.isFile()) { + const parsed = parse(file.name); + + // Ignore anything that's not a .js file + if (parsed.ext !== ".js") { + continue; + } + + // Rewrite any require statements to use .cjs + const content = await readFile(abs(`${source}/${file.name}`), "utf8"); + const rewritten = content.replace(/require\("(\..+?).js"\)/g, (_, p1) => { + return `require("${p1}.cjs")`; + }); + + // Rename the file to .cjs + const renamed = format({ name: parsed.name, ext: ".cjs" }); + + await writeFile(abs(`${dest}/${renamed}`), rewritten, "utf8"); + } + } +} + +moveAndRename("../dist-cjs", "../dist").catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/libs/create-langchain-integration/template/scripts/release-branch.sh b/libs/create-langchain-integration/template/scripts/release-branch.sh new file mode 100644 index 000000000000..7504238c5561 --- /dev/null +++ b/libs/create-langchain-integration/template/scripts/release-branch.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +if [[ $(git branch --show-current) == "main" ]]; then + git checkout -B release + git push -u origin release +fi diff --git a/libs/create-langchain-integration/template/src/chat_models.ts b/libs/create-langchain-integration/template/src/chat_models.ts new file mode 100644 index 000000000000..81683647a0bd --- /dev/null +++ b/libs/create-langchain-integration/template/src/chat_models.ts @@ -0,0 +1,88 @@ +import { type BaseMessage } from "@langchain/core/messages"; +import { type BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; + +import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; +import { + type BaseChatModelParams, + SimpleChatModel, +} from "@langchain/core/language_models/chat_models"; + +// Uncomment if implementing streaming + +// import { +// ChatGenerationChunk, +// } from "@langchain/core/outputs"; +// import { +// AIMessageChunk, +// } from "@langchain/core/messages"; + +/** + * Input to chat model class. + */ +export interface ChatIntegrationInput extends BaseChatModelParams {} + +/** + * Integration with a chat model. + */ +export class ChatIntegration< + CallOptions extends BaseLanguageModelCallOptions = BaseLanguageModelCallOptions + > + extends SimpleChatModel + implements ChatIntegrationInput +{ + // Used for tracing, replace with the same name as your class + static lc_name() { + return "ChatIntegration"; + } + + lc_serializable = true; + + constructor(fields?: ChatIntegrationInput) { + super(fields ?? {}); + } + + // Replace + _llmType() { + return "chat_integration"; + } + + /** + * For some given input messages and options, return a string output. + */ + _call( + _messages: BaseMessage[], + _options: this["ParsedCallOptions"], + _runManager?: CallbackManagerForLLMRun + ): Promise { + throw new Error("Not implemented."); + } + + /** + * Implement to support streaming. + * Should yield chunks iteratively. + */ + // async *_streamResponseChunks( + // messages: BaseMessage[], + // options: this["ParsedCallOptions"], + // runManager?: CallbackManagerForLLMRun + // ): AsyncGenerator { + // // All models have a built in `this.caller` property for retries + // const stream = await this.caller.call(async () => + // createStreamMethod() + // ); + // for await (const chunk of stream) { + // if (!chunk.done) { + // yield new ChatGenerationChunk({ + // text: chunk.response, + // message: new AIMessageChunk({ content: chunk.response }), + // }); + // await runManager?.handleLLMNewToken(chunk.response ?? ""); + // } + // } + // } + + /** @ignore */ + _combineLLMOutput() { + return []; + } +} diff --git a/libs/create-langchain-integration/template/src/index.ts b/libs/create-langchain-integration/template/src/index.ts new file mode 100644 index 000000000000..564fb4a3c181 --- /dev/null +++ b/libs/create-langchain-integration/template/src/index.ts @@ -0,0 +1,3 @@ +export * from "./chat_models.js"; +export * from "./llms.js"; +export * from "./vectorstores.js"; diff --git a/libs/create-langchain-integration/template/src/llms.ts b/libs/create-langchain-integration/template/src/llms.ts new file mode 100644 index 000000000000..c3ede8d1f29b --- /dev/null +++ b/libs/create-langchain-integration/template/src/llms.ts @@ -0,0 +1,73 @@ +import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; +import { LLM, type BaseLLMParams } from "@langchain/core/language_models/llms"; +import { type BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; + +// Uncomment if implementing streaming + +// import { +// GenerationChunk, +// } from "@langchain/core/outputs"; + +/** + * Input to LLM class. + */ +export interface LLMIntegrationInput extends BaseLLMParams {} + +/** + * Integration with an LLM. + */ +export class LLMIntegration + extends LLM + implements LLMIntegrationInput +{ + // Used for tracing, replace with the same name as your class + static lc_name() { + return "LLMIntegration"; + } + + lc_serializable = true; + + constructor(fields: LLMIntegrationInput) { + super(fields); + } + + // Replace + _llmType() { + return "llm_integration"; + } + + /** + * For some given input string and options, return a string output. + */ + async _call( + _prompt: string, + _options: this["ParsedCallOptions"], + _runManager?: CallbackManagerForLLMRun + ): Promise { + throw new Error("Not implemented."); + } + + /** + * Implement to support streaming. + * Should yield chunks iteratively. + */ + // async *_streamResponseChunks( + // prompt: string, + // options: this["ParsedCallOptions"], + // runManager?: CallbackManagerForLLMRun + // ): AsyncGenerator { + // const stream = await this.caller.call(async () => + // createStream() + // ); + // for await (const chunk of stream) { + // yield new GenerationChunk({ + // text: chunk.response, + // generationInfo: { + // ...chunk, + // response: undefined, + // }, + // }); + // await runManager?.handleLLMNewToken(chunk.response ?? ""); + // } + // } +} diff --git a/libs/create-langchain-integration/template/src/tests/chat_models.test.ts b/libs/create-langchain-integration/template/src/tests/chat_models.test.ts new file mode 100644 index 000000000000..5d609f496501 --- /dev/null +++ b/libs/create-langchain-integration/template/src/tests/chat_models.test.ts @@ -0,0 +1,5 @@ +import { test } from "@jest/globals"; + +test("Test chat model", async () => { + // Your test here +}); diff --git a/libs/create-langchain-integration/template/src/tests/integration.int.test.ts b/libs/create-langchain-integration/template/src/tests/integration.int.test.ts new file mode 100644 index 000000000000..7fce4ce53302 --- /dev/null +++ b/libs/create-langchain-integration/template/src/tests/integration.int.test.ts @@ -0,0 +1,5 @@ +import { test } from "@jest/globals"; + +test("Test chat model", async () => { + // Your integration test here +}); diff --git a/libs/create-langchain-integration/template/src/tests/llms.test.ts b/libs/create-langchain-integration/template/src/tests/llms.test.ts new file mode 100644 index 000000000000..3428ecaaf599 --- /dev/null +++ b/libs/create-langchain-integration/template/src/tests/llms.test.ts @@ -0,0 +1,5 @@ +import { test } from "@jest/globals"; + +test("Test LLM", async () => { + // Your test here +}); diff --git a/libs/create-langchain-integration/template/src/tests/vectorstores.test.ts b/libs/create-langchain-integration/template/src/tests/vectorstores.test.ts new file mode 100644 index 000000000000..023cfbd8b77c --- /dev/null +++ b/libs/create-langchain-integration/template/src/tests/vectorstores.test.ts @@ -0,0 +1,5 @@ +import { test } from "@jest/globals"; + +test("Test vectorstore", async () => { + // Your test here +}); diff --git a/libs/create-langchain-integration/template/src/vectorstores.ts b/libs/create-langchain-integration/template/src/vectorstores.ts new file mode 100644 index 000000000000..27c83543801b --- /dev/null +++ b/libs/create-langchain-integration/template/src/vectorstores.ts @@ -0,0 +1,80 @@ +import { Embeddings } from "@langchain/core/embeddings"; +import { VectorStore } from "@langchain/core/vectorstores"; +import { Document } from "@langchain/core/documents"; + +/** + * Database config for your vectorstore. + */ +export interface VectorstoreIntegrationParams {} + +/** + * Class for managing and operating vector search applications with + * Tigris, an open-source Serverless NoSQL Database and Search Platform. + */ +export class VectorstoreIntegration extends VectorStore { + // Replace + _vectorstoreType(): string { + return "vectorstore_integration"; + } + + constructor(embeddings: Embeddings, params: VectorstoreIntegrationParams) { + super(embeddings, params); + this.embeddings = embeddings; + } + + /** + * Method to add an array of documents to the vectorstore. + * + * Useful to override in case your vectorstore doesn't work directly with embeddings. + */ + async addDocuments( + documents: Document[], + options?: { ids?: string[] } | string[] + ): Promise { + const texts = documents.map(({ pageContent }) => pageContent); + await this.addVectors( + await this.embeddings.embedDocuments(texts), + documents, + options + ); + } + + /** + * Method to add raw vectors to the vectorstore. + */ + async addVectors( + _vectors: number[][], + _documents: Document[], + _options?: { ids?: string[] } | string[] + ) { + throw new Error("Not implemented."); + } + + /** + * Method to perform a similarity search over the vectorstore and return + * the k most similar vectors along with their similarity scores. + */ + async similaritySearchVectorWithScore( + _query: number[], + _k: number, + _filter?: object + ): Promise<[Document, number][]> { + throw new Error("Not implemented."); + } + + /** + * Static method to create a new instance of the vectorstore from an + * array of Document instances. + * + * Other common static initializer names are fromExistingIndex, initialize, and fromTexts. + */ + static async fromDocuments( + docs: Document[], + embeddings: Embeddings, + dbConfig: VectorstoreIntegrationParams + ): Promise { + const instance = new this(embeddings, dbConfig); + await instance.addDocuments(docs); + return instance; + } +} diff --git a/libs/create-langchain-integration/template/tsconfig.cjs.json b/libs/create-langchain-integration/template/tsconfig.cjs.json new file mode 100644 index 000000000000..83f5d513ef0f --- /dev/null +++ b/libs/create-langchain-integration/template/tsconfig.cjs.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "declaration": false + }, + "exclude": [ + "node_modules", + "dist", + "docs", + "**/tests" + ] +} \ No newline at end of file diff --git a/libs/create-langchain-integration/template/tsconfig.json b/libs/create-langchain-integration/template/tsconfig.json new file mode 100644 index 000000000000..ffc49dde54a6 --- /dev/null +++ b/libs/create-langchain-integration/template/tsconfig.json @@ -0,0 +1,33 @@ +{ + "extends": "@tsconfig/recommended", + "compilerOptions": { + "outDir": "../dist", + "rootDir": "./src", + "target": "ES2021", + "lib": [ + "ES2021", + "ES2022.Object", + "DOM" + ], + "module": "ES2020", + "moduleResolution": "nodenext", + "esModuleInterop": true, + "declaration": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useDefineForClassFields": true, + "strictPropertyInitialization": false, + "allowJs": true, + "strict": true + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist", + "docs" + ] +} diff --git a/libs/create-langchain-integration/tsconfig.json b/libs/create-langchain-integration/tsconfig.json new file mode 100644 index 000000000000..bbf3ce815068 --- /dev/null +++ b/libs/create-langchain-integration/tsconfig.json @@ -0,0 +1,31 @@ +{ + "extends": "@tsconfig/recommended", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./", + "target": "ES2021", + "lib": [ + "ES2021", + "ES2022.Object", + "DOM" + ], + "module": "ES2020", + "moduleResolution": "nodenext", + "esModuleInterop": true, + "declaration": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useDefineForClassFields": true, + "strictPropertyInitialization": false, + "allowJs": true, + "strict": true, + "resolveJsonModule": true + }, + "exclude": [ + "node_modules", + "dist", + "template" + ] +} diff --git a/yarn.lock b/yarn.lock index 4a4b0fe8b086..56a4e15ca49b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11727,6 +11727,16 @@ __metadata: languageName: node linkType: hard +"@types/prompts@npm:^2": + version: 2.4.9 + resolution: "@types/prompts@npm:2.4.9" + dependencies: + "@types/node": "*" + kleur: ^3.0.3 + checksum: 69b8372f4c790b45fea16a46ff8d1bcc71b14579481776b67bd6263637118a7ecb1f12e1311506c29fadc81bf618dc64f1a91f903cfd5be67a0455a227b3e462 + languageName: node + linkType: hard + "@types/prop-types@npm:*": version: 15.7.5 resolution: "@types/prop-types@npm:15.7.5" @@ -11979,6 +11989,13 @@ __metadata: languageName: node linkType: hard +"@types/validate-npm-package-name@npm:3.0.0": + version: 3.0.0 + resolution: "@types/validate-npm-package-name@npm:3.0.0" + checksum: 5de86612ecb6a3f87d22a5cf36b0657d8d0b583b1015040abcf9786bbe9b422a034b88ef07be472cad147c21e11e6dcc0e7c9fe425f331b5a24fd3369233dd33 + languageName: node + linkType: hard + "@types/webgl-ext@npm:0.0.30": version: 0.0.30 resolution: "@types/webgl-ext@npm:0.0.30" @@ -12457,6 +12474,17 @@ __metadata: languageName: node linkType: hard +"@vercel/ncc@npm:^0.34.0": + version: 0.34.0 + resolution: "@vercel/ncc@npm:0.34.0" + dependencies: + node-gyp: latest + bin: + ncc: dist/ncc/cli.js + checksum: 3282578e990572759454f7f8d7ef025f2ae060b261fcb28a96d165350e3665438dc8fb72caf122081d6c57277e4899dab30ea0d8759bfd90baf37286cb887597 + languageName: node + linkType: hard + "@vercel/postgres@npm:^0.5.0": version: 0.5.0 resolution: "@vercel/postgres@npm:0.5.0" @@ -12893,7 +12921,7 @@ __metadata: languageName: node linkType: hard -"ajv@npm:^8.0.0, ajv@npm:^8.12.0, ajv@npm:^8.9.0": +"ajv@npm:^8.0.0, ajv@npm:^8.12.0, ajv@npm:^8.6.3, ajv@npm:^8.9.0": version: 8.12.0 resolution: "ajv@npm:8.12.0" dependencies: @@ -13451,6 +13479,13 @@ __metadata: languageName: node linkType: hard +"atomically@npm:^1.7.0": + version: 1.7.0 + resolution: "atomically@npm:1.7.0" + checksum: 991153b17334597f93b58e831bea9851e57ed9cd41d8f33991be063f170b5cc8ec7ff8605f3eb95c1d389c2ad651039e9eb8f2b795e24833c2ceb944f347373a + languageName: node + linkType: hard + "autoprefixer@npm:^10.0.1, autoprefixer@npm:^10.4.12, autoprefixer@npm:^10.4.7": version: 10.4.16 resolution: "autoprefixer@npm:10.4.16" @@ -14161,6 +14196,15 @@ __metadata: languageName: node linkType: hard +"builtins@npm:^5.0.0": + version: 5.0.1 + resolution: "builtins@npm:5.0.1" + dependencies: + semver: ^7.0.0 + checksum: 66d204657fe36522822a95b288943ad11b58f5eaede235b11d8c4edaa28ce4800087d44a2681524c340494aadb120a0068011acabe99d30e8f11a7d826d83515 + languageName: node + linkType: hard + "bundle-name@npm:^3.0.0": version: 3.0.0 resolution: "bundle-name@npm:3.0.0" @@ -15217,6 +15261,24 @@ __metadata: languageName: node linkType: hard +"conf@npm:^10.2.0": + version: 10.2.0 + resolution: "conf@npm:10.2.0" + dependencies: + ajv: ^8.6.3 + ajv-formats: ^2.1.1 + atomically: ^1.7.0 + debounce-fn: ^4.0.0 + dot-prop: ^6.0.1 + env-paths: ^2.2.1 + json-schema-typed: ^7.0.3 + onetime: ^5.1.2 + pkg-up: ^3.1.0 + semver: ^7.3.5 + checksum: 27066f38a25411c1e72e81a5219e2c7ed675cd39d8aa2a2f1797bb2c9255725e92e335d639334177a23d488b22b1290bbe0708e9a005574e5d83d5432df72bd3 + languageName: node + linkType: hard + "config-chain@npm:^1.1.11": version: 1.1.13 resolution: "config-chain@npm:1.1.13" @@ -15514,6 +15576,27 @@ __metadata: languageName: node linkType: hard +"create-langchain-integration@workspace:libs/create-langchain-integration": + version: 0.0.0-use.local + resolution: "create-langchain-integration@workspace:libs/create-langchain-integration" + dependencies: + "@types/prompts": ^2 + "@types/validate-npm-package-name": 3.0.0 + "@vercel/ncc": ^0.34.0 + commander: ^2.20.0 + conf: ^10.2.0 + dotenv: ^16.3.1 + fast-glob: ^3.3.2 + picocolors: ^1.0.0 + prompts: ^2.4.2 + typescript: <5.2.0 + update-check: ^1.5.4 + validate-npm-package-name: ^5.0.0 + bin: + create-langchain-integration: ./dist/index.js + languageName: unknown + linkType: soft + "crlf-normalize@npm:^1.0.18": version: 1.0.18 resolution: "crlf-normalize@npm:1.0.18" @@ -15872,6 +15955,15 @@ __metadata: languageName: node linkType: hard +"debounce-fn@npm:^4.0.0": + version: 4.0.0 + resolution: "debounce-fn@npm:4.0.0" + dependencies: + mimic-fn: ^3.0.0 + checksum: 7bf8d142b46a88453bbd6eda083f303049b4c8554af5114bdadfc2da56031030664360e81211ae08b708775e6904db7e6d72a421c4ff473344f4521c2c5e4a22 + languageName: node + linkType: hard + "debug@npm:2.6.9, debug@npm:^2.2.0, debug@npm:^2.6.0": version: 2.6.9 resolution: "debug@npm:2.6.9" @@ -16518,6 +16610,13 @@ __metadata: languageName: node linkType: hard +"dotenv@npm:^16.3.1": + version: 16.3.1 + resolution: "dotenv@npm:16.3.1" + checksum: 15d75e7279018f4bafd0ee9706593dd14455ddb71b3bcba9c52574460b7ccaf67d5cf8b2c08a5af1a9da6db36c956a04a1192b101ee102a3e0cf8817bbcf3dfd + languageName: node + linkType: hard + "dpdm@npm:3.12.0": version: 3.12.0 resolution: "dpdm@npm:3.12.0" @@ -16761,7 +16860,7 @@ __metadata: languageName: node linkType: hard -"env-paths@npm:^2.2.0": +"env-paths@npm:^2.2.0, env-paths@npm:^2.2.1": version: 2.2.1 resolution: "env-paths@npm:2.2.1" checksum: 65b5df55a8bab92229ab2b40dad3b387fad24613263d103a97f91c9fe43ceb21965cd3392b1ccb5d77088021e525c4e0481adb309625d0cb94ade1d1fb8dc17e @@ -18313,7 +18412,7 @@ __metadata: languageName: node linkType: hard -"fast-glob@npm:^3.2.12": +"fast-glob@npm:^3.2.12, fast-glob@npm:^3.3.2": version: 3.3.2 resolution: "fast-glob@npm:3.3.2" dependencies: @@ -22353,6 +22452,13 @@ __metadata: languageName: node linkType: hard +"json-schema-typed@npm:^7.0.3": + version: 7.0.3 + resolution: "json-schema-typed@npm:7.0.3" + checksum: e861b19e97e3cc2b29a429147890157827eeda16ab639a0765b935cf3e22aeb6abbba108e23aef442da806bb1f402bdff21da9c5cb30015f8007594565e110b5 + languageName: node + linkType: hard + "json-stable-stringify-without-jsonify@npm:^1.0.1": version: 1.0.1 resolution: "json-stable-stringify-without-jsonify@npm:1.0.1" @@ -24070,6 +24176,13 @@ __metadata: languageName: node linkType: hard +"mimic-fn@npm:^3.0.0": + version: 3.1.0 + resolution: "mimic-fn@npm:3.1.0" + checksum: f7b167f9115b8bbdf2c3ee55dce9149d14be9e54b237259c4bc1d8d0512ea60f25a1b323f814eb1fe8f5a541662804bcfcfff3202ca58df143edb986849d58db + languageName: node + linkType: hard + "mimic-fn@npm:^4.0.0": version: 4.0.0 resolution: "mimic-fn@npm:4.0.0" @@ -27475,7 +27588,7 @@ __metadata: languageName: node linkType: hard -"rc@npm:1.2.8, rc@npm:^1.2.7, rc@npm:^1.2.8": +"rc@npm:1.2.8, rc@npm:^1.0.1, rc@npm:^1.1.6, rc@npm:^1.2.7, rc@npm:^1.2.8": version: 1.2.8 resolution: "rc@npm:1.2.8" dependencies: @@ -27978,6 +28091,16 @@ __metadata: languageName: node linkType: hard +"registry-auth-token@npm:3.3.2": + version: 3.3.2 + resolution: "registry-auth-token@npm:3.3.2" + dependencies: + rc: ^1.1.6 + safe-buffer: ^5.0.1 + checksum: c9d7ae160a738f1fa825556e3669e6c771d2c0239ce37679f7e8646157a97d0a76464738be075002a1f754ef9bfb913b689f4bbfd5296d28f136fbf98c8c2217 + languageName: node + linkType: hard + "registry-auth-token@npm:^4.0.0": version: 4.2.2 resolution: "registry-auth-token@npm:4.2.2" @@ -27996,6 +28119,15 @@ __metadata: languageName: node linkType: hard +"registry-url@npm:3.1.0": + version: 3.1.0 + resolution: "registry-url@npm:3.1.0" + dependencies: + rc: ^1.0.1 + checksum: 6d223da41b04e1824f5faa63905c6f2e43b216589d72794111573f017352b790aef42cd1f826463062f89d804abb2027e3d9665d2a9a0426a11eedd04d470af3 + languageName: node + linkType: hard + "registry-url@npm:^5.0.0": version: 5.1.0 resolution: "registry-url@npm:5.1.0" @@ -28817,25 +28949,25 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.1.2": - version: 7.5.3 - resolution: "semver@npm:7.5.3" +"semver@npm:^7.0.0, semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.8, semver@npm:^7.5.4": + version: 7.5.4 + resolution: "semver@npm:7.5.4" dependencies: lru-cache: ^6.0.0 bin: semver: bin/semver.js - checksum: 9d58db16525e9f749ad0a696a1f27deabaa51f66e91d2fa2b0db3de3e9644e8677de3b7d7a03f4c15bc81521e0c3916d7369e0572dbde250d9bedf5194e2a8a7 + checksum: 12d8ad952fa353b0995bf180cdac205a4068b759a140e5d3c608317098b3575ac2f1e09182206bf2eb26120e1c0ed8fb92c48c592f6099680de56bb071423ca3 languageName: node linkType: hard -"semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.8, semver@npm:^7.5.4": - version: 7.5.4 - resolution: "semver@npm:7.5.4" +"semver@npm:^7.1.2": + version: 7.5.3 + resolution: "semver@npm:7.5.3" dependencies: lru-cache: ^6.0.0 bin: semver: bin/semver.js - checksum: 12d8ad952fa353b0995bf180cdac205a4068b759a140e5d3c608317098b3575ac2f1e09182206bf2eb26120e1c0ed8fb92c48c592f6099680de56bb071423ca3 + checksum: 9d58db16525e9f749ad0a696a1f27deabaa51f66e91d2fa2b0db3de3e9644e8677de3b7d7a03f4c15bc81521e0c3916d7369e0572dbde250d9bedf5194e2a8a7 languageName: node linkType: hard @@ -30915,6 +31047,16 @@ __metadata: languageName: node linkType: hard +"typescript@npm:<5.2.0, typescript@npm:^5.1.6": + version: 5.1.6 + resolution: "typescript@npm:5.1.6" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: b2f2c35096035fe1f5facd1e38922ccb8558996331405eb00a5111cc948b2e733163cc22fab5db46992aba7dd520fff637f2c1df4996ff0e134e77d3249a7350 + languageName: node + linkType: hard + "typescript@npm:^4.9.4": version: 4.9.5 resolution: "typescript@npm:4.9.5" @@ -30945,13 +31087,13 @@ __metadata: languageName: node linkType: hard -"typescript@npm:^5.1.6": +"typescript@patch:typescript@<5.2.0#~builtin, typescript@patch:typescript@^5.1.6#~builtin": version: 5.1.6 - resolution: "typescript@npm:5.1.6" + resolution: "typescript@patch:typescript@npm%3A5.1.6#~builtin::version=5.1.6&hash=1f5320" bin: tsc: bin/tsc tsserver: bin/tsserver - checksum: b2f2c35096035fe1f5facd1e38922ccb8558996331405eb00a5111cc948b2e733163cc22fab5db46992aba7dd520fff637f2c1df4996ff0e134e77d3249a7350 + checksum: 21e88b0a0c0226f9cb9fd25b9626fb05b4c0f3fddac521844a13e1f30beb8f14e90bd409a9ac43c812c5946d714d6e0dee12d5d02dfc1c562c5aacfa1f49b606 languageName: node linkType: hard @@ -30985,16 +31127,6 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@^5.1.6#~builtin": - version: 5.1.6 - resolution: "typescript@patch:typescript@npm%3A5.1.6#~builtin::version=5.1.6&hash=1f5320" - bin: - tsc: bin/tsc - tsserver: bin/tsserver - checksum: 21e88b0a0c0226f9cb9fd25b9626fb05b4c0f3fddac521844a13e1f30beb8f14e90bd409a9ac43c812c5946d714d6e0dee12d5d02dfc1c562c5aacfa1f49b606 - languageName: node - linkType: hard - "typesense@npm:^1.5.3": version: 1.5.3 resolution: "typesense@npm:1.5.3" @@ -31422,6 +31554,16 @@ __metadata: languageName: node linkType: hard +"update-check@npm:^1.5.4": + version: 1.5.4 + resolution: "update-check@npm:1.5.4" + dependencies: + registry-auth-token: 3.3.2 + registry-url: 3.1.0 + checksum: 2c9f7de6f030364c5ea02a341e5ae2dfe76da6559b32d40dd3b047b3ac0927408cf92d322c51cd8e009688210a85ccbf1eba449762a65a0d1b14f3cdf1ea5c48 + languageName: node + linkType: hard + "update-notifier@npm:6.0.2": version: 6.0.2 resolution: "update-notifier@npm:6.0.2" @@ -31682,6 +31824,15 @@ __metadata: languageName: node linkType: hard +"validate-npm-package-name@npm:^5.0.0": + version: 5.0.0 + resolution: "validate-npm-package-name@npm:5.0.0" + dependencies: + builtins: ^5.0.0 + checksum: 5342a994986199b3c28e53a8452a14b2bb5085727691ea7aa0d284a6606b127c371e0925ae99b3f1ef7cc7d2c9de75f52eb61a3d1cc45e39bca1e3a9444cbb4e + languageName: node + linkType: hard + "value-equal@npm:^1.0.1": version: 1.0.1 resolution: "value-equal@npm:1.0.1" From 9507c04baa279f0273e14cadc99c59f4ee4a4b68 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Sun, 3 Dec 2023 16:16:19 -0800 Subject: [PATCH 2/2] Format --- .../.eslintrc.cjs | 70 ++++++++++++++ .../.prettierignore | 1 + libs/create-langchain-integration/LICENSE.md | 2 +- .../create-app.ts | 14 +-- .../helpers/copy.ts | 6 +- .../helpers/git.ts | 11 ++- .../helpers/is-folder-empty.ts | 6 +- .../helpers/is-url.ts | 2 +- .../helpers/is-writeable.ts | 2 +- .../helpers/make-dir.ts | 4 +- .../helpers/templates.ts | 91 +++++++++++++++++-- .../helpers/validate-pkg.ts | 2 +- libs/create-langchain-integration/index.ts | 22 +++-- .../create-langchain-integration/package.json | 6 +- .../template/README.md | 6 +- .../template/jest.config.cjs | 2 +- .../template/package.json | 1 + .../template/scripts/create-entrypoints.js | 13 ++- .../template/tsconfig.cjs.json | 9 +- .../template/tsconfig.json | 16 +--- .../tsconfig.json | 12 +-- yarn.lock | 1 + 22 files changed, 211 insertions(+), 88 deletions(-) create mode 100644 libs/create-langchain-integration/.eslintrc.cjs create mode 100644 libs/create-langchain-integration/.prettierignore diff --git a/libs/create-langchain-integration/.eslintrc.cjs b/libs/create-langchain-integration/.eslintrc.cjs new file mode 100644 index 000000000000..bc166270e4e9 --- /dev/null +++ b/libs/create-langchain-integration/.eslintrc.cjs @@ -0,0 +1,70 @@ +module.exports = { + extends: [ + "airbnb-base", + "eslint:recommended", + "prettier", + "plugin:@typescript-eslint/recommended", + ], + parserOptions: { + ecmaVersion: 12, + parser: "@typescript-eslint/parser", + project: "./tsconfig.json", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "no-instanceof", "eslint-plugin-jest"], + ignorePatterns: [ + "src/utils/@cfworker", + "src/utils/fast-json-patch", + "src/utils/js-sha1", + ".eslintrc.cjs", + "scripts", + "node_modules", + "dist", + "dist-cjs", + "*.js", + "*.cjs", + "*.d.ts", + ], + rules: { + "no-process-env": 2, + "no-instanceof/no-instanceof": 2, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-shadow": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/no-use-before-define": ["error", "nofunc"], + "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-floating-promises": "error", + "@typescript-eslint/no-misused-promises": "error", + camelcase: 0, + "class-methods-use-this": 0, + "import/extensions": [2, "ignorePackages"], + "import/no-extraneous-dependencies": [ + "error", + { devDependencies: ["**/*.test.ts"] }, + ], + "import/no-unresolved": 0, + "import/prefer-default-export": 0, + "keyword-spacing": "error", + "max-classes-per-file": 0, + "max-len": 0, + "no-await-in-loop": 0, + "no-bitwise": 0, + "no-console": 0, + "no-restricted-syntax": 0, + "no-shadow": 0, + "no-continue": 0, + "no-void": 0, + "no-underscore-dangle": 0, + "no-use-before-define": 0, + "no-useless-constructor": 0, + "no-return-await": 0, + "consistent-return": 0, + "no-else-return": 0, + "func-names": 0, + "no-lonely-if": 0, + "prefer-rest-params": 0, + "new-cap": ["error", { properties: false, capIsNew: false }], + "jest/no-focused-tests": "error", + }, +}; diff --git a/libs/create-langchain-integration/.prettierignore b/libs/create-langchain-integration/.prettierignore new file mode 100644 index 000000000000..53c37a16608c --- /dev/null +++ b/libs/create-langchain-integration/.prettierignore @@ -0,0 +1 @@ +dist \ No newline at end of file diff --git a/libs/create-langchain-integration/LICENSE.md b/libs/create-langchain-integration/LICENSE.md index d5c9d8189aa9..ff1c8d1ed328 100644 --- a/libs/create-langchain-integration/LICENSE.md +++ b/libs/create-langchain-integration/LICENSE.md @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +THE SOFTWARE. diff --git a/libs/create-langchain-integration/create-app.ts b/libs/create-langchain-integration/create-app.ts index c5456433633c..2db3f2162646 100644 --- a/libs/create-langchain-integration/create-app.ts +++ b/libs/create-langchain-integration/create-app.ts @@ -12,17 +12,15 @@ export type InstallAppArgs = { appPath: string; }; -export async function createApp({ - appPath, -}: InstallAppArgs): Promise { +export async function createApp({ appPath }: InstallAppArgs): Promise { const root = path.resolve(appPath); if (!(await isWriteable(path.dirname(root)))) { console.error( - "The application path is not writable, please check folder permissions and try again.", + "The application path is not writable, please check folder permissions and try again." ); console.error( - "It is likely you do not have write permissions for this folder.", + "It is likely you do not have write permissions for this folder." ); process.exit(1); } @@ -47,8 +45,6 @@ export async function createApp({ console.log(`${green("Success!")} Created ${appName} at ${appPath}`); console.log(); - console.log( - `Run "cd ${appPath} to see your new integration.`, - ); + console.log(`Run "cd ${appPath} to see your new integration.`); console.log(); -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/copy.ts b/libs/create-langchain-integration/helpers/copy.ts index a887b53d5881..514868ea36d0 100644 --- a/libs/create-langchain-integration/helpers/copy.ts +++ b/libs/create-langchain-integration/helpers/copy.ts @@ -14,7 +14,7 @@ const identity = (x: string) => x; export const copy = async ( src: string | string[], dest: string, - { cwd, rename = identity, parents = true }: CopyOption = {}, + { cwd, rename = identity, parents = true }: CopyOption = {} ) => { const source = typeof src === "string" ? [src] : src; @@ -45,6 +45,6 @@ export const copy = async ( await fs.promises.mkdir(path.dirname(to), { recursive: true }); return fs.promises.copyFile(from, to); - }), + }) ); -}; \ No newline at end of file +}; diff --git a/libs/create-langchain-integration/helpers/git.ts b/libs/create-langchain-integration/helpers/git.ts index 9d663afddcec..416b381402ec 100644 --- a/libs/create-langchain-integration/helpers/git.ts +++ b/libs/create-langchain-integration/helpers/git.ts @@ -43,9 +43,12 @@ export function tryGitInit(root: string): boolean { } execSync("git add -A", { stdio: "ignore" }); - execSync('git commit -m "Initial commit from create-langchain-integration"', { - stdio: "ignore", - }); + execSync( + 'git commit -m "Initial commit from create-langchain-integration"', + { + stdio: "ignore", + } + ); return true; } catch (e) { if (didInit) { @@ -55,4 +58,4 @@ export function tryGitInit(root: string): boolean { } return false; } -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/is-folder-empty.ts b/libs/create-langchain-integration/helpers/is-folder-empty.ts index 3cd93954e38b..390e65d791ca 100644 --- a/libs/create-langchain-integration/helpers/is-folder-empty.ts +++ b/libs/create-langchain-integration/helpers/is-folder-empty.ts @@ -35,7 +35,7 @@ export function isFolderEmpty(root: string, name: string): boolean { if (conflicts.length > 0) { console.log( - `The directory ${green(name)} contains files that could conflict:`, + `The directory ${green(name)} contains files that could conflict:` ); console.log(); for (const file of conflicts) { @@ -52,11 +52,11 @@ export function isFolderEmpty(root: string, name: string): boolean { } console.log(); console.log( - "Either try using a new directory name, or remove the files listed above.", + "Either try using a new directory name, or remove the files listed above." ); console.log(); return false; } return true; -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/is-url.ts b/libs/create-langchain-integration/helpers/is-url.ts index 2d262f99964f..eb87b975252f 100644 --- a/libs/create-langchain-integration/helpers/is-url.ts +++ b/libs/create-langchain-integration/helpers/is-url.ts @@ -5,4 +5,4 @@ export function isUrl(url: string): boolean { } catch (error) { return false; } -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/is-writeable.ts b/libs/create-langchain-integration/helpers/is-writeable.ts index 4bdb9aa84bbb..fa29d60558b3 100644 --- a/libs/create-langchain-integration/helpers/is-writeable.ts +++ b/libs/create-langchain-integration/helpers/is-writeable.ts @@ -7,4 +7,4 @@ export async function isWriteable(directory: string): Promise { } catch (err) { return false; } -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/make-dir.ts b/libs/create-langchain-integration/helpers/make-dir.ts index 29b89053619e..9289754987e8 100644 --- a/libs/create-langchain-integration/helpers/make-dir.ts +++ b/libs/create-langchain-integration/helpers/make-dir.ts @@ -2,7 +2,7 @@ import fs from "fs"; export function makeDir( root: string, - options = { recursive: true }, + options = { recursive: true } ): Promise { return fs.promises.mkdir(root, options); -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/templates.ts b/libs/create-langchain-integration/helpers/templates.ts index a5660b993f01..a6924e4025ab 100644 --- a/libs/create-langchain-integration/helpers/templates.ts +++ b/libs/create-langchain-integration/helpers/templates.ts @@ -4,40 +4,111 @@ import os from "os"; import { copy } from "./copy"; +const DEFAULT_ESLINTRC = `module.exports = { + extends: [ + "airbnb-base", + "eslint:recommended", + "prettier", + "plugin:@typescript-eslint/recommended", + ], + parserOptions: { + ecmaVersion: 12, + parser: "@typescript-eslint/parser", + project: "./tsconfig.json", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "no-instanceof"], + ignorePatterns: [ + ".eslintrc.cjs", + "scripts", + "node_modules", + "dist", + "dist-cjs", + "*.js", + "*.cjs", + "*.d.ts", + ], + rules: { + "no-process-env": 2, + "no-instanceof/no-instanceof": 2, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-shadow": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/no-use-before-define": ["error", "nofunc"], + "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-floating-promises": "error", + "@typescript-eslint/no-misused-promises": "error", + camelcase: 0, + "class-methods-use-this": 0, + "import/extensions": [2, "ignorePackages"], + "import/no-extraneous-dependencies": [ + "error", + { devDependencies: ["**/*.test.ts"] }, + ], + "import/no-unresolved": 0, + "import/prefer-default-export": 0, + "keyword-spacing": "error", + "max-classes-per-file": 0, + "max-len": 0, + "no-await-in-loop": 0, + "no-bitwise": 0, + "no-console": 0, + "no-restricted-syntax": 0, + "no-shadow": 0, + "no-continue": 0, + "no-void": 0, + "no-underscore-dangle": 0, + "no-use-before-define": 0, + "no-useless-constructor": 0, + "no-return-await": 0, + "consistent-return": 0, + "no-else-return": 0, + "func-names": 0, + "no-lonely-if": 0, + "prefer-rest-params": 0, + "new-cap": ["error", { properties: false, capIsNew: false }], + }, +}; +`; + /** * Install a internal template to a given `root` directory. */ -export async function installTemplate({ - appName, - root, -}: any) { - +export async function installTemplate({ appName, root }: any) { /** * Copy the template files to the target directory. */ const templatePath = path.join(__dirname, "..", "template"); const copySource = ["**"]; - + console.log(`Initializing project...`); await copy(copySource, root, { parents: true, cwd: templatePath, }); - + /** * Update the package.json scripts. */ const packageJsonFile = path.join(root, "package.json"); const packageJson: any = JSON.parse( - await fs.readFile(packageJsonFile, "utf8"), + await fs.readFile(packageJsonFile, "utf8") ); packageJson.name = appName; await fs.writeFile( packageJsonFile, - JSON.stringify(packageJson, null, 2) + os.EOL, + JSON.stringify(packageJson, null, 2) + os.EOL ); + await fs.writeFile( + path.join(root, ".gitignore"), + ["node_modules", "dist", ".yarn"].join("\n") + os.EOL + ); + + await fs.writeFile(path.join(root, ".eslintrc.cjs"), DEFAULT_ESLINTRC); + console.log("\nDone!\n"); -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/helpers/validate-pkg.ts b/libs/create-langchain-integration/helpers/validate-pkg.ts index 3c001bda8efe..68317653c8e4 100644 --- a/libs/create-langchain-integration/helpers/validate-pkg.ts +++ b/libs/create-langchain-integration/helpers/validate-pkg.ts @@ -17,4 +17,4 @@ export function validateNpmName(name: string): { ...(nameValidation.warnings || []), ], }; -} \ No newline at end of file +} diff --git a/libs/create-langchain-integration/index.ts b/libs/create-langchain-integration/index.ts index b8adf56c817e..155457b7c23e 100644 --- a/libs/create-langchain-integration/index.ts +++ b/libs/create-langchain-integration/index.ts @@ -81,7 +81,7 @@ async function run(): Promise { ` ${cyan(program.name())} ${green("")}\n` + "For example:\n" + ` ${cyan(program.name())} ${green("my-langchain-integration")}\n\n` + - `Run ${cyan(`${program.name()} --help`)} to see all options.`, + `Run ${cyan(`${program.name()} --help`)} to see all options.` ); process.exit(1); } @@ -93,8 +93,8 @@ async function run(): Promise { if (!valid) { console.error( `Could not create a project called ${red( - `"${projectName}"`, - )} because of npm naming restrictions:`, + `"${projectName}"` + )} because of npm naming restrictions:` ); problems!.forEach((p) => console.error(` ${red(bold("*"))} ${p}`)); @@ -112,7 +112,7 @@ async function run(): Promise { process.exit(1); } - const preferences = (conf.get("preferences") || {}); + const preferences = conf.get("preferences") || {}; await createApp({ appPath: resolvedProjectPath, @@ -130,15 +130,17 @@ async function notifyUpdate(): Promise { packageManager === "yarn" ? "yarn global add create-langchain-integration@latest" : packageManager === "pnpm" - ? "pnpm add -g create-langchain-integration@latest" - : "npm i -g create-langchain-integration@latest"; + ? "pnpm add -g create-langchain-integration@latest" + : "npm i -g create-langchain-integration@latest"; console.log( - yellow(bold("A new version of `create-langchain-integration` is available!")) + + yellow( + bold("A new version of `create-langchain-integration` is available!") + ) + "\n" + "You can update by running: " + cyan(updateMessage) + - "\n", + "\n" ); } process.exit(); @@ -157,7 +159,7 @@ run() } else { console.log( red("Unexpected error. Please report it as a bug:") + "\n", - reason, + reason ); } console.log(); @@ -165,4 +167,4 @@ run() await notifyUpdate(); process.exit(1); - }); \ No newline at end of file + }); diff --git a/libs/create-langchain-integration/package.json b/libs/create-langchain-integration/package.json index ca82f5cafb3f..91622e85427e 100644 --- a/libs/create-langchain-integration/package.json +++ b/libs/create-langchain-integration/package.json @@ -1,6 +1,6 @@ { "name": "create-langchain-integration", - "version": "0.0.2", + "version": "0.0.3", "repository": { "type": "git", "url": "https://github.com/langchain-ai/langchainjs", @@ -10,7 +10,8 @@ "scripts": { "dev": "ncc build ./index.ts -w -o dist/", "build": "ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register", - "lint": "eslint . --ignore-pattern dist" + "format": "prettier --write \"./\"", + "format:check": "prettier --check \"./\"" }, "devDependencies": { "@types/prompts": "^2", @@ -21,6 +22,7 @@ "dotenv": "^16.3.1", "fast-glob": "^3.3.2", "picocolors": "^1.0.0", + "prettier": "^2.8.3", "prompts": "^2.4.2", "typescript": "<5.2.0", "update-check": "^1.5.4", diff --git a/libs/create-langchain-integration/template/README.md b/libs/create-langchain-integration/template/README.md index 4c35f7533d21..2df62645edfb 100644 --- a/libs/create-langchain-integration/template/README.md +++ b/libs/create-langchain-integration/template/README.md @@ -4,7 +4,7 @@ Template repo for [LangChain.js](https://github.com/langchain-ai/langchainjs) in ## Usage -Source code lives in the `src/` directory. +Source code lives in the `src/` directory. By default, there is a single entrypoint under `langchain_integration/src/index.ts` that you should re-export all relevant classes and functions from. If you want to add more entrypoints, add them in `langchain_integration/scripts/create-entrypoints.js`. @@ -13,7 +13,7 @@ To build your source code, run the following commands: ```bash $ yarn build -``` +``` The build process will automatically create build artifacts for both ESM and CJS. @@ -39,4 +39,4 @@ end in `.int.test.ts`: ```bash $ yarn test $ yarn test:int -``` \ No newline at end of file +``` diff --git a/libs/create-langchain-integration/template/jest.config.cjs b/libs/create-langchain-integration/template/jest.config.cjs index 65bec1efdaf8..5cc0b1ab72c6 100644 --- a/libs/create-langchain-integration/template/jest.config.cjs +++ b/libs/create-langchain-integration/template/jest.config.cjs @@ -7,7 +7,7 @@ module.exports = { "^(\\.{1,2}/.*)\\.js$": "$1", }, transform: { - '^.+\\.tsx?$': ['@swc/jest'], + "^.+\\.tsx?$": ["@swc/jest"], }, transformIgnorePatterns: [ "/node_modules/", diff --git a/libs/create-langchain-integration/template/package.json b/libs/create-langchain-integration/template/package.json index cafe065c68ac..be75fdabefeb 100644 --- a/libs/create-langchain-integration/template/package.json +++ b/libs/create-langchain-integration/template/package.json @@ -42,6 +42,7 @@ "@tsconfig/recommended": "^1.0.3", "@typescript-eslint/eslint-plugin": "^6.12.0", "@typescript-eslint/parser": "^6.12.0", + "dotenv": "^16.3.1", "dpdm": "^3.12.0", "eslint": "^8.33.0", "eslint-config-airbnb-base": "^15.0.0", diff --git a/libs/create-langchain-integration/template/scripts/create-entrypoints.js b/libs/create-langchain-integration/template/scripts/create-entrypoints.js index 634b63f9c342..01a4daeb25ce 100644 --- a/libs/create-langchain-integration/template/scripts/create-entrypoints.js +++ b/libs/create-langchain-integration/template/scripts/create-entrypoints.js @@ -2,11 +2,7 @@ import * as fs from "fs"; import * as path from "path"; // .gitignore -const DEFAULT_GITIGNORE_PATHS = [ - "node_modules", - "dist", - ".yarn", -] +const DEFAULT_GITIGNORE_PATHS = ["node_modules", "dist", ".yarn"]; // This lists all the entrypoints for the library. Each key corresponds to an // importable path, eg. `import { AgentExecutor } from "langchain/agents"`. @@ -14,7 +10,7 @@ const DEFAULT_GITIGNORE_PATHS = [ // This is used to generate the `exports` field in package.json. // Order is not important. const entrypoints = { - "index": "index" + index: "index", }; // Entrypoints in this list require an optional dependency to be installed. @@ -78,7 +74,10 @@ const updateConfig = () => { }); // Update .gitignore - fs.writeFileSync("./.gitignore", filenames.join("\n") + "\n" + DEFAULT_GITIGNORE_PATHS.join("\n") + "\n"); + fs.writeFileSync( + "./.gitignore", + filenames.join("\n") + "\n" + DEFAULT_GITIGNORE_PATHS.join("\n") + "\n" + ); }; const cleanGenerated = () => { diff --git a/libs/create-langchain-integration/template/tsconfig.cjs.json b/libs/create-langchain-integration/template/tsconfig.cjs.json index 83f5d513ef0f..3b7026ea406c 100644 --- a/libs/create-langchain-integration/template/tsconfig.cjs.json +++ b/libs/create-langchain-integration/template/tsconfig.cjs.json @@ -4,10 +4,5 @@ "module": "commonjs", "declaration": false }, - "exclude": [ - "node_modules", - "dist", - "docs", - "**/tests" - ] -} \ No newline at end of file + "exclude": ["node_modules", "dist", "docs", "**/tests"] +} diff --git a/libs/create-langchain-integration/template/tsconfig.json b/libs/create-langchain-integration/template/tsconfig.json index ffc49dde54a6..bc85d83b6229 100644 --- a/libs/create-langchain-integration/template/tsconfig.json +++ b/libs/create-langchain-integration/template/tsconfig.json @@ -4,11 +4,7 @@ "outDir": "../dist", "rootDir": "./src", "target": "ES2021", - "lib": [ - "ES2021", - "ES2022.Object", - "DOM" - ], + "lib": ["ES2021", "ES2022.Object", "DOM"], "module": "ES2020", "moduleResolution": "nodenext", "esModuleInterop": true, @@ -22,12 +18,6 @@ "allowJs": true, "strict": true }, - "include": [ - "src/**/*" - ], - "exclude": [ - "node_modules", - "dist", - "docs" - ] + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "docs"] } diff --git a/libs/create-langchain-integration/tsconfig.json b/libs/create-langchain-integration/tsconfig.json index bbf3ce815068..8994aa4b3798 100644 --- a/libs/create-langchain-integration/tsconfig.json +++ b/libs/create-langchain-integration/tsconfig.json @@ -4,11 +4,7 @@ "outDir": "./dist", "rootDir": "./", "target": "ES2021", - "lib": [ - "ES2021", - "ES2022.Object", - "DOM" - ], + "lib": ["ES2021", "ES2022.Object", "DOM"], "module": "ES2020", "moduleResolution": "nodenext", "esModuleInterop": true, @@ -23,9 +19,5 @@ "strict": true, "resolveJsonModule": true }, - "exclude": [ - "node_modules", - "dist", - "template" - ] + "exclude": ["node_modules", "dist", "template"] } diff --git a/yarn.lock b/yarn.lock index 56a4e15ca49b..e044f63324bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15588,6 +15588,7 @@ __metadata: dotenv: ^16.3.1 fast-glob: ^3.3.2 picocolors: ^1.0.0 + prettier: ^2.8.3 prompts: ^2.4.2 typescript: <5.2.0 update-check: ^1.5.4