Merge pull request #13 from indexnetwork/refactor-indexer

Migration to LangChainJS

indexer/.env_example

@@ -0,0 +1,21 @@
+OPENAI_API_KEY=
+MODEL_EMBEDDING="text-embedding-ada-002"
+MODEL_CHAT="gpt-3.5-turbo-16k"
+
+PORT=3012
+
+CHROMA_URL=http://localhost:8001
+CHROMA_API_KEY=
+COLLECTION_NAME=chroma-indexer
+
+MISTRAL_API_KEY=
+MISTRAL_MODEL_EMBEDDING=mistral-embed
+MISTRAL_MODEL_CHAT=mistral-small
+
+UNSTRUCTURED_URL=http://localhost:8000/general/v0/general
+UNSTRUCTURED_API_KEY=
+
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
+export LANGCHAIN_API_KEY=
+export LANGCHAIN_PROJECT="Local"

indexer/.eslintrc.js

@@ -0,0 +1,25 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    tsconfigRootDir: __dirname,
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+  },
+};

indexer/.gitignore

@@ -1,10 +1,39 @@
-.chroma/*
-chroma-indexes/*
-.idea/*
-app/__pycache__/*
+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+pnpm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
 .DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+tmp/*
+tmp
 .env
-files
-/env/lib/*
-/env/bin/*
-/env/*

indexer/.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}

indexer/Dockerfile

@@ -1,6 +1,7 @@
-FROM python:3.11.5
-WORKDIR /code
-COPY ./app /code/app
-COPY ./requirements.txt /code/requirements.txt
-RUN pip3 install -r requirements.txt
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"]
+FROM node:20-alpine
+COPY package*.json ./
+COPY yarn.lock ./
+COPY tsconfig.json ./
+RUN yarn install
+COPY src/ src/
+ENTRYPOINT yarn start