diff --git a/.github/workflows/release-chromadb.yml b/.github/workflows/release-chromadb.yml index b7b0b770816..3f65c60e0db 100644 --- a/.github/workflows/release-chromadb.yml +++ b/.github/workflows/release-chromadb.yml @@ -242,6 +242,31 @@ jobs: prerelease: ${{ needs.check-tag.outputs.tag_matches != 'true' }} makeLatest: ${{ needs.check-tag.outputs.tag_matches == 'true' }} + release-cloudformation: + name: Release CloudFormation + runs-on: ubuntu-latest + needs: release-github # Ensure it runs after the GitHub release + if: ${{ needs.check-tag.outputs.tag_matches == 'true' }} # Only run if tag matches + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set Version + run: | + sed -i "s/LATEST_VERSION/${{ needs.get-version.outputs.version }}/g" ./deployments/aws/chroma.cf.json + cat ./deployments/aws/chroma.cf.json # Optional: output the modified file for verification + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::369178033109:role/github-action-generate-cf-template + aws-region: us-east-1 + + - name: Upload CloudFormation Template to S3 + run: | + aws s3 cp ./deployments/aws/chroma.cf.json s3://public.trychroma.com/cloudformation/latest/chroma.cf.json --acl public-read + + release-docs: name: Deploy docs to Vercel runs-on: ubuntu-latest diff --git a/deployments/aws/chroma.cf.json b/deployments/aws/chroma.cf.json new file mode 100644 index 00000000000..7bca1a30d99 --- /dev/null +++ b/deployments/aws/chroma.cf.json @@ -0,0 +1,247 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Create a stack that runs Chroma hosted on a single instance", + "Parameters": { + "KeyName": { + "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance", + "Type": "String", + "ConstraintDescription": "If present, must be the name of an existing EC2 KeyPair.", + "Default": "" + }, + "InstanceType": { + "Description": "EC2 instance type", + "Type": "String", + "Default": "t3.small" + }, + "ChromaVersion": { + "Description": "Chroma version to install", + "Type": "String", + "Default": "LATEST_VERSION" + }, + "ChromaServerAuthCredentials": { + "Description": "Chroma authentication credentials", + "Type": "String", + "Default": "" + }, + "ChromaServerAuthProvider" : { + "Description": "Chroma authentication provider", + "Type": "String", + "Default": "", + "AllowedValues": [ + "", + "chromadb.auth.token_authn.TokenAuthenticationServerProvider", + "chromadb.auth.basic_authn.BasicAuthenticationServerProvider" + ] + }, + "ChromaAuthTokenTransportHeader": { + "Description": "Chroma authentication custom token header", + "Type": "String", + "Default": "" + } + }, + "Conditions": { + "HasKeyName": { + "Fn::Not": [ + { + "Fn::Equals": [ + { + "Ref": "KeyName" + }, + "" + ] + } + ] + } + }, + "Resources": { + "ChromaInstance": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": { + "Fn::FindInMap": [ + "Region2AMI", + { + "Ref": "AWS::Region" + }, + "AMI" + ] + }, + "InstanceType": { + "Ref": "InstanceType" + }, + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#!/bin/bash\n", + "amazon-linux-extras install docker -y\n", + "usermod -a -G docker ec2-user\n", + "curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose\n", + "chmod +x /usr/local/bin/docker-compose\n", + "ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose\n", + "systemctl enable docker\n", + "systemctl start docker\n", + "\n", + "mkdir -p /home/ec2-user/config\n", + "curl -o /home/ec2-user/docker-compose.yml https://s3.amazonaws.com/public.trychroma.com/cloudformation/assets/docker-compose.yml\n", + { + "Fn::Sub": "sed -i 's/CHROMA_VERSION/${ChromaVersion}/g' /home/ec2-user/docker-compose.yml\n" + }, + "chown ec2-user:ec2-user /home/ec2-user/docker-compose.yml\n", + "# Create .env file\n", + { + "Fn::Sub": "echo 'CHROMA_SERVER_AUTHN_CREDENTIALS=${ChromaServerAuthCredentials}' >> /home/ec2-user/.env\n" + }, + { + "Fn::Sub": "echo 'CHROMA_SERVER_AUTHN_PROVIDER=${ChromaServerAuthProvider}' >> /home/ec2-user/.env\n" + }, + { + "Fn::Sub": "echo 'CHROMA_AUTH_TOKEN_TRANSPORT_HEADER=${ChromaAuthTokenTransportHeader}' >> /home/ec2-user/.env\n" + }, + "chown ec2-user:ec2-user /home/ec2-user/.env\n", + "cd /home/ec2-user\n", + "sudo -u ec2-user docker-compose up -d\n" + ] + ] + } + }, + "SecurityGroupIds": [ + { + "Ref": "ChromaInstanceSecurityGroup" + } + ], + "KeyName": { + "Fn::If": [ + "HasKeyName", + { + "Ref": "KeyName" + }, + { + "Ref": "AWS::NoValue" + } + ] + }, + "BlockDeviceMappings": [ + { + "DeviceName": { + "Fn::FindInMap": [ + "Region2AMI", + { + "Ref": "AWS::Region" + }, + "RootDeviceName" + ] + }, + "Ebs": { + "VolumeSize": 24 + } + } + ] + } + }, + "ChromaInstanceSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Chroma Instance Security Group", + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": "22", + "ToPort": "22", + "CidrIp": "0.0.0.0/0" + }, + { + "IpProtocol": "tcp", + "FromPort": "8000", + "ToPort": "8000", + "CidrIp": "0.0.0.0/0" + } + ] + } + } + }, + "Outputs": { + "ServerIp": { + "Description": "IP address of the Chroma server", + "Value": { + "Fn::GetAtt": [ + "ChromaInstance", + "PublicIp" + ] + } + } + }, + "Mappings": { + "Region2AMI": { + "ap-south-1": { + "AMI": "ami-0700df939e7249d03", + "RootDeviceName": "/dev/xvda" + }, + "eu-north-1": { + "AMI": "ami-040bf52c6d056e2e8", + "RootDeviceName": "/dev/xvda" + }, + "eu-west-3": { + "AMI": "ami-04e3030019c3f0f44", + "RootDeviceName": "/dev/xvda" + }, + "eu-west-2": { + "AMI": "ami-0d729d2846a86a9e7", + "RootDeviceName": "/dev/xvda" + }, + "eu-west-1": { + "AMI": "ami-0ea0f26a6d50850c5", + "RootDeviceName": "/dev/xvda" + }, + "ap-northeast-3": { + "AMI": "ami-0f8a30a8b49a72ed4", + "RootDeviceName": "/dev/xvda" + }, + "ap-northeast-2": { + "AMI": "ami-00a5b84c0873f8c3f", + "RootDeviceName": "/dev/xvda" + }, + "ap-northeast-1": { + "AMI": "ami-072bfb8ae2c884cc4", + "RootDeviceName": "/dev/xvda" + }, + "ca-central-1": { + "AMI": "ami-0843f7c45354d48b5", + "RootDeviceName": "/dev/xvda" + }, + "sa-east-1": { + "AMI": "ami-0555c5c3b52744258", + "RootDeviceName": "/dev/xvda" + }, + "ap-southeast-1": { + "AMI": "ami-05bf0125f616dc488", + "RootDeviceName": "/dev/xvda" + }, + "ap-southeast-2": { + "AMI": "ami-06fdec94cc3067ad1", + "RootDeviceName": "/dev/xvda" + }, + "eu-central-1": { + "AMI": "ami-0e2031728ef69a466", + "RootDeviceName": "/dev/xvda" + }, + "us-east-1": { + "AMI": "ami-026b57f3c383c2eec", + "RootDeviceName": "/dev/xvda" + }, + "us-east-2": { + "AMI": "ami-089a545a9ed9893b6", + "RootDeviceName": "/dev/xvda" + }, + "us-west-1": { + "AMI": "ami-0e0ece251c1638797", + "RootDeviceName": "/dev/xvda" + }, + "us-west-2": { + "AMI": "ami-0b71e9be6a79aec29", + "RootDeviceName": "/dev/xvda" + } + } + } +} diff --git a/deployments/aws/docker-compose.yml b/deployments/aws/docker-compose.yml new file mode 100644 index 00000000000..2b0705c97b1 --- /dev/null +++ b/deployments/aws/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3.9' + +networks: + net: + driver: bridge + +services: + server: + image: ghcr.io/chroma-core/chroma:CHROMA_VERSION + volumes: + - index_data:/index_data + ports: + - "8000:8000" + networks: + - net + env_file: + - .env + +volumes: + index_data: + driver: local + backups: + driver: local diff --git a/docs/docs.trychroma.com/components/markdoc/Note.tsx b/docs/docs.trychroma.com/components/markdoc/Note.tsx index cd6bf3629d5..5f1dd83bd4a 100644 --- a/docs/docs.trychroma.com/components/markdoc/Note.tsx +++ b/docs/docs.trychroma.com/components/markdoc/Note.tsx @@ -20,10 +20,10 @@ export function Note({ type, title, children }: NoteProps) { >
- {(type === 'tip')? : null} - {(type === 'default')? : null} - {(type === 'warning')? : null} - {(type === 'caution')? : null} + {(type === 'tip')? : null} + {(type === 'default')? : null} + {(type === 'warning')? : null} + {(type === 'caution')? : null}
{title ? ( diff --git a/docs/docs.trychroma.com/next-env.d.ts b/docs/docs.trychroma.com/next-env.d.ts index 4f11a03dc6c..a4a7b3f5cfa 100644 --- a/docs/docs.trychroma.com/next-env.d.ts +++ b/docs/docs.trychroma.com/next-env.d.ts @@ -2,4 +2,4 @@ /// // NOTE: This file should not be edited -// see https://nextjs.org/docs/basic-features/typescript for more information. +// see https://nextjs.org/docs/pages/building-your-application/configuring/typescript for more information. diff --git a/docs/docs.trychroma.com/pages/deployment/_sidenav.js b/docs/docs.trychroma.com/pages/deployment/_sidenav.js index 2e0516c512c..7f874d89c4e 100644 --- a/docs/docs.trychroma.com/pages/deployment/_sidenav.js +++ b/docs/docs.trychroma.com/pages/deployment/_sidenav.js @@ -8,6 +8,19 @@ export const items = [ // { href: '/deployment/kubernetes', children: 'Kubernetes' }, // ] // }, + { + title: "Chroma Server", + links: [ + { href: '/deployment/client-server-mode', children: "Client/Server Mode"}, + { href: '/deployment/thin-client', children: "Python Thin-Client"}, + ] + }, + { + title: 'Containers', + links: [ + { href: '/deployment/docker', children: 'Docker' }, + ] + }, { title: 'Cloud Providers', links: [ @@ -19,6 +32,7 @@ export const items = [ { title: 'Administration', links: [ + { href: '/deployment/performance', children: "🚀 Performance"}, // { href: '/deployment/logging', children: 'Logging' }, { href: '/deployment/observability', children: '👀 Observability' }, { href: '/deployment/migration', children: '✈️ Migration' }, diff --git a/docs/docs.trychroma.com/pages/deployment/auth.md b/docs/docs.trychroma.com/pages/deployment/auth.md index 714c58734d8..7a3b723e5a9 100644 --- a/docs/docs.trychroma.com/pages/deployment/auth.md +++ b/docs/docs.trychroma.com/pages/deployment/auth.md @@ -27,189 +27,184 @@ Supported authentication methods: | Client/Python | ✅ `Alpha` | ✅ `Alpha` | | Client/JS | ✅ `Alpha` | ✅ `Alpha` | -### Basic Authentication +*** -#### Server Setup +In this guide we will add authentication to a simple Chroma server running locally using our CLI: + +```shell +chroma run --path +``` + +We also have dedicated auth guides for various deployments: +* [Docker](/deployment/docker#authentication-with-docker) +* [AWS](/deployment/aws#authentication-with-AWS) +* GCP - Coming Soon! +* Azure - Coming Soon! + +### Encrypted User:Password Authentication + +#### Server Set-Up ##### Generate Server-Side Credentials {% note type="note" title="Security Practices" %} -A good security practice is to store the password securely. In the example below we use [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) (currently the only supported hash in Chroma server side auth) to hash the plaintext password. +A good security practice is to store the password securely. In the example below we use [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) (currently the only supported hash in Chroma server side auth) to hash the plaintext password. If you'd like to see support for additional hash functions, feel free to [contribute](../contributing) new ones! {% /note %} -To generate the password hash, run the following command: +To generate the password hash, run the following command (you may need to install `httpasswd`): ```bash -docker run --rm --entrypoint htpasswd httpd:2 -Bbn admin admin > server.htpasswd +htpasswd -Bbn admin admin > server.htpasswd ``` -This creates the bcrypt password hash for the password `admin` and puts it into `server.htpasswd` alongside the user `admin`. It will look like `admin:`. +This creates the bcrypt password hash for the password `admin`, for the `admin` user, and puts it into `server.htpasswd` in your current working directory. It will look like `admin:`. ##### Running the Server Set the following environment variables: ```bash -export CHROMA_SERVER_AUTHN_CREDENTIALS_FILE="server.htpasswd" +export CHROMA_SERVER_AUTHN_CREDENTIALS_FILE="" export CHROMA_SERVER_AUTHN_PROVIDER="chromadb.auth.basic_authn.BasicAuthenticationServerProvider" ``` -And run the server as normal: +And run the Chroma server: ```bash -chroma run --path /db_path +chroma run --path ``` -#### Client Setup +#### Client Set-Up {% tabs group="code-lang" hideTabs=true %} {% tab label="Python" %} +We will use Chroma's `Settings` object to define the authentication method on the client. + ```python import chromadb from chromadb.config import Settings client = chromadb.HttpClient( - settings=Settings(chroma_client_auth_provider="chromadb.auth.basic_authn.BasicAuthClientProvider",chroma_client_auth_credentials="admin:admin")) -client.heartbeat() # this should work with or without authentication - it is a public endpoint - -client.get_version() # this should work with or without authentication - it is a public endpoint - -client.list_collections() # this is a protected endpoint and requires authentication + host="localhost", + port=8000, + settings=Settings( + chroma_client_auth_provider="chromadb.auth.basic_authn.BasicAuthClientProvider", + chroma_client_auth_credentials="admin:admin" + ) +) + +chroma_client.heartbeat() ``` {% /tab %} {% tab label="Javascript" %} -##### Basic authentication (username & password) ```javascript -const client = new ChromaClient({ - path: "http://localhost:8000" - auth: { provider: "basic", credentials: "admin:admin" }, -}); -``` +import { ChromaClient } from "chromadb"; -##### Token authentication -In this method, we use the Bearer scheme. Namely, the token is sent as: `Authorization: Bearer test-token` -```javascript -const client = new ChromaClient({ - path: "http://localhost:8000", - auth: { provider: "token", credentials: "test-token" }, -}); -``` +const chromaClient = new ChromaClient({ + path: "http://localhost:8000", + auth: { + provider: "basic", + credentials: "admin:admin" + } +}) -##### Token authentication (custom header) -In this method, we send the token in a custom header. The header is `X-Chroma-Token`. -```javascript -const client = new ChromaClient({ - path: URL, - auth: { - provider: "token", - credentials: "test-token", - tokenHeaderType: "X_CHROMA_TOKEN", - }, -}); +chromaClient.heartbeat() ``` - {% /tab %} - {% /tabs %} -### Static API Token Authentication +We recommend setting the environment variable `CHROMA_CLIENT_AUTH_CREDENTIALS` instead of specifying the credentials in code. -{% note type="note" title="Tokens" %} -Tokens must be alphanumeric ASCII strings. Tokens are case-sensitive. -{% /note %} +### Static API Token Authentication -#### Server Setup +#### Server Set-Up {% note type="note" title="Security Note" %} -Current implementation of static API token auth supports only ENV based tokens. +Current implementation of static API token auth supports only ENV based tokens. Tokens must be alphanumeric ASCII strings. Tokens are case-sensitive. {% /note %} -##### Running the Server - -Set the following environment variables to use `Authorization: Bearer test-token` to be your authentication header. All environment variables can also be set as [Settings](https://docs.trychroma.com/deployment/aws#step-5:-configure-the-chroma-library). +If, for example, you want the static API token to be "test-token", set the following environment variables. This will set `Authorization: Bearer test-token` as your authentication header. ```bash export CHROMA_SERVER_AUTHN_CREDENTIALS="test-token" export CHROMA_SERVER_AUTHN_PROVIDER="chromadb.auth.token_authn.TokenAuthenticationServerProvider" ``` -To configure multiple tokens and use them for role-based access control (RBAC), use a file like [this](https://github.com/chroma-core/chroma/blob/main/examples/basic_functionality/authz/authz.yaml) and the following configuration settings: +To use `X-Chroma-Token: test-token` type of authentication header you can set the `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` environment variable: ```bash -export CHROMA_SERVER_AUTHN_CREDENTIALS_FILE= -export CHROMA_SERVER_AUTHZ_CONFIG_FILE= # Note: these are the same! +export CHROMA_SERVER_AUTHN_CREDENTIALS="test-token" export CHROMA_SERVER_AUTHN_PROVIDER="chromadb.auth.token_authn.TokenAuthenticationServerProvider" -export CHROMA_SERVER_AUTHZ_PROVIDER="chromadb.auth.simple_rbac_authz.SimpleRBACAuthorizationProvider" +export CHROMA_AUTH_TOKEN_TRANSPORT_HEADER="X-Chroma-Token" ``` -To use `X-Chroma-Token: test-token` type of authentication header you can set the `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` environment variable or configuration setting. +Then, run the Chroma server: ```bash -export CHROMA_SERVER_AUTHN_CREDENTIALS="test-token" +chroma run --path +``` + +To configure multiple tokens and use them for role-based access control (RBAC), use a file like [this](https://github.com/chroma-core/chroma/blob/main/examples/basic_functionality/authz/authz.yaml) and the following environment variables: + +```bash +export CHROMA_SERVER_AUTHN_CREDENTIALS_FILE="" +export CHROMA_SERVER_AUTHZ_CONFIG_FILE="" # Note: these are the same! export CHROMA_SERVER_AUTHN_PROVIDER="chromadb.auth.token_authn.TokenAuthenticationServerProvider" -export CHROMA_AUTH_TOKEN_TRANSPORT_HEADER="X_CHROMA_TOKEN" +export CHROMA_SERVER_AUTHZ_PROVIDER="chromadb.auth.simple_rbac_authz.SimpleRBACAuthorizationProvider" +``` + +#### Client Set-Up {% tabs group="code-lang" hideTabs=true %} {% tab label="Python" %} -#### Client Setup +We will use Chroma's `Settings` object to define the authentication method on the client. ```python import chromadb from chromadb.config import Settings client = chromadb.HttpClient( - settings=Settings(chroma_client_auth_provider="chromadb.auth.token_authn.TokenAuthClientProvider", - chroma_client_auth_credentials="test-token")) -client.heartbeat() # this should work with or without authentication - it is a public endpoint + host="localhost", + port=8000, + settings=Settings( + chroma_client_auth_provider="chromadb.auth.token_authn.TokenAuthClientProvider", + chroma_client_auth_credentials="test-token" + ) +) + +chroma_client.heartbeat() +``` -client.get_version() # this should work with or without authentication - it is a public endpoint +If you are using a custom `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` (like `X-Chroma-Token`), add it to your `Settings`: -client.list_collections() # this is a protected endpoint and requires authentication +```python +chroma_auth_token_transport_header="X-Chroma-Token" ``` {% /tab %} {% tab label="Javascript" %} -#### Client Setup - -Using the default `Authorization: Bearer ` header: - -```js +```javascript import { ChromaClient } from "chromadb"; -const client = new ChromaClient({ - auth: { provider: "token", credentials: "test-token" }, -}); -//or explicitly specifying the auth header type -const client = new ChromaClient({ - auth: { - provider: "token", - credentials: "test-token", - tokenHeaderType: "AUTHORIZATION", - }, -}); -``` - -Using custom Chroma auth token `X-Chroma-Token: ` header: +const chromaClient = new ChromaClient({ + path: "http://localhost:8000", + auth: { + provider: "token", + credentials: "test-token", + tokenHeaderType: "X-Chroma-Token" + } +}) -```js -import { ChromaClient } from "chromadb"; - -const client = new ChromaClient({ - auth: { - provider: "token", - credentials: "test-token", - tokenHeaderType: "X_CHROMA_TOKEN", - }, -}); +chromaClient.heartbeat() ``` - {% /tab %} - {% /tabs %} + +We recommend setting the environment variable `CHROMA_CLIENT_AUTH_CREDENTIALS` instead of specifying the token in code. Similarly, you can read the value of `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` in the client construction. diff --git a/docs/docs.trychroma.com/pages/deployment/aws.md b/docs/docs.trychroma.com/pages/deployment/aws.md index 7a2f9151e1a..24b51d9e0f0 100644 --- a/docs/docs.trychroma.com/pages/deployment/aws.md +++ b/docs/docs.trychroma.com/pages/deployment/aws.md @@ -2,88 +2,64 @@ title: "☁️ AWS Deployment" --- -{% note type="caution" title="Alpha" %} -Chroma Server is currently in Alpha. We are working hard to move Chroma from an in-memory single-process oriented library to a distributed production-grade DB! +{% tabs group="code-lang" hideContent=true %} -- [x] Alpha <- Currently -- [ ] Technical Preview - ~1 month away, powered by a completely new backend -- [ ] Full production -- [ ] GA - General Availability +{% tab label="Python" %} +{% /tab %} -{% /note %} - -# ☁️ Deployment - -You can also deploy Chroma on a long-running server, and connect to it -remotely. - -There are many possible configurations, but for convenience we have -provided a very simple AWS CloudFormation template to experiment with -deploying Chroma to EC2 on AWS. - -## Hosted Chroma +{% tab label="Javascript" %} +{% /tab %} -We want to offer hosted Chroma, and we need your help. +{% /tabs %} -Fill out the survey to jump the wait-list. Coming Q3 2023. +{% note type="tip" title="Hosted Chroma" %} +Chroma Cloud, our fully managed hosted service, is in early access. Fill out the survey to jump the waitlist and get the best retrieval experience. Full access coming Q1 2025. [📝 30 second survey](https://airtable.com/shrOAiDUtS2ILy5vZ) -## Docker - -You can run a Chroma server in a Docker container. - -You can get the Chroma Docker image from [Docker Hub](https://hub.docker.com/r/chromadb/chroma), or from the [Chroma GitHub Container Registry](https://github.com/chroma-core/chroma/pkgs/container/chroma) - -```sh -docker pull chromadb/chroma -docker run -p 8000:8000 chromadb/chroma -``` - -You can also build the Docker image yourself from the Dockerfile in the [Chroma GitHub repository](https://github.com/chroma-core/chroma) - -```sh -git clone git@github.com:chroma-core/chroma.git -cd chroma -docker-compose up -d --build -``` - -The Chroma client can then be configured to connect to the server running in the Docker container. - -```python -import chromadb -chroma_client = chromadb.HttpClient(host='localhost', port=8000) -``` - -### Authentication with Docker +{% /note %} -By default, the Docker image will run with no authentication. Follow the [Authentication](./usage-guide#authentication) section of the Usage Guide to configure authentication in the Docker container. +{% note type="tip" title="" %} +If you are using Chroma in production, please fill out [this form](https://airtable.com/appqd02UuQXCK5AuY/pagr1D0NFQoNpUpNZ/form), and we will add you to a dedicated Slack workspace for supporting production users. We would love to help you think through the design of your system, or if you would be a good fit for our upcoming distributed cloud service. You can also join the [#production-chroma](https://discord.com/channels/1073293645303795742/1292554909694300211) channel on Discord to join our community! +This is the best place to +1. Get support with building with Chroma in prod. +2. Stay up-to-date with exciting new features. +3. Get swag! +{% /note %} -You can also create a `.chroma_env` file setting the required environment variables and pass it to the Docker container with the `--env-file` flag when running the container. +## A Simple AWS Deployment -```sh -docker run --env-file ./.chroma_env -p 8000:8000 chromadb/chroma -``` +You can deploy Chroma on a long-running server, and connect to it +remotely. -## Simple AWS Deployment +There are many possible configurations, but for convenience we have +provided a very simple AWS CloudFormation template to experiment with +deploying Chroma to EC2 on AWS. -:warning: Chroma and its underlying database need at least 2gb of RAM, +{% note type="warning" title="" %} +Chroma and its underlying database [need at least 2GB of RAM](./performance#results-summary), which means it won't fit on the 1gb instances provided as part of the -AWS Free Tier. This template uses a `t3.small` EC2 instance, which -costs about two cents an hour, or $15 for a full month. If you follow these +AWS Free Tier. This template uses a [`t3.small`](https://aws.amazon.com/ec2/instance-types/t3/#Product%20Details) EC2 instance, which +costs about two cents an hour, or $15 for a full month, and gives you 2GiB of memory. If you follow these instructions, AWS will bill you accordingly. +{% /note %} -:warning: This basic stack doesn't support any kind of authentication; -anyone who knows your server IP will be able to add and query for -embeddings. To secure this endpoint, you'll need to put it behind +{% note type="warning" title="" %} +In this guide we show you how to secure your endpoint using [Chroma's +native authentication support](./aws#authentication-with-aws). Alternatively, you can put it behind [AWS API Gateway](https://aws.amazon.com/api-gateway/) or add your own -authenticating proxy. +authenticating proxy. This basic stack doesn't support any kind of authentication; +anyone who knows your server IP will be able to add and query for +embeddings. +{% /note %} -:warning: By default, this template saves all data on a single +{% note type="warning" title="" %} +By default, this template saves all data on a single volume. When you delete or replace it, the data will disappear. For serious production use (with high availability, backups, etc.) please read and understand the CloudFormation template and use it as a basis for what you need, or reach out to the Chroma team for assistance. +{% /note %} ### Step 1: Get an AWS Account @@ -102,34 +78,30 @@ and will be using environment variables to configure AWS. Export the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables in your shell: -``` - +```shell export AWS_ACCESS_KEY_ID=**\*\***\*\*\*\***\*\*** export AWS_SECRET_ACCESS_KEY=****\*\*****\*\*****\*\***** - ``` You can also configure AWS to use a region of your choice using the `AWS_REGION` environment variable: -``` - +```shell export AWS_REGION=us-east-1 - ``` ### Step 3: Run CloudFormation -Chroma publishes Cloudformation templates to S3 for each release. +Chroma publishes a [CloudFormation template](https://s3.amazonaws.com/public.trychroma.com/cloudformation/latest/chroma.cf.json) to S3 for each release. -To launch the template using AWS CloudFormation, run the following command line invocation: +To launch the template using AWS CloudFormation, run the following command line invocation. + +Replace `--stack-name my-chroma-stack` with a different stack name, if you wish. ```sh aws cloudformation create-stack --stack-name my-chroma-stack --template-url https://s3.amazonaws.com/public.trychroma.com/cloudformation/latest/chroma.cf.json ``` -Replace `--stack-name my-chroma-stack` with a different stack name, if you wish. - Wait a few minutes for the server to boot up, and Chroma will be available! You can get the public IP address of your new Chroma server using the AWS console, or using the following command: @@ -137,7 +109,9 @@ available! You can get the public IP address of your new Chroma server using the aws cloudformation describe-stacks --stack-name my-chroma-stack --query 'Stacks[0].Outputs' ``` -### Step 4: Customize the Stack (optional) +Note that even after the IP address of your instance is available, it may still take a few minutes for Chroma to be up and running. + +#### Customize the Stack (optional) The CloudFormation template allows you to pass particular key/value pairs to override aspects of the stack. Available keys are: @@ -154,53 +128,153 @@ above, but on a `m5.4xlarge` EC2 instance, and adding a KeyPair named `mykey` so anyone with the associated private key can SSH into the machine: -``` - +```shell aws cloudformation create-stack --stack-name my-chroma-stack --template-url https://s3.amazonaws.com/public.trychroma.com/cloudformation/latest/chroma.cf.json \ --parameters ParameterKey=KeyName,ParameterValue=mykey \ ParameterKey=InstanceType,ParameterValue=m5.4xlarge +``` + +### Step 4: Chroma Client Set-Up + +Once your EC2 instance is up and running with Chroma, all +you need to do is configure your `HttpClient` to use the server's IP address and port +`8000`. Since you are running a Chroma server on AWS, our [thin-client package](./thin-client.md) may be enough for your application. + +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Python" %} + +```python +import chromadb + +chroma_client = chromadb.HttpClient( + host="", + port=8000 +) +chroma_client.heartbeat() +``` + +{% /tab %} +{% tab label="Javascript" %} + +```javascript +import { ChromaClient } from "chromadb"; + +const chromaClient = new ChromaClient({ + path: "", + port: 8000 +}) +chromaClient.heartbeat() +``` + +{% /tab %} +{% /tabs %} + +### Step 5: Clean Up (optional). + +To destroy the stack and remove all AWS resources, use the AWS CLI `delete-stack` command. + +{% note type="warning" title="Note" %} +This will destroy all the data in your Chroma database, +unless you've taken a snapshot or otherwise backed it up. +{% /note %} +```shell +aws cloudformation delete-stack --stack-name my-chroma-stack ``` -### Step 5: Configure the Chroma Library +## Authentication with AWS + +By default, the EC2 instance created by our CloudFormation template will run with no authentication. There are many ways to secure your Chroma instance on AWS. In this guide we will use a simple set-up using Chroma's native authentication support. + +You can learn more about authentication with Chroma in the [Auth Guide](/deployment/auth). + +### Static API Token Authentication + +#### Customize Chroma's CloudFormation Stack -When you launch the Chroma client library to actually use Chroma, all -you need to do is configure it to use the server's IP address and port -`8000`. You can do this in two ways: +{% note type="note" title="Security Note" %} +Current implementation of static API token auth supports only ENV based tokens. Tokens must be alphanumeric ASCII strings. Tokens are case-sensitive. +{% /note %} -###### Using Environment Variables +If, for example, you want the static API token to be "test-token", pass the following parameters when creating your Chroma stack. This will set `Authorization: Bearer test-token` as your authentication header. +```shell +aws cloudformation create-stack --stack-name my-chroma-stack --template-url https://s3.amazonaws.com/public.trychroma.com/cloudformation/latest/chroma.cf.json \ + --parameters ParameterKey=ChromaServerAuthCredentials,ParameterValue="test-token" \ + ParameterKey=ChromaServerAuthProvider,ParameterValue="chromadb.auth.token_authn.TokenAuthenticationServerProvider" ``` -export CHROMA_API_IMPL=rest -export CHROMA_SERVER_HOST= -export CHROMA_SERVER_HTTP_PORT=8000 +To use `X-Chroma-Token: test-token` type of authentication header you can set the `ChromaAuthTokenTransportHeader` parameter: +```shell +aws cloudformation create-stack --stack-name my-chroma-stack --template-url https://s3.amazonaws.com/public.trychroma.com/cloudformation/latest/chroma.cf.json \ + --parameters ParameterKey=ChromaServerAuthCredentials,ParameterValue="test-token" \ + ParameterKey=ChromaServerAuthProvider,ParameterValue="chromadb.auth.token_authn.TokenAuthenticationServerProvider" \ + ParameterKey=ChromaAuthTokenTransportHeader,ParameterValue="X-Chroma-Token" ``` -###### In Code +#### Client Set-Up +Add the `CHROMA_CLIENT_AUTH_CREDENTIALS` environment variable to your local environment, and set it to the token you provided the server (`test-token` in this example): + +```shell +export CHROMA_CLIENT_AUTH_CREDENTIALS="test-token" ``` +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Python" %} + +We will use Chroma's `Settings` object to define the authentication method on the client. + +```python +import os import chromadb from chromadb.config import Settings -chroma = chromadb.HttpClient(host=, port=8000) +from dotenv import load_dotenv -``` +load_dotenv() -### Step 6: Clean Up (optional). +client = chromadb.HttpClient( + host="", + port=8000, + settings=Settings( + chroma_client_auth_provider="chromadb.auth.token_authn.TokenAuthClientProvider", + chroma_client_auth_credentials=os.getenv("CHROMA_CLIENT_AUTH_CREDENTIALS") + ) +) -To destroy the stack and remove all AWS resources, use the AWS CLI `delete-stack` command. +client.heartbeat() +``` + +If you are using a custom `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` (like `X-Chroma-Token`), add it to your `Settings`: +```python +chroma_auth_token_transport_header=os.getenv("CHROMA_AUTH_TOKEN_TRANSPORT_HEADER") ``` -aws cloudformation delete-stack --stack-name my-chroma-stack +{% /tab %} +{% tab label="Javascript" %} + +```javascript +import { ChromaClient } from "chromadb"; + +const chromaClient = new ChromaClient({ + path: "", + auth: { + provider: "token", + credentials: process.env.CHROMA_CLIENT_AUTH_CREDENTIALS, + tokenHeaderType: process.env.CHROMA_AUTH_TOKEN_TRANSPORT_HEADER + } +}) +chromaClient.heartbeat() ``` -:warning: This will destroy all the data in your Chroma database, -unless you've taken a snapshot or otherwise backed it up. +{% /tab %} +{% /tabs %} + +## Troubleshooting -### Troubleshooting +#### Error: No default VPC for this user If you get an error saying `No default VPC for this user` when creating `ChromaInstanceSecurityGroup`, head to [AWS VPC section](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs) and create a default VPC for your user. diff --git a/docs/docs.trychroma.com/pages/deployment/client-server-mode.md b/docs/docs.trychroma.com/pages/deployment/client-server-mode.md new file mode 100644 index 00000000000..3d5c48e47fa --- /dev/null +++ b/docs/docs.trychroma.com/pages/deployment/client-server-mode.md @@ -0,0 +1,67 @@ +--- +title: "Running Chroma in Client-Server Mode" +--- + +{% tabs group="code-lang" hideContent=true %} + +{% tab label="Python" %} +{% /tab %} + +{% tab label="Javascript" %} +{% /tab %} + +{% /tabs %} + +Chroma can also be configured to run in client/server mode. In this mode, the Chroma client connects to a Chroma server running in a separate process. + +This means that you can deploy single-node Chroma to a [Docker container](./docker), or a machine hosted by a cloud provider like [AWS](./aws), GCP, Azure, and others. Then, you can access your Chroma server from your application using our `HttpClient`. + +You can quickly experiment locally with Chroma in client/server mode by using our CLI: + +```shell +chroma run --path /db_path +``` + +Then use the Chroma `HttpClient` to connect to the server: + +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Python" %} + +```python +import chromadb +chroma_client = chromadb.HttpClient(host='localhost', port=8000) +``` + +Chroma also provides an `AsyncHttpClient`. The behaviors and method signatures are identical to the synchronous client, but all methods that would block are now async: + +```python +import asyncio +import chromadb + +async def main(): + client = await chromadb.AsyncHttpClient() + collection = await client.create_collection(name="my_collection") + await collection.add( + documents=["hello world"], + ids=["id1"] + ) + +asyncio.run(main()) +``` + +If you intend to deploy your Chroma server, you may want to consider our [thin-client package](./thin-client) for client-side interactions. + +{% /tab %} + +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Javascript" %} + +```javascript +// CJS +const { ChromaClient } = require("chromadb"); +// ESM +import { ChromaClient } from "chromadb"; +const client = new ChromaClient(); +``` + +{% /tab %} diff --git a/docs/docs.trychroma.com/pages/deployment/docker.md b/docs/docs.trychroma.com/pages/deployment/docker.md index abdb9da05ea..90975454abc 100644 --- a/docs/docs.trychroma.com/pages/deployment/docker.md +++ b/docs/docs.trychroma.com/pages/deployment/docker.md @@ -1,3 +1,271 @@ --- title: Docker --- + +{% tabs group="code-lang" hideContent=true %} + +{% tab label="Python" %} +{% /tab %} + +{% tab label="Javascript" %} +{% /tab %} + +{% /tabs %} + +{% note type="tip" title="Hosted Chroma" %} +Chroma Cloud, our fully managed hosted service, is in early access. Fill out the survey to jump the waitlist and get the best retrieval experience. Full access coming Q1 2025. + +[📝 30 second survey](https://airtable.com/shrOAiDUtS2ILy5vZ) + +{% /note %} + +## Run Chroma in a Docker Container + +You can run a Chroma server in a Docker container, and access it using the `HttpClient`. + +If you are using Chroma in production, please fill out [this form](https://airtable.com/appqd02UuQXCK5AuY/pagr1D0NFQoNpUpNZ/form), and we will add you to a dedicated Slack workspace for supporting production users. We would love to help you think through the design of your system, or if you would be a good fit for our upcoming distributed cloud service. You can also join the [#production-chroma](https://discord.com/channels/1073293645303795742/1292554909694300211) channel on Discord to join our community! + +If you are using a client in a separate container from the one running your Chroma server, you may only need the [thin-client package](./thin-client) + +You can get the Chroma Docker image from [Docker Hub](https://hub.docker.com/r/chromadb/chroma), or from the [Chroma GitHub Container Registry](https://github.com/chroma-core/chroma/pkgs/container/chroma) + +```sh +docker pull chromadb/chroma +docker run -p 8000:8000 chromadb/chroma +``` + +You can also build the Docker image yourself from the Dockerfile in the [Chroma GitHub repository](https://github.com/chroma-core/chroma) + +```sh +git clone git@github.com:chroma-core/chroma.git +cd chroma +docker-compose up -d --build +``` + +The Chroma client can then be configured to connect to the server running in the Docker container. + +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Python" %} + +```python +import chromadb +chroma_client = chromadb.HttpClient(host='localhost', port=8000) +chroma_client.heartbeat() +``` + +{% /tab %} +{% tab label="Javascript" %} + +```javascript +import { ChromaClient } from "chromadb"; + +const chromaClient = new ChromaClient({ path: "http://localhost:8000" }) +chromaClient.heartbeat() +``` + +{% /tab %} +{% /tabs %} + +## Authentication with Docker + +By default, the Docker image will run with no authentication. In client/server mode, Chroma supports the following authentication methods: +* [RFC 7617](https://www.rfc-editor.org/rfc/rfc7617) Basic Auth with `user:password` base64-encoded `Authorization` header. +* Static auth token in `Authorization: Bearer ` or in `X-Chroma-Token: ` headers. + +You can learn more about authentication with Chroma in the [Auth Guide](/deployment/auth). + +### Encrypted User:Password Authentication + +#### Server Set-Up + +##### Generate Server-Side Credentials + +{% note type="note" title="Security Practices" %} +A good security practice is to store the password securely. In the example below we use [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) (currently the only supported hash in Chroma server side auth) to hash the plaintext password. If you'd like to see support for additional hash functions, feel free to [contribute](../contributing) new ones! +{% /note %} + +To generate the password hash, run the following command: + +```bash +docker run --rm --entrypoint htpasswd httpd:2 -Bbn admin admin > server.htpasswd +``` + +This creates the bcrypt password hash for the password `admin`, for the `admin` user, and puts it into `server.htpasswd` in your current working directory. It will look like `admin:`. + +##### Running the Server + +Create a `.chroma_env` file, and set in it the following environment variables: + +```text +CHROMA_SERVER_AUTHN_CREDENTIALS= +CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.basic_authn.BasicAuthenticationServerProvider +``` + +Then, run the Chroma container, and pass it your `.chroma_env` using the `--env-file` flag: + +```bash +docker run --env-file ./.chroma_env -p 8000:8000 chromadb/chroma +``` + +#### Client Set-Up + +In your client environment, set the `CHROMA_CLIENT_AUTH_CREDENTIALS` variable to the user:password combination (`admin:admin` in this example): + +```shell +export CHROMA_CLIENT_AUTH_CREDENTIALS="admin:admin" +``` + +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Python" %} + +Install `python-dotenv`. This will allow us to read the environment variables from `.chroma_env` easily: + +```shell +pip install python-dotenv +``` + +We will use Chroma's `Settings` object to define the authentication method on the client. + +```python +import os +import chromadb +from chromadb.config import Settings +from dotenv import load_dotenv + +load_dotenv('/path/to/your/.chroma_env') + +client = chromadb.HttpClient( + host="localhost", + port=8000, + settings=Settings( + chroma_client_auth_provider="chromadb.auth.basic_authn.BasicAuthClientProvider", + chroma_client_auth_credentials=os.getenv("CHROMA_CLIENT_AUTH_CREDENTIALS") + ) +) + +chroma_client.heartbeat() +``` + +{% /tab %} +{% tab label="Javascript" %} + +```javascript +import { ChromaClient } from "chromadb"; + +const chromaClient = new ChromaClient({ + path: "http://localhost:8000", + auth: { + provider: "basic", + credentials: process.env.CHROMA_CLIENT_AUTH_CREDENTIALS + } +}) + +chromaClient.heartbeat() +``` + +{% /tab %} +{% /tabs %} + +### Static API Token Authentication + +#### Server Set-Up + +{% note type="note" title="Security Note" %} +Current implementation of static API token auth supports only ENV based tokens. Tokens must be alphanumeric ASCII strings. Tokens are case-sensitive. +{% /note %} + +If, for example, you want the static API token to be "test-token", add the following environment variables to your `.chroma_env`. This will set `Authorization: Bearer test-token` as your authentication header. + +```text +CHROMA_SERVER_AUTHN_CREDENTIALS=test-token +CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider +``` + +If instead of the default `Authorization: Bearer ` header, you want to use a custom one like `X-Chroma-Token: test-token`, you can set the `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` environment variable: + +```text +CHROMA_SERVER_AUTHN_CREDENTIALS=test-token +CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider +CHROMA_AUTH_TOKEN_TRANSPORT_HEADER=X-Chroma-Token +``` + +Then, run the Chroma server: + +```bash +docker run --env-file ./.chroma_env -p 8000:8000 chromadb/chroma +``` + +To configure multiple tokens and use them for role-based access control (RBAC), use a file like [this](https://github.com/chroma-core/chroma/blob/main/examples/basic_functionality/authz/authz.yaml) and the following environment variables: + +```text +CHROMA_SERVER_AUTHN_CREDENTIALS_FILE= +CHROMA_SERVER_AUTHZ_CONFIG_FILE= # Note: these are the same! +CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider +CHROMA_SERVER_AUTHZ_PROVIDER=chromadb.auth.simple_rbac_authz.SimpleRBACAuthorizationProvider +``` + +In this case, you will have to set up a volume to allow the Chroma Docker container to use your `authz.yaml` file: + +```bash +docker run --env-file ./.chroma_env -v :/chroma/ -p 8000:8000 chromadb/chroma +``` + +#### Client Set-Up + +{% tabs group="code-lang" hideTabs=true %} +{% tab label="Python" %} + +Install `python-dotenv`. This will allow us to read the environment variables from `.chroma_env` easily: + +```shell +pip install python-dotenv +``` + +We will use Chroma's `Settings` object to define the authentication method on the client. + +```python +import os +import chromadb +from chromadb.config import Settings +from dotenv import load_dotenv + +load_dotenv('/path/to/your/.chroma_env') + +client = chromadb.HttpClient( + host="localhost", + port=8000, + settings=Settings( + chroma_client_auth_provider="chromadb.auth.token_authn.TokenAuthClientProvider", + chroma_client_auth_credentials=os.getenv("CHROMA_CLIENT_AUTH_CREDENTIALS") + ) +) + +chroma_client.heartbeat() +``` + +If you are using a custom `CHROMA_AUTH_TOKEN_TRANSPORT_HEADER` (like `X-Chroma-Token`), add it to your `Settings`: + +```python +chroma_auth_token_transport_header=os.getenv("CHROMA_AUTH_TOKEN_TRANSPORT_HEADER") +``` + +{% /tab %} +{% tab label="Javascript" %} + +```javascript +import { ChromaClient } from "chromadb"; + +const chromaClient = new ChromaClient({ + path: "http://localhost:8000", + auth: { + provider: "token", + credentials: process.env.CHROMA_CLIENT_AUTH_CREDENTIALS, + tokenHeaderType: process.env.CHROMA_AUTH_TOKEN_TRANSPORT_HEADER + } +}) + +chromaClient.heartbeat() +``` + +{% /tab %} +{% /tabs %} diff --git a/docs/docs.trychroma.com/pages/deployment/index.md b/docs/docs.trychroma.com/pages/deployment/index.md index 565cf938b2d..a829a10fadf 100644 --- a/docs/docs.trychroma.com/pages/deployment/index.md +++ b/docs/docs.trychroma.com/pages/deployment/index.md @@ -4,17 +4,26 @@ title: ☁️ Deployment *** -### Cloud Providers +{% note type="tip" title="Hosted Chroma" %} +Chroma Cloud, our fully managed hosted service, is in early access. Fill out the survey to jump the waitlist and get the best retrieval experience. Full access coming Q1 2025. + +[📝 30 second survey](https://airtable.com/shrOAiDUtS2ILy5vZ) -Chroma single-node is easy to deploy to a variety of cloud providers. Please use these guides to get started. +{% /note %} -{% special_table %} -{% /special_table %} +You can run Chroma single-node in [client/server mode](/deployment/client-server-mode), and easily deploy it. In this section, we also show you how to make sure your Chroma server is secure and reliable, and how to understand its performance at scale. -| Provider | -|--------------| -| [AWS](/deployment/aws) | -| 🚧 *More Coming Soon* | +If you are using Chroma in production, please fill out [this form](https://airtable.com/appqd02UuQXCK5AuY/pagr1D0NFQoNpUpNZ/form), and we will add you to a dedicated Slack workspace for supporting production users. We would love to help you think through the design of your system, or if you would be a good fit for our upcoming distributed cloud service. You can also join the [#production-chroma](https://discord.com/channels/1073293645303795742/1292554909694300211) channel on Discord to join our community! + +### Containers +* 🐳 [Docker](/deployment/docker) +* Kubernetes - Coming Soon! + +### Cloud Providers + +* [AWS](/deployment/aws) +* GCP - Coming Soon! +* Azure - Coming Soon! *** @@ -22,12 +31,7 @@ Chroma single-node is easy to deploy to a variety of cloud providers. Please use Running a server in production requires a few additional steps to ensure the server is secure and reliable. -{% special_table %} -{% /special_table %} - -| | -|--------------| -| [👀 Observability](/deployment/observability) | -| [✈️ Migration](/deployment/migration) | -| [🔒 Auth](/deployment/auth) | -| 🚧 *More Coming Soon* | +* [🚀 Performance](/deployment/performance) +* [👀 Observability](/deployment/observability) +* [✈️ Migration](/deployment/migration) +* [🔒 Auth](/deployment/auth) | diff --git a/docs/docs.trychroma.com/pages/deployment/performance.md b/docs/docs.trychroma.com/pages/deployment/performance.md index 609b952fe5c..28a2baf9e75 100644 --- a/docs/docs.trychroma.com/pages/deployment/performance.md +++ b/docs/docs.trychroma.com/pages/deployment/performance.md @@ -1,3 +1,131 @@ --- -title: Performance +title: "Single-Node Chroma: Performance and Limitations" --- + +The single-node version of Chroma is designed to be easy to deploy and maintain, while still providing robust performance that satisfies a broad range of production applications. + +To help you understand when single-node Chroma is a good fit for your use case, we have performed a series of stress tests and performance experiments to probe the system’s capabilities and discover its limitations and edge cases. We analyzed these boundaries across a range of hardware configurations, to determine what sort of deployment is appropriate for different workloads. + +This document describes these findings, as well as some general principles for getting the most out of your Chroma deployment. + +## Results Summary + +Roughly speaking, here is the sort of performance you can expect from Chroma on different EC2 instance types with a very typical workload: + +- 1024 dimensional embeddings +- Small documents (100-200 words) +- Three metadata fields per record. + +{% special_table %} +{% /special_table %} + +| Instance Type | System RAM | Approx. Max Collection Size | Mean Latency (insert) | 99.9% Latency (insert) | Mean Latency (query) | 99.9% Latency (query) | Monthly Cost | +|-----------------|------------|-----------------------------|-----------------------|------------------------|----------------------|-----------------------|--------------| +| **t3.small** | 2 | 250,000 | 55ms | 250ms | 22ms | 72ms | $15.936 | +| **t3.medium** | 4 | 700,000 | 37ms | 120ms | 14ms | 41ms | $31.072 | +| **t3.large** | 8 | 1,700,000 | 30ms | 100ms | 13ms | 35ms | $61.344 | +| **t3.xlarge** | 16 | 3,600,000 | 30ms | 100ms | 13ms | 30ms | $121.888 | +| **t3.2xlarge** | 32 | 7,500,000 | 30ms | 100ms | 13ms | 30ms | $242.976 | +| **r7i.2xlarge** | 64 | 15,000,000 | 13ms | 50ms | 7ms | 13ms | $386.944 | + +{% br %}{% /br %} + +Deploying Chroma on a system with less than 2GB of RAM is **not** recommended. + +Note that the latency figures in this table are for small collections. Latency increases as collections grow: see [Latency and collection size](./performance#latency-and-collection-size) below for a full analysis. + +## Memory and collection size + +Chroma uses a fork of [`hnswlib`](https://github.com/nmslib/hnswlib) to efficiently index and search over embedding vectors. The HNSW algorithm requires that the embedding index reside in system RAM to query or update. + +As such, the amount of available system memory defines an upper bound on the size of a Chroma collection (or multiple collections, if they are being used concurrently.) If a collection grows larger than available memory, insert and query latency spike rapidly as the operating system begins swapping memory to disk. The memory layout of the index is not amenable to swapping, and the system quickly becomes unusable. + +Therefore, users should always plan on having enough RAM provisioned to accommodate the anticipated total number of embeddings. + +To analyze how much RAM is required, we launched an an instance of Chroma on variously sized EC2 instances, then inserted embeddings until each system became non-responsive. As expected, this failure point corresponded linearly to RAM and embedding count. + +For 1024 dimensional embeddings, with three metadata records and a small document per embedding, this works out to `N = R * 0.245` where `N` is the max collection size in millions, and `R` is the amount of system RAM required in gigabytes. Remember, you wil also need reserve at least a gigabyte for the system’s other needs, in addition to the memory required by Chroma. + +This pattern holds true up through about 7 million embeddings, which is as far as we tested. At this point Chroma is still fast and stable, and we did not find a strict upper bound on the size of a Chroma database. + +## Disk space and collection size + +Chroma durably persists each collection to disk. The amount of space required is a combination of the space required to save the HNSW embedding index, and the space required by the sqlite database used to store documents and embedding metadata. + +The calculations for persisting the HNSW index are similar to that for calculating RAM size. As a rule of thumb, just make sure a system’s storage is at least as big as its RAM, plus several gigabytes to account for the overhead of the operating system and other applications. + +The amount of space required by the sqlite database is highly variable, and depends entirely on whether documents and metadata are being saved in Chroma, and if so, how large they are. Fully exploring all permutations of this are beyond the scope of the experiments we were able to run. + +However, as a single data point, the sqlite database for a collection with ~40k documents of 1000 words each, and ~600k metadata entries was about 1.7gb. + +There is no strict upper bound on the size of the metadata database: sqlite itself supports databases into the terabyte range, and can page to disk effectively. + +In most realistic use cases, it’s likely that the size and performance of the HNSW index in RAM becomes the limiting factor on a Chroma collection’s size long before the metadata database does. + +## Latency and collection size + +As collections get larger and the size of the index grows, inserts and queries both take longer to complete. The rate of increase starts out fairly flat then grow roughly linearly, with the inflection point and slope depending on the quantity and speed of CPUs available. + +### Query Latency + +![query-latency](/img/query-latency.png) + +### Insert Latency + +![insert-latency](/img/insert-latency.png) + +{% note type="tip" title="" %} +If you’re using multiple collections, performance looks quite similar, based on the total number of embeddings across collections. Splitting collections into multiple smaller collections doesn’t help, but it doesn’t hurt, either, as long as they all fit in memory at once. +{% /note %} + +## Concurrency + +Although aspects of HNSW’s algorithm are multithreaded internally, only one thread can read or write to a given index at a time. For the most part, single-node Chroma is fundamentally single threaded. If a operation is executed while another is still in progress, it blocks until the first one is complete. + +This means that under concurrent load, the average latency of each request will increase. + +When writing, the increased latency is more pronounced with larger batch sizes, as the system is more completely saturated. We have experimentally verified this: as the number of concurrent writers is increased, average latency increases linearly. + +![concurrent-writes](/img/concurrent-writes.png) + +![concurrent-queries](/img/concurrent-queries.png) + +Despite the effect on latency, Chroma does remain stable with high concurrent load. Too many concurrent users can eventually increase latency to the point where the system does not perform acceptably, but this typically only happens with larger batch sizes. As the above graphs shows, the system remains usable with dozens to hundreds of operations per second. + +See the [Insert Throughput](./performance#insert-throughput) section below for a discussion of optimizing user count for maximum throughput when the concurrency is under your control, such as when inserting bulk data. + +# CPU speed, core count & type + +As a CPU bound application, it’s not surprising that CPU speed and type makes a difference for average latency. + +As the data demonstrates, although it is not fully parallelized, Chroma can still take some advantage of multiple CPU cores for better throughput. + +![cpu-mean-query-latency](/img/cpu-mean-query-latency.png) + +{% note type="tip" title="" %} +Note the slightly increased latency for the t3.2xlarge instance. Logically, it should be faster than the other t3 series instances, since it has the same class of CPU, and more of them. + +This data point is left in as an important reminder that the performance of EC2 instances is slightly variable, and it’s entirely possible to end up with an instance that has performance differences for no discernible reason. +{% /note %} + +# Insert Throughput + +A question that is often relevant is: given bulk data to insert, how fast is it possible to do so, and what’s the best way to insert a lot of data quickly? + +The first important factor to consider is the number of concurrent insert requests. + +As mentioned in the [Concurrency](./performance#concurrency) section above, actual insertion throughput does not benefit from concurrency. However, there is some amount of network and HTTP overhead which can be parallelized. Therefore, to saturate Chroma while keeping latencies as low as possible, we recommend 2 concurrent client processes or threads inserting as fast as possible. + +The second factor to consider is the batch size of each request. Performance is mostly linear with respect to batch size, with a constant overhead to process the HTTP request itself. + +Experimentation confirms this: overall throughput (total number of embeddings inserted, across batch size and request count) remains fairly flat between batch sizes of 100-500: + +![concurrent-inserts](/img/concurrent-inserts.png) + +Given that smaller batches have lower, more consistent latency and are less likely to lead to timeout errors, we recommend batches on the smaller side of this curve: anything between 50 and 250 is a reasonable choice. + +## Conclusion + +Users should feel comfortable relying on Chroma for use cases approaching tens of millions of embeddings, when deployed on the right hardware. It’s average and upper-bound latency for both reads and writes make it a good platform for all but the largest AI-based applications, supporting potentially thousands of simultaneous human users (depending on your application’s backend access patterns.) + +As a single-node solution, though, it won’t scale forever. If you find your needs exceeding the parameters laid out in this analysis, we are extremely interested in hearing from you. Please fill out [this form](https://airtable.com/appqd02UuQXCK5AuY/pagr1D0NFQoNpUpNZ/form), and we will add you to a dedicated Slack workspace for supporting production users. We would love to help you think through the design of your system, whether Chroma has a place in it, or if you would be a good fit for our upcoming distributed cloud service. You can also join the [#production-chroma](https://discord.com/channels/1073293645303795742/1292554909694300211) channel on Discord to join our community! diff --git a/docs/docs.trychroma.com/pages/deployment/thin-client.md b/docs/docs.trychroma.com/pages/deployment/thin-client.md new file mode 100644 index 00000000000..47e8a6b7f8a --- /dev/null +++ b/docs/docs.trychroma.com/pages/deployment/thin-client.md @@ -0,0 +1,27 @@ +--- +title: "Chroma's Thin-Client" +--- + +If you are running Chroma in client-server mode in a Python application, you may not need the full Chroma Python library. Instead, you can use the lightweight client-only library. +In this case, you can install the `chromadb-client` package **instead** of our `chromadb` package. + +The `chromadb-client` package is a lightweight HTTP client for the server with a minimal dependency footprint. + + +```python +pip install chromadb-client +``` + +```python +import chromadb +# Example setup of the client to connect to your chroma server +client = chromadb.HttpClient(host='localhost', port=8000) + +# Or for async usage: +async def main(): + client = await chromadb.AsyncHttpClient(host='localhost', port=8000) +``` + +Note that the `chromadb-client` package is a subset of the full Chroma library and does not include all the dependencies. If you want to use the full Chroma library, you can install the `chromadb` package instead. + +Most importantly, the thin-client package has no default embedding functions. If you `add()` documents without embeddings, you must have manually specified an embedding function and installed the dependencies for it. diff --git a/docs/docs.trychroma.com/pages/getting-started.md b/docs/docs.trychroma.com/pages/getting-started.md index f3214253fc3..8bc3cad1f9f 100644 --- a/docs/docs.trychroma.com/pages/getting-started.md +++ b/docs/docs.trychroma.com/pages/getting-started.md @@ -50,7 +50,7 @@ pnpm add chromadb chromadb-default-embed # [!code $] {% /codetab %} {% /codetabs %} -Install chroma via `pypi` to easily run the backend server. (Docker also available) +Install chroma via `pypi` to easily run the backend server. ([Docker](./deployment/docker) also available) ```bash pip install chromadb # [!code $] diff --git a/docs/docs.trychroma.com/pages/guides/index.md b/docs/docs.trychroma.com/pages/guides/index.md index ac13e1fb035..54ec5eb2243 100644 --- a/docs/docs.trychroma.com/pages/guides/index.md +++ b/docs/docs.trychroma.com/pages/guides/index.md @@ -84,7 +84,7 @@ await client.reset() # Empties and completely resets the database. ⚠️ This i Chroma can also be configured to run in client/server mode. In this mode, the Chroma client connects to a Chroma server running in a separate process. -To start the Chroma server, run the following command: +To start the Chroma server locally, run the following command: ```bash chroma run --path /db_path @@ -165,7 +165,7 @@ To run Chroma in client server mode, first install the chroma library and CLI vi pip install chromadb ``` -Then start the Chroma server: +Then start the Chroma server locally: ```bash chroma run --path /db_path @@ -183,7 +183,7 @@ import { ChromaClient } from "chromadb"; const client = new ChromaClient(); ``` -You can also run the Chroma server in a docker container, or deployed to a cloud provider. See the [deployment docs](./deployment.md) for more information. +You can also run the Chroma server in a docker container, or deployed to a cloud provider. See the [deployment docs](./deployment/docker) for more information. {% /tab %} diff --git a/docs/docs.trychroma.com/public/img/concurrent-inserts.png b/docs/docs.trychroma.com/public/img/concurrent-inserts.png new file mode 100644 index 00000000000..79c937c60df Binary files /dev/null and b/docs/docs.trychroma.com/public/img/concurrent-inserts.png differ diff --git a/docs/docs.trychroma.com/public/img/concurrent-queries.png b/docs/docs.trychroma.com/public/img/concurrent-queries.png new file mode 100644 index 00000000000..e234c901a42 Binary files /dev/null and b/docs/docs.trychroma.com/public/img/concurrent-queries.png differ diff --git a/docs/docs.trychroma.com/public/img/concurrent-writes.png b/docs/docs.trychroma.com/public/img/concurrent-writes.png new file mode 100644 index 00000000000..279f4cd110b Binary files /dev/null and b/docs/docs.trychroma.com/public/img/concurrent-writes.png differ diff --git a/docs/docs.trychroma.com/public/img/cpu-mean-query-latency.png b/docs/docs.trychroma.com/public/img/cpu-mean-query-latency.png new file mode 100644 index 00000000000..987dd142f33 Binary files /dev/null and b/docs/docs.trychroma.com/public/img/cpu-mean-query-latency.png differ diff --git a/docs/docs.trychroma.com/public/img/insert-latency.png b/docs/docs.trychroma.com/public/img/insert-latency.png new file mode 100644 index 00000000000..54cdf3de550 Binary files /dev/null and b/docs/docs.trychroma.com/public/img/insert-latency.png differ diff --git a/docs/docs.trychroma.com/public/img/query-latency.png b/docs/docs.trychroma.com/public/img/query-latency.png new file mode 100644 index 00000000000..f83d27efa79 Binary files /dev/null and b/docs/docs.trychroma.com/public/img/query-latency.png differ