diff --git a/.bumpversion.cfg b/.bumpversion.cfg index eacf6f05c049..739f28cd0184 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -19,3 +19,7 @@ serialize = [bumpversion:file:kube/overlays/stable/.env] [bumpversion:file:kube/overlays/stable/kustomization.yaml] + +[bumpversion:file:kube/overlays/stable-with-resource-limits/.env] + +[bumpversion:file:kube/overlays/stable-with-resource-limits/kustomization.yaml] diff --git a/docs/deploying-airbyte/on-kubernetes.md b/docs/deploying-airbyte/on-kubernetes.md index 259719863165..8c5627a5962f 100644 --- a/docs/deploying-airbyte/on-kubernetes.md +++ b/docs/deploying-airbyte/on-kubernetes.md @@ -10,7 +10,7 @@ Airbyte allows scaling sync workloads horizontally using Kubernetes. The core co For local testing we recommend following one of the following setup guides: * [Docker Desktop (Mac)](https://docs.docker.com/desktop/kubernetes/) * [Minikube](https://minikube.sigs.k8s.io/docs/start/) - * NOTE: Start Minikube with at least 4gb RAM to Minikube with `minikube start --memory=4000` + * NOTE: Start Minikube with at least 4gb RAM with `minikube start --memory=4000` * [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) For testing on GKE you can [create a cluster with the command line or the Cloud Console UI](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster). @@ -135,6 +135,17 @@ Now visit [http://localhost:8000](http://localhost:8000) in your browser and sta ## Production Airbyte on Kubernetes +### Setting resource limits + +* Core container pods + * Instead of launching Airbyte with `kubectl apply -k kube/overlays/stable`, you can run with `kubectl apply -k kube/overlays/stable-with-resource-limits`. + * The `kube/overlays/stable-with-resource-limits/set-resource-limits.yaml` file can be modified to provide different resource requirements for core pods. +* Connector pods + * By default, connector pods launch without resource limits. + * To add resource limits, configure the "Docker Resource Limits" section of the `.env` file in the overlay folder you're using. +* Volume sizes + * You can modify `kube/resources/volume-*` files to specify different volume sizes for the persistent volumes backing Airbyte. + ### Cloud logging Airbyte writes logs to two directories. App logs, including server and scheduler logs, are written to the `app-logging` directory. @@ -163,10 +174,6 @@ there are any other issues blocking your adoption of Airbyte or if you would lik * The server and scheduler deployments must run on the same node. ([#4232](https://github.com/airbytehq/airbyte/issues/4232)) * Some UI operations have higher latency on Kubernetes than Docker-Compose. ([#4233](https://github.com/airbytehq/airbyte/issues/4233)) -* Pod histories must be cleaned up manually. ([#3634](https://github.com/airbytehq/airbyte/issues/3634)) -* Specifying resource limits for pods is not supported yet. ([#3638](https://github.com/airbytehq/airbyte/issues/3638)) -* Pods Airbyte launches to run connector jobs are always launched in the `default` namespace. ([#3636](https://github.com/airbytehq/airbyte/issues/3636)) -* S3 is the only Cloud Storage currently supported. ([#4200](https://github.com/airbytehq/airbyte/issues/4200)) * Large log files might take a while to load. ([#4201](https://github.com/airbytehq/airbyte/issues/4201)) * UI does not include configured buckets in the displayed log path. ([#4204](https://github.com/airbytehq/airbyte/issues/4204)) * Logs are not reset when Airbyte is re-deployed. ([#4235](https://github.com/airbytehq/airbyte/issues/4235)) diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env new file mode 100644 index 000000000000..c4c811c5151c --- /dev/null +++ b/kube/overlays/stable-with-resource-limits/.env @@ -0,0 +1,55 @@ +AIRBYTE_VERSION=0.27.2-alpha + +# Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +DATABASE_USER=docker +DATABASE_PASSWORD=docker +DATABASE_HOST=airbyte-db-svc +DATABASE_PORT=5432 +DATABASE_DB=airbyte +# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte + +# When using the airbyte-db via default docker image: +CONFIG_ROOT=/configs +DATA_DOCKER_MOUNT=airbyte_data +DB_DOCKER_MOUNT=airbyte_db + +# Temporal.io worker configuration +TEMPORAL_HOST=airbyte-temporal-svc:7233 +TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030 + +# Workspace storage for running jobs (logs, etc) +WORKSPACE_ROOT=/workspace +WORKSPACE_DOCKER_MOUNT=airbyte_workspace + +LOCAL_ROOT=/tmp/airbyte_local + +# Miscellaneous +TRACKING_STRATEGY=segment +WEBAPP_URL=airbyte-webapp-svc:80 +API_URL=/api/v1/ +INTERNAL_API_HOST=airbyte-server-svc:8001 + +WORKER_ENVIRONMENT=kubernetes +PAPERCUPS_STORYTIME=enabled +FULLSTORY=enabled +IS_DEMO=false +LOG_LEVEL=INFO + +# S3/Minio Log Configuration +S3_LOG_BUCKET=airbyte-dev-logs +S3_LOG_BUCKET_REGION=us-east-1 +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=minio123 +S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +S3_PATH_STYLE_ACCESS=true + +# GCS Log Configuration +GCP_STORAGE_BUCKET= +GOOGLE_APPLICATION_CREDENTIALS= + +# Docker Resource Limits +RESOURCE_CPU_REQUEST= +RESOURCE_CPU_LIMIT= +RESOURCE_MEMORY_REQUEST= +RESOURCE_MEMORY_LIMIT= diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml new file mode 100644 index 000000000000..8b24d5430bf5 --- /dev/null +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: default + +bases: + - ../../resources + +images: + - name: airbyte/seed + newTag: 0.27.2-alpha + - name: airbyte/db + newTag: 0.27.2-alpha + - name: airbyte/scheduler + newTag: 0.27.2-alpha + - name: airbyte/server + newTag: 0.27.2-alpha + - name: airbyte/webapp + newTag: 0.27.2-alpha + - name: temporalio/auto-setup + newTag: 1.7.0 + +configMapGenerator: + - name: airbyte-env + env: .env + +patchesStrategicMerge: + - set-resource-limits.yaml diff --git a/kube/overlays/stable/set-resource-limits.yaml b/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml similarity index 97% rename from kube/overlays/stable/set-resource-limits.yaml rename to kube/overlays/stable-with-resource-limits/set-resource-limits.yaml index 90b48d4c3bb4..4916de45ee26 100644 --- a/kube/overlays/stable/set-resource-limits.yaml +++ b/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml @@ -10,7 +10,7 @@ spec: resources: limits: cpu: 2 - memory: 4Gi + memory: 2Gi --- apiVersion: apps/v1 kind: Deployment diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index 3024c25da163..c2a3da630e93 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -23,6 +23,3 @@ images: configMapGenerator: - name: airbyte-env env: .env - -patchesStrategicMerge: - - set-resource-limits.yaml