diff --git a/.github/actions/build-and-push-branch/action.yml b/.github/actions/build-and-push-branch/action.yml index c1c5f3a10893..dd983b79aeff 100644 --- a/.github/actions/build-and-push-branch/action.yml +++ b/.github/actions/build-and-push-branch/action.yml @@ -13,6 +13,16 @@ inputs: runs: using: "composite" steps: + - name: Delete default old docker and replace it with a new one + shell: bash + run: | + sudo rm /var/lib/dpkg/lock + sudo rm /var/lib/dpkg/lock-frontend + sudo apt-get remove docker.io || sudo apt-get remove docker + curl -fsSL https://get.docker.com | bash - + sudo rm -f /var/lib/dpkg/lock + sudo rm -f /var/lib/dpkg/lock-frontend + - name: Build id: build uses: ./.github/actions/build-branch @@ -29,5 +39,5 @@ runs: run: | GIT_REVISION=$(git rev-parse HEAD) [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1 - VERSION=${{ steps.build.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose-cloud.buildx.yaml push + VERSION=${{ steps.build.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker compose -f docker-compose-cloud.buildx.yaml push shell: bash diff --git a/.github/workflows/commands-for-testing-tool.yml b/.github/workflows/commands-for-testing-tool.yml index bdaa9464cdc0..460d8e4872f1 100644 --- a/.github/workflows/commands-for-testing-tool.yml +++ b/.github/workflows/commands-for-testing-tool.yml @@ -110,7 +110,7 @@ jobs: mkdir secrets mkdir result - name: Run Airbyte - run: docker-compose up -d + run: docker compose up -d - name: Connect to secret manager uses: jsdaniell/create-json@1.1.2 with: diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1d1fbcc6ed16..fedc29169aa3 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -467,6 +467,16 @@ jobs: with: python-version: "3.9" + - name: Delete default old docker and replace it with a new one + shell: bash + run: | + sudo rm /var/lib/dpkg/lock + sudo rm /var/lib/dpkg/lock-frontend + sudo apt-get remove docker.io || sudo apt-get remove docker + curl -fsSL https://get.docker.com | bash - + sudo rm -f /var/lib/dpkg/lock + sudo rm -f /var/lib/dpkg/lock-frontend + - name: Set up CI Gradle Properties run: | mkdir -p ~/.gradle/ @@ -592,6 +602,16 @@ jobs: with: node-version: "lts/*" + - name: Delete default old docker and replace it with a new one + shell: bash + run: | + sudo rm /var/lib/dpkg/lock + sudo rm /var/lib/dpkg/lock-frontend + sudo apt-get remove docker.io || sudo apt-get remove docker + curl -fsSL https://get.docker.com | bash - + sudo rm -f /var/lib/dpkg/lock + sudo rm -f /var/lib/dpkg/lock-frontend + - name: Set up CI Gradle Properties run: | mkdir -p ~/.gradle/ diff --git a/README.md b/README.md index a22ea084e77c..2ed7f770f812 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ You can run Airbyte locally with Docker. ```bash git clone https://github.com/airbytehq/airbyte.git cd airbyte -docker-compose up +docker compose up ``` Login to the web app at [http://localhost:8000](http://localhost:8000) by entering the default credentials found in your .env file. diff --git a/airbyte-integrations/connectors/source-zuora/README.md b/airbyte-integrations/connectors/source-zuora/README.md index 5924b61fdfe2..c6f033db98a2 100644 --- a/airbyte-integrations/connectors/source-zuora/README.md +++ b/airbyte-integrations/connectors/source-zuora/README.md @@ -130,7 +130,7 @@ To run acceptance and custom integration tests: # Run Airbyte using docker-compose Under Airbyte's root directory: ``` -docker-compose up -d +docker compose up -d ``` Open the web-page of Airbyte: ``` diff --git a/airbyte-webapp-e2e-tests/README.md b/airbyte-webapp-e2e-tests/README.md index c0014ec5982a..0eaedd66587e 100644 --- a/airbyte-webapp-e2e-tests/README.md +++ b/airbyte-webapp-e2e-tests/README.md @@ -10,7 +10,7 @@ Steps: 2) Build the OSS backend for the current commit with `SUB_BUILD=PLATFORM ../gradlew clean build`. 3) Create the test database: `npm run createdbsource` and `npm run createdbdestination`. 4) When running the connector builder tests, start the dummy API server: `npm run createdummyapi` -5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker-compose --file ../docker-compose.yaml up`. If you want, follow this with `docker-compose stop webapp` to turn off the dockerized frontend build; interactive cypress sessions don't use it. +5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker compose --file ../docker-compose.yaml up`. If you want, follow this with `docker compose stop webapp` to turn off the dockerized frontend build; interactive cypress sessions don't use it. 6) The following two commands will start a separate long-running server, so open another terminal window. In it, `cd` into the `airbyte-webapp/` directory. 7) If you have not already done so, run `npm install` to install the frontend app's dependencies. 8) Start the frontend development server with `npm start`. @@ -26,7 +26,7 @@ Steps: 2) Build the OSS backend for the current commit with `SUB_BUILD=PLATFORM ../gradlew clean build`. 3) Create the test database: `npm run createdbsource` and `npm run createdbdestination`. 4) When running the connector builder tests, start the dummy API server: `npm run createdummyapi` -5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker-compose --file ../docker-compose.yaml up`. +5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker compose --file ../docker-compose.yaml up`. 6) Start the cypress test run with `npm run cypress:ci` or `npm run cypress:ci:record`. ## Test setup @@ -35,4 +35,4 @@ When the tests are run as described above, the platform under test is started vi For testing the connector builder UI, a dummy api server based on a node script is started (`createdummyapi`). It is providing a simple http API with bearer authentication returning a few records of hardcoded data. By running it in the internal airbyte network, the connector builder server can access it under its container name. -The tests in here are instrumenting a Chrome instance to test the full functionality of Airbyte from the frontend, so other components of the platform (scheduler, worker, connector builder server) are also tested in a rudimentary way. \ No newline at end of file +The tests in here are instrumenting a Chrome instance to test the full functionality of Airbyte from the frontend, so other components of the platform (scheduler, worker, connector builder server) are also tested in a rudimentary way. diff --git a/docker-compose-cloud.buildx.yaml b/docker-compose-cloud.buildx.yaml index 3f725f5bb0a6..cbd8ced9ea7c 100644 --- a/docker-compose-cloud.buildx.yaml +++ b/docker-compose-cloud.buildx.yaml @@ -1,7 +1,7 @@ # Defines the minimum set of images needed to run Cloud. # Used to push OSS images that Cloud depends on. -version: "3.7" +version: "3.8" services: bootloader: diff --git a/docker-compose.acceptance-test.yaml b/docker-compose.acceptance-test.yaml index 5366ea1ba112..f9a1486f44f6 100644 --- a/docker-compose.acceptance-test.yaml +++ b/docker-compose.acceptance-test.yaml @@ -1,7 +1,7 @@ # Adds ports to the db and access to the temporal UI for debugging purposes. # Expected to be used like this: -# VERSION=dev docker-compose -f docker-compose.yaml -f docker-compose.debug.yaml up -version: "3.7" +# VERSION=dev docker compose -f docker-compose.yaml -f docker-compose.debug.yaml up +version: "3.8" x-logging: &default-logging options: max-size: "100m" diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml index 0e509dca55ad..31132e06ac21 100644 --- a/docker-compose.build.yaml +++ b/docker-compose.build.yaml @@ -1,4 +1,4 @@ -version: "3.7" +version: "3.8" services: init: diff --git a/docker-compose.datadog.yaml b/docker-compose.datadog.yaml index 0463bfcb2df4..db67e7e89d2a 100644 --- a/docker-compose.datadog.yaml +++ b/docker-compose.datadog.yaml @@ -5,7 +5,7 @@ # 1. create an API Key in datadog # 2. wget -O dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' # 3. DD_API_KEY=[datadog api key] VERSION=dev docker-compose -f docker-compose.yaml -f docker-compose.datadog.yaml up -d -version: "3.7" +version: "3.8" x-datadog: &datadogged volumes: diff --git a/docker-compose.debug.yaml b/docker-compose.debug.yaml index 1ba4e41e7737..a93fa0b4b286 100644 --- a/docker-compose.debug.yaml +++ b/docker-compose.debug.yaml @@ -1,7 +1,7 @@ # Adds ports to the db and access to the temporal UI for debugging purposes. # Expected to be used like this: -# VERSION=dev docker-compose -f docker-compose.yaml -f docker-compose.debug.yaml up -version: "3.7" +# VERSION=dev docker compose -f docker-compose.yaml -f docker-compose.debug.yaml up +version: "3.8" x-logging: &default-logging options: max-size: "100m" diff --git a/docker-compose.yaml b/docker-compose.yaml index 0e43d41cb58f..f01aef60365f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,4 @@ -version: "3.7" +version: "3.8" #https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments x-logging: &default-logging options: @@ -30,6 +30,9 @@ services: - LOG_LEVEL=${LOG_LEVEL} networks: - airbyte_internal + depends_on: + init: + condition: service_completed_successfully db: image: airbyte/db:${VERSION} logging: *default-logging @@ -114,6 +117,9 @@ services: - 9000 networks: - airbyte_internal + depends_on: + bootloader: + condition: service_completed_successfully server: image: airbyte/server:${VERSION} logging: *default-logging @@ -156,6 +162,9 @@ services: - ${LOCAL_ROOT}:${LOCAL_ROOT} networks: - airbyte_internal + depends_on: + bootloader: + condition: service_completed_successfully webapp: image: airbyte/webapp:${VERSION} logging: *default-logging @@ -175,6 +184,9 @@ services: - TRACKING_STRATEGY=${TRACKING_STRATEGY} networks: - airbyte_internal + depends_on: + bootloader: + condition: service_completed_successfully airbyte-temporal: image: airbyte/temporal:${VERSION} logging: *default-logging @@ -214,6 +226,9 @@ services: - workspace:${WORKSPACE_ROOT} networks: - airbyte_internal + depends_on: + bootloader: + condition: service_completed_successfully airbyte-connector-builder-server: image: airbyte/connector-builder-server:${VERSION} logging: *default-logging @@ -225,6 +240,9 @@ services: - AIRBYTE_VERSION=${VERSION} networks: - airbyte_internal + depends_on: + bootloader: + condition: service_completed_successfully airbyte-proxy: image: airbyte/proxy:${VERSION} container_name: airbyte-proxy diff --git a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md index 38845f67a560..f230b5dc2c14 100644 --- a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md +++ b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md @@ -1078,7 +1078,7 @@ $ docker images | head If the Airbyte server isn't already running, start it by running **from the Airbyte repository root**: ```bash -docker-compose up +docker compose up ``` When Airbyte server is done starting up, it prints the following banner in the log output \(it can take 10-20 seconds for the server to start\): @@ -1098,7 +1098,7 @@ airbyte-server | Version: dev airbyte-server | ``` -After you see the above banner printed out in the terminal window where you are running `docker-compose up`, visit [http://localhost:8000](http://localhost:8000) in your browser and log in with the default credentials: username `airbyte` and password `password`. +After you see the above banner printed out in the terminal window where you are running `docker compose up`, visit [http://localhost:8000](http://localhost:8000) in your browser and log in with the default credentials: username `airbyte` and password `password`. If this is the first time using the Airbyte UI, then you will be prompted to go through a first-time wizard. To skip it, click the "Skip Onboarding" button. diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 23350c4f730e..1f601e00c0d1 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -73,7 +73,7 @@ These instructions explain how to run a version of Airbyte that you are developi ```bash SUB_BUILD=PLATFORM ./gradlew build -VERSION=dev docker-compose up +VERSION=dev docker compose up ``` The build will take a few minutes. Once it completes, Airbyte compiled at current git revision will be running in `dev` mode in your environment. @@ -88,7 +88,7 @@ These instructions explain how to run a version of an Airbyte connector that you ```bash SUB_BUILD=PLATFORM ./gradlew build -VERSION=dev docker-compose up +VERSION=dev docker compose up ``` - Then, build the connector image: @@ -169,7 +169,7 @@ Note: If you are contributing a Python file without imports or function definiti - Spin up Airbyte locally so the UI can make requests against the local API. ```bash -BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" docker-compose up +BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" docker compose up ``` Note: [basic auth](https://docs.airbyte.com/operator-guides/security#network-security) must be disabled by setting `BASIC_AUTH_USERNAME` and `BASIC_AUTH_PASSWORD` to empty values, otherwise requests from the development server will fail against the local API. @@ -192,8 +192,8 @@ The Configuration API caches connector specifications. This is done to avoid nee 2. Restart the server by running the following commands: ```bash -VERSION=dev docker-compose down -v -VERSION=dev docker-compose up +VERSION=dev docker compose down -v +VERSION=dev docker compose up ``` ### Resetting the Airbyte developer environment @@ -203,7 +203,7 @@ Sometimes you'll want to reset the data in your local environment. One common ca - Delete the datastore volumes in docker ```bash - VERSION=dev docker-compose down -v + VERSION=dev docker compose down -v ``` - Remove the data on disk @@ -217,7 +217,7 @@ Sometimes you'll want to reset the data in your local environment. One common ca ```bash SUB_BUILD=PLATFORM ./gradlew clean build - VERSION=dev docker-compose up -V + VERSION=dev docker compose up -V ``` While not as common as the above steps, you may also get into a position where want to erase all of the data on your local docker server. This is useful if you've been modifying image tags while developing. @@ -257,4 +257,4 @@ env JAVA_HOME=/usr/lib/jvm/java-14-openjdk ./gradlew :airbyte-integrations:conn ### Inspecting the messages passed between connectors -You can enable `LOG_CONNECTOR_MESSAGES=true` to log the messages the Airbyte platform receives from the source and destination when debugging locally. e.g. `LOG_CONNECTOR_MESSAGES=true VERSION=dev docker-compose up` +You can enable `LOG_CONNECTOR_MESSAGES=true` to log the messages the Airbyte platform receives from the source and destination when debugging locally. e.g. `LOG_CONNECTOR_MESSAGES=true VERSION=dev docker compose up` diff --git a/docs/deploying-airbyte/local-deployment.md b/docs/deploying-airbyte/local-deployment.md index 65153e46f5e7..89fccd9a5fa2 100644 --- a/docs/deploying-airbyte/local-deployment.md +++ b/docs/deploying-airbyte/local-deployment.md @@ -13,7 +13,7 @@ These instructions have been tested on MacOS, Windows 10 and Ubuntu 20.04. ```bash git clone https://github.com/airbytehq/airbyte.git cd airbyte -docker-compose up +docker compose up ``` - In your browser, just visit [http://localhost:8000](http://localhost:8000) @@ -53,7 +53,7 @@ Make sure to select the options: ```bash git clone https://github.com/airbytehq/airbyte.git cd airbyte -docker-compose up +docker compose up ``` - In your browser, just visit [http://localhost:8000](http://localhost:8000) diff --git a/docs/deploying-airbyte/on-aws-ec2.md b/docs/deploying-airbyte/on-aws-ec2.md index 7c982a3b4c8a..441f0b8e2d9c 100644 --- a/docs/deploying-airbyte/on-aws-ec2.md +++ b/docs/deploying-airbyte/on-aws-ec2.md @@ -38,9 +38,8 @@ sudo usermod -a -G docker $USER 3. To install `docker-compose`, run the following command in your ssh session on the instance terminal: ``` bash -sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose -sudo chmod +x /usr/local/bin/docker-compose -docker-compose --version +sudo yum install -y docker-compose-plugin +docker compose version ``` 4. To close the SSH connection, run the following command in your SSH session on the instance terminal: @@ -64,7 +63,7 @@ ssh -i $SSH_KEY ec2-user@$INSTANCE_IP ``` bash mkdir airbyte && cd airbyte wget https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml} -docker-compose up -d # run the Docker container +docker compose up -d # run the Docker container ``` ## Connect to Airbyte diff --git a/docs/deploying-airbyte/on-azure-vm-cloud-shell.md b/docs/deploying-airbyte/on-azure-vm-cloud-shell.md index 0b6ac3a3ad9e..9926555ba311 100644 --- a/docs/deploying-airbyte/on-azure-vm-cloud-shell.md +++ b/docs/deploying-airbyte/on-azure-vm-cloud-shell.md @@ -36,14 +36,13 @@ Install Docker and Docker Compose in the VM: 4. To install Docker Compose, run the following command: ```bash - sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose + sudo apt-get install docker-compose-plugin -y ``` 5. Check Docker Compose version: ```bash - docker-compose --version + docker compose version ``` 6. Close the SSH connection to ensure that the group modification is considered: @@ -84,7 +83,7 @@ Download Airbyte and deploy it in the VM using Docker Compose: 4. To start Airbyte, run the following command: ```bash - sudo docker-compose up -d + sudo docker compose up -d ``` ## Connect to Airbyte diff --git a/docs/deploying-airbyte/on-digitalocean-droplet.md b/docs/deploying-airbyte/on-digitalocean-droplet.md index a14ead2affc6..b9adf4d99eff 100644 --- a/docs/deploying-airbyte/on-digitalocean-droplet.md +++ b/docs/deploying-airbyte/on-digitalocean-droplet.md @@ -31,9 +31,8 @@ To deploy Airbyte Open Source on DigitalOcean: 4. To install Docker-Compose, run the following command: ```bash - sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version + sudo apt install docker-compose-plugin + docker compose version ``` ## Install Airbyte @@ -45,7 +44,7 @@ To install and start Airbyte : ```bash mkdir airbyte && cd airbyte wget https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml} - docker-compose up -d + docker compose up -d ``` 2. Verify the connection by visiting [http://localhost:8000](http://localhost:8000) in your browser. diff --git a/docs/deploying-airbyte/on-gcp-compute-engine.md b/docs/deploying-airbyte/on-gcp-compute-engine.md index a3eac4e37d0f..7579ecca8f46 100644 --- a/docs/deploying-airbyte/on-gcp-compute-engine.md +++ b/docs/deploying-airbyte/on-gcp-compute-engine.md @@ -59,10 +59,8 @@ sudo usermod -a -G docker $USER 7. Install `docker-compose` on your VM instance by following the below commands in your VM terminal: ```bash -sudo apt-get -y install wget -sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose -sudo chmod +x /usr/local/bin/docker-compose -docker-compose --version +sudo apt-get -y install docker-compose-plugin +docker compose version ``` 8. Close the SSH connection on your VM instance to ensure the group modification is taken into account by following the below command in your VM terminal: @@ -86,7 +84,7 @@ gcloud --project=$PROJECT_ID beta compute ssh $INSTANCE_NAME ```bash mkdir airbyte && cd airbyte wget https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml} -docker-compose up -d +docker compose up -d ``` ## Connect to Airbyte diff --git a/docs/deploying-airbyte/on-oci-vm.md b/docs/deploying-airbyte/on-oci-vm.md index 8b431d4d0364..77fa22e6efb0 100644 --- a/docs/deploying-airbyte/on-oci-vm.md +++ b/docs/deploying-airbyte/on-oci-vm.md @@ -45,11 +45,9 @@ sudo usermod -a -G docker $USER In the terminal connected to your OCI Instance for Airbyte, run the following commands: ```bash -sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose +sudo yum install -y docker-compose-plugin -sudo chmod +x /usr/local/bin/docker-compose - -sudo /usr/local/bin/docker-compose --version +docker compose version ``` ## Install and start Airbyte @@ -68,9 +66,9 @@ Download the Airbyte repository and deploy it on the VM: ```bash - which docker-compose + which docker - sudo /usr/local/bin/docker-compose up -d + sudo /usr/local/bin/docker compose up -d ``` diff --git a/docs/operator-guides/collecting-metrics.md b/docs/operator-guides/collecting-metrics.md index 03013efb75a6..a6b01edec363 100644 --- a/docs/operator-guides/collecting-metrics.md +++ b/docs/operator-guides/collecting-metrics.md @@ -45,7 +45,7 @@ Run the following commands to have it up and running. 2. Change `OTEL_COLLECTOR_ENDPOINT` to `"http://host.docker.internal:4317"` because Open Telemetry Collector has enabled port forward from localhost:4317 to container port 4317. To send data to Collector container port 4317, we want to need to export data to physical machine's localhost:4317, which in docker will be represented as `http://host.docker.internal:4317`. > Do *not* use `localhost:4317` or you will send data to the same container where Airbyte Worker is running. - 3. Start Airbyte server by running `docker-compose up` under airbyte repository. Go to `localhost:8000` to visit Airbyte and start a sync, then go to `localhost:9090` to access Prometheus - you should be able to see the metrics there. Alternatively, + 3. Start Airbyte server by running `docker compose up` under airbyte repository. Go to `localhost:8000` to visit Airbyte and start a sync, then go to `localhost:9090` to access Prometheus - you should be able to see the metrics there. Alternatively, ### Run Opentelemetry and Airbyte on kubernetes diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 3bac571ffeeb..d6ded79547b3 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -31,10 +31,16 @@ If you use custom connectors, this upgrade requires all of your connector specs ## Upgrading on Docker +:::note + +Airbyte version 0.40.27 or later requires [Docker Compose V2](https://docs.docker.com/compose/compose-v2/) to be [installed](https://docs.docker.com/compose/install/) before upgrading. + +::: + 1. In a terminal, on the host where Airbyte is running, turn off Airbyte. ```bash - docker-compose down + docker compose down ``` 2. Upgrade the docker instance to new version. @@ -46,7 +52,7 @@ If you use custom connectors, this upgrade requires all of your connector specs 3. Bring Airbyte back online. ```bash - docker-compose up + docker compose up ``` ### Resetting your Configuration diff --git a/docs/project-overview/changelog/README.md b/docs/project-overview/changelog/README.md index 97baf9e0fae7..887369c805d2 100644 --- a/docs/project-overview/changelog/README.md +++ b/docs/project-overview/changelog/README.md @@ -604,7 +604,7 @@ Airbyte is comprised of 2 parts: ### Production v. Dev Releases -The "production" version of Airbyte is the version of the app specified in `.env`. With each production release, we update the version in the `.env` file. This version will always be available for download on DockerHub. It is the version of the app that runs when a user runs `docker-compose up`. +The "production" version of Airbyte is the version of the app specified in `.env`. With each production release, we update the version in the `.env` file. This version will always be available for download on DockerHub. It is the version of the app that runs when a user runs `docker compose up`. The "development" version of Airbyte is the head of master branch. It is the version of the app that runs when a user runs `./gradlew build && VERSION=dev docker compose up`. diff --git a/docs/quickstart/deploy-airbyte.md b/docs/quickstart/deploy-airbyte.md index 007c31cdb1bc..c0d075378776 100644 --- a/docs/quickstart/deploy-airbyte.md +++ b/docs/quickstart/deploy-airbyte.md @@ -8,7 +8,7 @@ Deploying Airbyte Open-Source just takes two steps. ```bash git clone https://github.com/airbytehq/airbyte.git cd airbyte -docker-compose up +docker compose up ``` Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000](http://localhost:8000)! You will be asked for a username and password. By default, that's username `airbyte` and password `password`. Once you deploy airbyte to your servers, **be sure to change these** in your `.env` file. diff --git a/docs/quickstart/getting-started.md b/docs/quickstart/getting-started.md index 3521eacdc2e5..6c898cbed698 100644 --- a/docs/quickstart/getting-started.md +++ b/docs/quickstart/getting-started.md @@ -11,7 +11,7 @@ First of all, make sure you have Docker and Docker Compose installed. Then run t ```text git clone https://github.com/airbytehq/airbyte.git cd airbyte -docker-compose -f docker-compose.yaml up +docker compose -f docker-compose.yaml up ``` Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000/](http://localhost:8000/). diff --git a/docs/troubleshooting/new-connection.md b/docs/troubleshooting/new-connection.md index 9a9322707e2c..b8734d72cc1c 100644 --- a/docs/troubleshooting/new-connection.md +++ b/docs/troubleshooting/new-connection.md @@ -17,7 +17,7 @@ One workaround is to manually pull the latest version of every connector you'll 1. Decide which connectors you'd like to use. For this example let's say you want the Postgres source and the Snowflake destination. 2. Find the Docker image name of those connectors. Look [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-config/init/src/main/resources/seed/source_definitions.yaml) for sources and [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml) for destinations. For each of the connectors you'd like to use, copy the value of the `dockerRepository` and `dockerImageTag` fields. For example, for the Postgres source this would be `airbyte/source-postgres` and e.g `0.1.6`. 3. For **each of the connectors** you'd like to use, from your shell run `docker pull :`, replacing `` and `` with the values copied from the step above e.g: `docker pull airbyte/source-postgres:0.1.6`. -4. Once you've finished downloading all the images, from the Airbyte repository root run `docker-compose down -v` followed by `docker-compose up`. +4. Once you've finished downloading all the images, from the Airbyte repository root run `docker compose down -v` followed by `docker compose up`. 5. The issue should be resolved. If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). @@ -30,7 +30,7 @@ If you are running into connection refused errors when running Airbyte via Docke ## I don’t see a form when selecting a connector -We’ve had that issue once. \(no spinner & 500 http error\). We don’t know why. Resolution: try to stop airbyte \(`docker-compose down`\) & restart \(`docker-compose up`\) +We’ve had that issue once. \(no spinner & 500 http error\). We don’t know why. Resolution: try to stop airbyte \(`docker compose down`\) & restart \(`docker compose up`\) ## Connection hangs when trying to run the discovery step diff --git a/docs/troubleshooting/on-deploying.md b/docs/troubleshooting/on-deploying.md index 6bcb0681a7ab..5a67cadc0983 100644 --- a/docs/troubleshooting/on-deploying.md +++ b/docs/troubleshooting/on-deploying.md @@ -6,10 +6,10 @@ description: Common issues and their workarounds when trying to deploy Airbyte ## Stuck in onboarding, can’t skip or do anything -To fully reset Airbyte, you also need to delete the docker volumes associated with Airbyte. This is where data is stored. Assuming that you are running Airbyte by running `docker-compose up`, then what you need to do is: +To fully reset Airbyte, you also need to delete the docker volumes associated with Airbyte. This is where data is stored. Assuming that you are running Airbyte by running `docker compose up`, then what you need to do is: -* Turn off Airbyte completely: `docker-compose down -v` -* Turn Airbyte back on: `docker-compose up` +* Turn off Airbyte completely: `docker compose down -v` +* Turn Airbyte back on: `docker compose up` that should handle you getting reset to the beginning. I would be curious if we can see the logs associated with the failure you are seeing. I would say if after you reset you run into it again we can debug that. @@ -25,11 +25,11 @@ git reset --hard HEAD However it's worth pointing out that the `core.longpaths` option is defaulted to false for a reason, so use with caution. This git configuration is only changed within the cloned Airbyte repo, so you won't need to worry about changing this setting for other repositories. Find more details about this issue in [this stack overflow question](https://stackoverflow.com/questions/22575662/filename-too-long-in-git-for-windows). -Instead of cloning the repo, you can alternatively download the latest Airbyte release [here](https://github.com/airbytehq/airbyte/releases). Unzip the downloaded file, access the unzipped file using PowerShell terminal, and run `docker-compose up`. After this, you should see the Airbyte containers in the Docker application as in the image below. +Instead of cloning the repo, you can alternatively download the latest Airbyte release [here](https://github.com/airbytehq/airbyte/releases). Unzip the downloaded file, access the unzipped file using PowerShell terminal, and run `docker compose up`. After this, you should see the Airbyte containers in the Docker application as in the image below. ![](../.gitbook/assets/airbyte_deploy_windows_docker.png) -## I have run `docker-compose up` and can not access the interface +## I have run `docker compose up` and can not access the interface * If you see a blank screen and not a loading icon: @@ -49,8 +49,8 @@ b88d94652268 airbyte/db:1.11.1-alpha "docker-entrypoint.s…" 2 hou You must see 4 containers running. If you are not seeing execute the following steps: -* `docker-compose down -v` -* `docker-compose up` +* `docker compose down -v` +* `docker compose up` Keep in mind the commands above will delete ALL containers, volumes and data created by Airbyte. @@ -83,9 +83,9 @@ This happens \(sometimes\) on Windows system when you first install `docker`. Yo ## Getting a weird error related to setting up the Airbyte server when running Docker Compose -- wondering if this is because I played around with Airbyte in a past version? -If you are okay with losing your previous Airbyte configurations, you can run `docker-compose down -v` and that should fix things then `docker-compose up`. +If you are okay with losing your previous Airbyte configurations, you can run `docker compose down -v` and that should fix things then `docker compose up`. -## `unauthorized: incorrect username or password` when running `docker-compose up` +## `unauthorized: incorrect username or password` when running `docker compose up` If you see the following error: @@ -94,7 +94,7 @@ ERROR: Head "https://registry-1.docker.io/v2/airbyte/init/manifests/{XXX}": unau ``` You are most likely logged into Docker with your email address instead of your Docker ID. -Log out of Docker by running `docker logout` and try running `docker-compose up` again. +Log out of Docker by running `docker logout` and try running `docker compose up` again. ## Protocol Version errors from the bootloader when trying to upgrade diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 3983e1856ded..11d43cb33354 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -130,7 +130,7 @@ services: Other commands besides `apply` can be run like so: ```bash -docker-compose run octavia-cli ` +docker compose run octavia-cli ` ``` ## Commands reference diff --git a/tools/bin/acceptance_test.sh b/tools/bin/acceptance_test.sh index 8040fe623d71..628ab3e24525 100755 --- a/tools/bin/acceptance_test.sh +++ b/tools/bin/acceptance_test.sh @@ -13,7 +13,7 @@ get_epoch_time() { } check_success() { - docker-compose ps | grep "^$1" | grep -e 'Exit 0' -e 'exited (0)' >/dev/null || (echo "$1 didn't run successfully"; exit 1) + docker compose ps | grep "^$1" | grep -ie 'exit 0' -ie 'exited (0)' >/dev/null || (echo "$1 didn't run successfully"; exit 1) } ## @@ -23,12 +23,12 @@ echo "Starting app..." # Detach so we can run subsequent commands # NOTE: this passes APPLY_FIELD_SELECTION=true, which enables a feature -- field selection -- which is currently disabled by default. # We want to run our CI tests against the new feature while we prepare to release it. -VERSION=dev TRACKING_STRATEGY=logging USE_STREAM_CAPABLE_STATE=true BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" APPLY_FIELD_SELECTION=true docker-compose -f docker-compose.yaml -f docker-compose.acceptance-test.yaml up -d +VERSION=dev TRACKING_STRATEGY=logging USE_STREAM_CAPABLE_STATE=true BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" APPLY_FIELD_SELECTION=true docker compose -f docker-compose.yaml -f docker-compose.acceptance-test.yaml up -d # Sometimes source/dest containers using airbyte volumes survive shutdown, which need to be killed in order to shut down properly. -shutdown_cmd="docker-compose down -v || docker kill \$(docker ps -a -f volume=airbyte_workspace -f volume=airbyte_data -f volume=airbyte_db -q) && docker-compose down -v" +shutdown_cmd="docker compose down -v || docker kill \$(docker ps -a -f volume=airbyte_workspace -f volume=airbyte_data -f volume=airbyte_db -q) && docker compose down -v" # Uncomment for debugging. Warning, this is verbose. -# trap "echo 'docker-compose logs:' && docker-compose logs -t --tail 1000 && $shutdown_cmd" EXIT +# trap "echo 'docker compose logs:' && docker compose logs -t --tail 1000 && $shutdown_cmd" EXIT echo "Waiting for services to begin" starttime=`get_epoch_time` @@ -38,7 +38,7 @@ do echo "Waiting for docker deployment.." currenttime=`get_epoch_time` if [[ $(( $currenttime - $starttime )) -gt $maxtime ]]; then - docker-compose ps + docker compose ps echo "Platform is taking more than ${maxtime}s to start. Aborting..." exit 1 fi @@ -46,7 +46,7 @@ do done # Getting a snapshot of the docker compose state -docker-compose ps +docker compose ps # Make sure init containers ran successfully check_success 'init' diff --git a/tools/bin/e2e_test.sh b/tools/bin/e2e_test.sh index 6d1f5796cae5..037b54a4b359 100755 --- a/tools/bin/e2e_test.sh +++ b/tools/bin/e2e_test.sh @@ -15,9 +15,9 @@ echo "Starting app..." mkdir -p /tmp/airbyte_local # Detach so we can run subsequent commands -VERSION=dev BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" TRACKING_STRATEGY=logging docker-compose up -d +VERSION=dev BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" TRACKING_STRATEGY=logging docker compose up -d # Uncomment for debugging. Warning, this is verbose. -# trap 'echo "docker-compose logs:" && docker-compose logs -t --tail 1000 && docker-compose down && docker stop airbyte_ci_pg' EXIT +# trap 'echo "docker compose logs:" && docker compose logs -t --tail 1000 && docker compose down && docker stop airbyte_ci_pg' EXIT docker run --rm -d -p 5433:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_source --name airbyte_ci_pg_source postgres docker run --rm -d -p 5434:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_destination --name airbyte_ci_pg_destination postgres diff --git a/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh b/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh index 4464b8f7b2fe..f7f3d50f1504 100755 --- a/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh +++ b/tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh @@ -15,7 +15,7 @@ echo "Tag" $TAG docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD" VERSION=$TAG ./gradlew build -VERSION=$TAG docker-compose -f docker-compose.build.yaml push +VERSION=$TAG docker compose -f docker-compose.build.yaml push # For running on Mac #sed -i .bak 's/default/'$NAMESPACE'/g' kube/overlays/dev/kustomization.yaml diff --git a/tools/bin/gke-kube-helm-acceptance-test/acceptance_test_kube_gke.sh b/tools/bin/gke-kube-helm-acceptance-test/acceptance_test_kube_gke.sh index 53418fd014f3..e8055b43cf4e 100755 --- a/tools/bin/gke-kube-helm-acceptance-test/acceptance_test_kube_gke.sh +++ b/tools/bin/gke-kube-helm-acceptance-test/acceptance_test_kube_gke.sh @@ -15,7 +15,7 @@ echo "Tag" $TAG docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD" VERSION=$TAG ./gradlew build -# VERSION=$TAG docker-compose -f docker-compose.build.yaml push +# VERSION=$TAG docker compose -f docker-compose.build.yaml push # For running on Mac #sed -i .bak 's/default/'$NAMESPACE'/g' kube/overlays/dev/kustomization.yaml diff --git a/tools/bin/integration_tests_octavia.sh b/tools/bin/integration_tests_octavia.sh index 5719a03a2ca8..41bb7d343262 100755 --- a/tools/bin/integration_tests_octavia.sh +++ b/tools/bin/integration_tests_octavia.sh @@ -9,11 +9,11 @@ assert_root echo "Starting app..." # Detach so we can run subsequent commands -VERSION=dev TRACKING_STRATEGY=logging BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" docker-compose up -d +VERSION=dev TRACKING_STRATEGY=logging BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" docker compose up -d # Sometimes source/dest containers using airbyte volumes survive shutdown, which need to be killed in order to shut down properly. -shutdown_cmd="docker-compose down -v || docker kill \$(docker ps -a -f volume=airbyte_workspace -f volume=airbyte_data -f volume=airbyte_db -q) && docker-compose down -v" -trap "echo 'docker-compose logs:' && docker-compose logs -t --tail 1000 && $shutdown_cmd" EXIT +shutdown_cmd="docker compose down -v || docker kill \$(docker ps -a -f volume=airbyte_workspace -f volume=airbyte_data -f volume=airbyte_db -q) && docker compose down -v" +trap "echo 'docker compose logs:' && docker compose logs -t --tail 1000 && $shutdown_cmd" EXIT echo "Waiting for services to begin" while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/api/v1/health)" != "200" ]]; do echo "Waiting for docker deployment.."; sleep 5; done diff --git a/tools/bin/load_test/README.md b/tools/bin/load_test/README.md index ef45405e8917..188c557a7e5b 100644 --- a/tools/bin/load_test/README.md +++ b/tools/bin/load_test/README.md @@ -13,7 +13,7 @@ From your top-level `/airbyte` directory, run the following to perform a load te By default, the script assumes that the Airbyte instance's server is accessible at `localhost:8001`. This is the default server location when -deploying Airbyte with `docker-compose up`. +deploying Airbyte with `docker compose up`. Additionally, the E2E Test Source created by the script will take 10 minutes to complete a sync by default. diff --git a/tools/bin/test-temporal-migration.sh b/tools/bin/test-temporal-migration.sh index a5bc1eaec9fd..efb349c4d1bd 100755 --- a/tools/bin/test-temporal-migration.sh +++ b/tools/bin/test-temporal-migration.sh @@ -18,13 +18,13 @@ git pull --no-rebase SUB_BUILD=PLATFORM "$SCRIPT_DIR"/../../gradlew -p "$SCRIPT_DIR"/../.. generate-docker cd "$SCRIPT_DIR"/../.. -VERSION=dev docker-compose -f "$SCRIPT_DIR"/../../docker-compose.yaml up & +VERSION=dev docker compose -f "$SCRIPT_DIR"/../../docker-compose.yaml up & sleep 75 -VERSION=dev docker-compose down +VERSION=dev docker compose down git stash git checkout $NEW_HASH SUB_BUILD=PLATFORM "$SCRIPT_DIR"/../../gradlew -p "$SCRIPT_DIR"/../.. generate-docker -VERSION=dev docker-compose -f "$SCRIPT_DIR"/../../docker-compose.yaml up +VERSION=dev docker compose -f "$SCRIPT_DIR"/../../docker-compose.yaml up