Revert "Bmoric/remove docker compose for build (airbytehq#7500)" (air…

…bytehq#7698)

This reverts commit 4e17fa2.

.bumpversion.cfg

@@ -10,8 +10,6 @@ serialize =
 
 [bumpversion:file:.env]
 
-[bumpversion:file:airbyte-migration/Dockerfile]
-
 [bumpversion:file:airbyte-server/Dockerfile]
 
 [bumpversion:file:airbyte-workers/Dockerfile]

.github/workflows/gradle.yml

@@ -339,7 +339,7 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan
 
       - name: Run End-to-End Frontend Tests
         run: ./tools/bin/e2e_test.sh
@@ -457,7 +457,7 @@ jobs:
           HOME: /home/runner
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew assemble --scan
+        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
 
       - name: Run Kubernetes End-to-End Acceptance Tests
         env:

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir")
-dockerBuildTask.dependsOn(copyDocker)
-assemble.dependsOn(dockerBuildTask)
+plugins {
+    id "airbyte-docker"
+}

airbyte-config/init/.dockerignore

@@ -0,0 +1,3 @@
+*
+!src
+!scripts

airbyte-config/init/Dockerfile

@@ -5,4 +5,4 @@ WORKDIR /app
 
 # the sole purpose of this image is to seed the data volume with the default data
 # that the app should have when it is first installed.
-COPY bin/scripts scripts
+COPY scripts scripts

airbyte-config/init/build.gradle

@@ -11,14 +11,3 @@ dependencies {
     implementation project(':airbyte-commons-docker')
     implementation project(':airbyte-json-validation')
 }
-
-task copyScripts(type: Copy) {
-    dependsOn copyDocker
-
-    from('scripts')
-    into 'build/docker/bin/scripts'
-}
-
-Task dockerBuildTask = getDockerBuildTask("init",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyScripts)
-assemble.dependsOn(dockerBuildTask)

airbyte-db/lib/.dockerignore

@@ -0,0 +1,2 @@
+*
+!src

airbyte-db/lib/Dockerfile

@@ -1,3 +1,3 @@
 FROM postgres:13-alpine
 
-COPY bin/init.sql /docker-entrypoint-initdb.d/000_init.sql
+COPY src/main/resources/init.sql /docker-entrypoint-initdb.d/000_init.sql

airbyte-db/lib/build.gradle

@@ -69,16 +69,3 @@ task(dumpJobsSchema, dependsOn: 'classes', type: JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
     args 'jobs', 'dump_schema'
 }
-
-task copyInitSql(type: Copy) {
-    dependsOn copyDocker
-
-    from('src/main/resources') {
-        include 'init.sql'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("db",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyInitSql)
-assemble.dependsOn(dockerBuildTask)

airbyte-migration/.dockerignore

@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build

airbyte-migration/Dockerfile

@@ -6,7 +6,7 @@ ENV APPLICATION airbyte-migration
 WORKDIR /app
 
 # Move and run scheduler
-COPY bin/${APPLICATION}-0.30.31-alpha.tar ${APPLICATION}.tar
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 

airbyte-migration/build.gradle

@@ -15,16 +15,3 @@ application {
     mainClass = 'io.airbyte.migrate.MigrationRunner'
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn distTar
-    dependsOn copyDocker
-
-    from('build/distributions') {
-        include 'airbyte-migration-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("migration",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java

@@ -25,6 +25,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

airbyte-scheduler/app/.dockerignore

@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build

airbyte-scheduler/app/Dockerfile

@@ -5,7 +5,7 @@ ENV APPLICATION airbyte-scheduler
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.30.31-alpha.tar /app
+ADD build/distributions/${APPLICATION}-0.30.31-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.31-alpha/bin/${APPLICATION}"]

airbyte-scheduler/app/build.gradle

@@ -57,17 +57,3 @@ run {
     environment "TEMPORAL_HOST", "localhost:7233"
 
 }
-
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-scheduler-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("scheduler",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-server/.dockerignore

@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build

airbyte-server/Dockerfile

@@ -7,7 +7,7 @@ ENV APPLICATION airbyte-server
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.30.31-alpha.tar /app
+ADD build/distributions/${APPLICATION}-0.30.31-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.31-alpha/bin/${APPLICATION}"]

airbyte-server/build.gradle

@@ -127,17 +127,3 @@ run {
     environment "AIRBYTE_ROLE", System.getenv('AIRBYTE_ROLE')
     environment "TEMPORAL_HOST", "localhost:7233"
 }
-
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-server-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("server",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-webapp/.dockerignore

@@ -0,0 +1,4 @@
+*
+!Dockerfile
+!build
+!nginx

airbyte-webapp/Dockerfile

@@ -2,6 +2,8 @@ FROM nginx:1.19-alpine as webapp
 
 EXPOSE 80
 
-COPY bin/docs docs/
-COPY bin/build /usr/share/nginx/html
-COPY bin/nginx/default.conf.template /etc/nginx/templates/default.conf.template
+COPY build/docs docs/
+# docs get copied twice because npm gradle plugin ignores output dir.
+COPY build /usr/share/nginx/html
+RUN rm -rf /usr/share/nginx/html/docs
+COPY nginx/default.conf.template /etc/nginx/templates/default.conf.template

airbyte-webapp/build.gradle

@@ -32,38 +32,11 @@ task test(type: NpmTask) {
 assemble.dependsOn npm_run_build
 build.finalizedBy test
 
-task copyBuild(type: Copy) {
-    dependsOn copyDocker
-
-    from "${project.projectDir}/build"
-    into "build/docker/bin/build"
-    exclude ".docker"
-    exclude "docker"
-}
-
 task copyDocs(type: Copy) {
-    dependsOn copyDocker
-
-    from "${project.rootProject.projectDir}/docs/integrations"
-    into "build/docker/bin/docs/integrations"
+    from "${System.getProperty("user.dir")}/docs/integrations"
+    into "${buildDir}/docs/integrations"
     duplicatesStrategy DuplicatesStrategy.INCLUDE
 }
 
-task copyNginx(type: Copy) {
-    dependsOn copyDocker
-
-    from "${project.projectDir}/nginx"
-    into "build/docker/bin/nginx"
-}
-
-copyBuild.dependsOn npm_run_build
-copyNginx.dependsOn npm_run_build
 copyDocs.dependsOn npm_run_build
 assemble.dependsOn copyDocs
-copyDocker.dependsOn(npm_run_build)
-
-Task dockerBuildTask = getDockerBuildTask("webapp",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyBuild)
-dockerBuildTask.dependsOn(copyNginx)
-dockerBuildTask.dependsOn(copyDocs)
-assemble.dependsOn(dockerBuildTask)

airbyte-workers/Dockerfile

@@ -23,7 +23,7 @@ ENV APPLICATION airbyte-workers
 WORKDIR /app
 
 # Move worker app
-ADD bin/${APPLICATION}-0.30.31-alpha.tar /app
+ADD build/distributions/${APPLICATION}-0.30.31-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.30.31-alpha/bin/${APPLICATION}"]

airbyte-workers/build.gradle

@@ -62,17 +62,3 @@ application {
     mainClass = mainClassName
     applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0']
 }
-
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-workers-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("worker",  "$project.projectDir")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

build.gradle

@@ -1,16 +1,3 @@
-import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
-
-buildscript {
-    repositories {
-        maven {
-            url "https://plugins.gradle.org/m2/"
-        }
-    }
-    dependencies {
-        classpath 'com.bmuschko:gradle-docker-plugin:7.1.0'
-    }
-}
-
 plugins {
     id 'base'
     id 'pmd'
@@ -146,19 +133,26 @@ def Task getDockerBuildTask(String artifactName, String projectDir) {
 }
 
 allprojects {
-    apply plugin: 'com.bmuschko.docker-remote-api'
-
-    task copyDocker(type: Copy) {
-        delete "build/docker"
+    apply plugin: 'base'
 
-        from "${project.projectDir}/Dockerfile"
-        into "build/docker/"
+    afterEvaluate { project ->
+        def composeDeps = [
+                ":airbyte-config:init",
+                ":airbyte-db:lib",
+                ":airbyte-migration",
+                ":airbyte-scheduler:app",
+                ":airbyte-workers",
+                ":airbyte-server",
+                ":airbyte-webapp",
+        ].toSet().asImmutable()
+
+        if (project.getPath() in composeDeps) {
+            composeBuild.dependsOn(project.getPath() + ':assemble')
+        }
     }
 }
 
 allprojects {
-    apply plugin: 'base'
-
     // by default gradle uses directory as the project name. That works very well in a single project environment but
     // projects clobber each other in an environments with subprojects when projects are in directories named identically.
     def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
@@ -260,7 +254,6 @@ subprojects {
         testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
         testImplementation 'org.mockito:mockito-junit-jupiter:3.12.4'
         testImplementation 'org.assertj:assertj-core:3.21.0'
-
     }
 
     tasks.withType(Tar) {
@@ -272,6 +265,28 @@ subprojects {
     }
 }
 
+task composeBuild {
+    def buildTag = System.getenv('VERSION') ?: 'dev'
+    def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64'
+    def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64'
+    def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2'
+    def dockerComposeFile = buildArch == 'arm64' ? 'docker-compose.build-m1.yaml' : 'docker-compose.build.yaml'
+    doFirst {
+        exec {
+            workingDir rootDir
+            commandLine 'docker-compose', '-f', dockerComposeFile, 'build', '--parallel', '--quiet'
+            environment 'VERSION', buildTag
+            environment 'DOCKER_BUILD_PLATFORM', buildPlatform
+            environment 'DOCKER_BUILD_ARCH', buildArch
+            environment 'JDK_VERSION', jdkVersion
+        }
+    }
+}
+
+if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") {
+    build.dependsOn(composeBuild)
+}
+
 task('generate') {
     dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
     dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }

docs/SUMMARY.md

@@ -199,7 +199,6 @@
 * [Contributing to Airbyte](contributing-to-airbyte/README.md)
   * [Code of Conduct](contributing-to-airbyte/code-of-conduct.md)
   * [Developing Locally](contributing-to-airbyte/developing-locally.md)
-  * [Developing on Docker](contributing-to-airbyte/developing-on-docker.md)
   * [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md)
   * [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md)
   * [Code Style](contributing-to-airbyte/code-style.md)