build.gradle

plugins {
    id 'base'
    id 'pmd'
    id 'com.diffplug.spotless' version '5.7.0'
    id 'com.github.hierynomus.license' version '0.16.1'
}

repositories {
    mavenCentral()
}

Properties env = new Properties()
rootProject.file('.env').withInputStream { env.load(it) }

if (!env.containsKey('VERSION')) {
    throw new Exception('Version not specified in .env file...')
}

def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
    /*
    In java, we don't have a second linter/styling tool other than spotless so it doesn't really
    matter if we write a newline or not for startComment/endComment.

    However, in python, we are using black that double-checks and reformats the code.
    Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a
    big deal and would be reformatted (removed) because of black's specs.
    */
    def tmp = File.createTempFile('tmp', '.tmp')
    tmp.withWriter {
        def w = it
        if (startComment.length() > 0 || !isPython) {
            w.writeLine(startComment)
        }
        license.eachLine {
            w << lineComment
            w.writeLine(it)
        }
        if (endComment.length() > 0 || !isPython) {
            w.writeLine(endComment)
        }
        w.writeLine("")
        if (isPython) {
            w.writeLine("")
        }
    }
    return tmp
}

def createPythonLicenseWith = { license ->
    return createLicenseWith(license, '', '', '', true)
}

def createJavaLicenseWith = { license ->
    return createLicenseWith(license, '/*', ' */', ' * ', false)
}

// We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a
// monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable.
def createSpotlessTarget = { pattern ->
    def excludes = [
        '.gradle',
        'node_modules',
        '.eggs',
        '.mypy_cache',
        '.venv',
        '*.egg-info',
        'build',
        'dbt-project-template',
        'dbt-project-template-mssql',
        'dbt-project-template-mysql',
        'dbt-project-template-oracle',
        'dbt_test_config',
        'normalization_test_output',
        'tools',
        'secrets',
        'charts' // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
    ]

    if (System.getenv().containsKey("SUB_BUILD")) {
        excludes.add("airbyte-integrations/connectors")
    }

    return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" })
}

spotless {
    java {
        target createSpotlessTarget('**/*.java')

        importOrder()

        eclipse('4.16.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))

        licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT'))
        removeUnusedImports()
        trimTrailingWhitespace()
    }
    groovyGradle {
        target createSpotlessTarget('**/*.gradle')
    }
    sql {
        target createSpotlessTarget('**/*.sql')

        dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
    }
    format 'styling', {
        target createSpotlessTarget(['**/*.yaml', '**/*.json'])

        prettier()
    }
}
check.dependsOn 'spotlessApply'

allprojects {
    apply plugin: 'base'

    afterEvaluate { project ->
        def composeDeps = [
                ":airbyte-config:init",
                ":airbyte-db:lib",
                ":airbyte-migration",
                ":airbyte-scheduler:app",
                ":airbyte-workers",
                ":airbyte-server",
                ":airbyte-webapp",
        ].toSet().asImmutable()

        if (project.getPath() in composeDeps) {
            composeBuild.dependsOn(project.getPath() + ':assemble')
        }
    }
}

allprojects {
    // by default gradle uses directory as the project name. That works very well in a single project environment but
    // projects clobber each other in an environments with subprojects when projects are in directories named identically.
    def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
    group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
    project.archivesBaseName = "${project.group}-${project.name}"

    version = env.VERSION
}

// Java projects common configurations
subprojects {

    if (project.name == 'airbyte-webapp' || project.name == 'airbyte-e2e-testing') {
        return
    }

    apply plugin: 'java'
    apply plugin: 'jacoco'

    sourceCompatibility = JavaVersion.VERSION_14
    targetCompatibility = JavaVersion.VERSION_14

    repositories {
        mavenCentral()
        maven {
            // TODO(Issue-4915): Remove this when upstream is merged in.
            url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
        }
    }

    pmd {
        consoleOutput = true

        rulesMinimumPriority = 5
        ruleSets = []
        ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml'))
    }

    test {
        useJUnitPlatform()
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    jacocoTestReport {
        dependsOn test
    }

    dependencies {
        if (project.name != 'airbyte-commons') {
            implementation project(':airbyte-commons')
        }

        implementation(platform("com.fasterxml.jackson:jackson-bom:2.10.4"))
        implementation(platform("org.glassfish.jersey:jersey-bom:2.31"))


        // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here.
        implementation 'com.fasterxml.jackson.core:jackson-databind'
        implementation 'com.fasterxml.jackson.core:jackson-annotations'
        implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
        implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'

        implementation 'com.google.guava:guava:30.1.1-jre'

        implementation 'commons-io:commons-io:2.7'

        implementation 'org.apache.commons:commons-compress:1.20'
        implementation 'org.apache.commons:commons-lang3:3.11'

        implementation 'org.slf4j:slf4j-api:1.7.30'


        // SLF4J as a facade over Log4j2 required dependencies
        implementation 'org.apache.logging.log4j:log4j-api:2.11.0'
        implementation 'org.apache.logging.log4j:log4j-core:2.11.0'
        implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.11.0'
        implementation 'org.apache.logging.log4j:log4j-web:2.14.1'

        // Bridges from other logging implementations to SLF4J
        implementation 'org.slf4j:jul-to-slf4j:1.7.30' // JUL bridge
        implementation 'org.slf4j:jcl-over-slf4j:1.7.30' //JCL bridge
        implementation 'org.slf4j:log4j-over-slf4j:1.7.30' // log4j1.2 bridge

        // Dependencies for logging to cloud storage, as well as the various clients used to do so.
        implementation 'com.therealvan:appender-log4j2:3.1.2-SNAPSHOT'
        implementation 'com.amazonaws:aws-java-sdk-s3:1.12.6'
        implementation 'com.google.cloud:google-cloud-storage:1.115.0'

        implementation 'software.amazon.awssdk:s3:2.16.84'


        testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.2'
        testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.2'
        testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
        testImplementation 'org.mockito:mockito-junit-jupiter:3.12.4'
        testImplementation 'org.assertj:assertj-core:3.21.0'
    }

    tasks.withType(Tar) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(Zip) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }
}

task composeBuild {
    def buildTag = System.getenv('VERSION') ?: 'dev'
    def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64'
    def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64'
    def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2'
    def dockerComposeFile = buildArch == 'arm64' ? 'docker-compose.build-m1.yaml' : 'docker-compose.build.yaml'
    doFirst {
        exec {
            workingDir rootDir
            commandLine 'docker-compose', '-f', dockerComposeFile, 'build', '--parallel', '--quiet'
            environment 'VERSION', buildTag
            environment 'DOCKER_BUILD_PLATFORM', buildPlatform
            environment 'DOCKER_BUILD_ARCH', buildArch
            environment 'JDK_VERSION', jdkVersion
        }
    }
}

if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") {
    build.dependsOn(composeBuild)
}

task('generate') {
    dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
    dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }
}

task('format') {
    dependsOn generate
    dependsOn spotlessApply
    dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) }
}

// add licenses for python projects.
subprojects {
    def pythonFormatTask = project.tasks.findByName('blackFormat')

    if (pythonFormatTask != null) {
        apply plugin: "com.github.hierynomus.license"
        task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) {
            header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
            source = fileTree(dir: projectDir)
                    .include("**/*.py")
                    .exclude(".venv/**/*.py")
                    .exclude("**/__init__.py")
            strictCheck = true
        }
        def licenseTask = project.tasks.findByName('licenseFormatPython')
        blackFormat.dependsOn licenseTask
        isortFormat.dependsOn licenseTask
        flakeCheck.dependsOn licenseTask

        def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles')
        if (generateFilesTask != null) {
            licenseTask.dependsOn generateFilesTask
        }
    }
}

// produce reproducible archives
// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
tasks.withType(AbstractArchiveTask) {
    preserveFileTimestamps = false
    reproducibleFileOrder = true
}

// TODO: Davin will turn this back on.
//// Configure what subprojects to publish.
//String[] toPublish = [
//        ':airbyte-analytics',
//        ':airbyte-api',
//        ':airbyte-commons',
//        ':airbyte-commons-docker',
//        ':airbyte-config:init',
//        ':airbyte-config:models',
//        ':airbyte-config:persistence',
//        ':airbyte-db:lib',
//        ':airbyte-json-validation',
//        ':airbyte-migration',
//        ':airbyte-notification',
//        ':airbyte-protocol:models',
//        ':airbyte-scheduler:client',
//        ':airbyte-scheduler:models',
//        ':airbyte-scheduler:persistence',
//        ':airbyte-server',
//        ':airbyte-workers'
//]
//configure(subprojects.findAll { toPublish.contains(it.getPath()) }) {
//    apply plugin: 'maven-publish'
//
//    publishing {
//        repositories {
//            publications {
//                // This block is present so Gradle knows to publish a Maven jar.
//                maven(MavenPublication) {
//                    from components.java
//                    // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id.
//                    // e.g. the subproject :airbyte-scheduler:models is imported at io.airbyte.airbyte-config:persistence:<version-number>.
//                }
//            }
//
//            maven {
//                credentials {
//                    name 'cloudrepo'
//                    username System.getenv('CLOUDREPO_USER')
//                    password System.getenv('CLOUDREPO_PASSWORD')
//                }
//                url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars'
//            }
//
//            mavenLocal()
//        }
//    }
//}