build.gradle

import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
import com.github.spotbugs.snom.SpotBugsTask

buildscript {
    repositories {
        maven {
            url "https://plugins.gradle.org/m2/"
        }
    }
    dependencies {
        classpath 'com.bmuschko:gradle-docker-plugin:7.2.0'
    }
}

plugins {
    id 'base'
    id 'pmd'
    id 'com.diffplug.spotless' version '6.7.1'
    id 'com.github.hierynomus.license' version '0.16.1'
    id 'com.github.spotbugs' version '5.0.6'
    id 'version-catalog'
    id 'maven-publish'
}

apply from: "$rootDir/publish-repositories.gradle"

repositories {
    mavenCentral()
}

Properties env = new Properties()
rootProject.file('.env').withInputStream { env.load(it) }

if (!env.containsKey('VERSION')) {
    throw new Exception('Version not specified in .env file...')
}

// `version` is used as the application build version for artifacts like jars
// `image_tag` is used as the docker tag applied to built images.
// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable.
// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'.
ext {
    version = System.getenv("VERSION") ?: env.VERSION
    image_tag = System.getenv("VERSION") ?: 'dev'
}

def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
    /*
    In java, we don't have a second linter/styling tool other than spotless so it doesn't really
    matter if we write a newline or not for startComment/endComment.

    However, in python, we are using black that double-checks and reformats the code.
    Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a
    big deal and would be reformatted (removed) because of black's specs.
    */
    def tmp = File.createTempFile('tmp', '.tmp')
    tmp.withWriter {
        def w = it
        if (startComment.length() > 0 || !isPython) {
            w.writeLine(startComment)
        }
        license.eachLine {
            w << lineComment
            w.writeLine(it)
        }
        if (endComment.length() > 0 || !isPython) {
            w.writeLine(endComment)
        }
        w.writeLine("")
        if (isPython) {
            w.writeLine("")
        }
    }
    return tmp
}

def createPythonLicenseWith = { license ->
    return createLicenseWith(license, '', '', '', true)
}

def createJavaLicenseWith = { license ->
    return createLicenseWith(license, '/*', ' */', ' * ', false)
}

// We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a
// monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable.
def createSpotlessTarget = { pattern ->
    def excludes = [
        '.gradle',
        'node_modules',
        '.eggs',
        '.mypy_cache',
        '.venv',
        '*.egg-info',
        'build',
        'dbt-project-template',
        'dbt-project-template-mssql',
        'dbt-project-template-mysql',
        'dbt-project-template-oracle',
        'dbt-project-template-clickhouse',
        'dbt-project-template-snowflake',
        'dbt_test_config',
        'normalization_test_output',
        'tools',
        'secrets',
        'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
        'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file.
        'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad
    ]

    if (System.getenv().containsKey("SUB_BUILD")) {
        excludes.add("airbyte-integrations/connectors")
    }

    return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" })
}

spotless {
    java {
        target createSpotlessTarget('**/*.java')

        importOrder()

        eclipse('4.21.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))

        licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT'))
        removeUnusedImports()
        trimTrailingWhitespace()
    }
    groovyGradle {
        target createSpotlessTarget('**/*.gradle')
    }
    sql {
        target createSpotlessTarget('**/*.sql')

        dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
    }
    format 'styling', {
        target createSpotlessTarget(['**/*.yaml', '**/*.json'])

        prettier()
    }
}
check.dependsOn 'spotlessApply'

@SuppressWarnings('GroovyAssignabilityCheck')
def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) {
    return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
        // This is currently only used for connectors.
        def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'

        def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
        def isArm64 = arch == "aarch64" || arch == "arm64"

        def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
        def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
        def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine'
        def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:19-slim-bullseye" : "amd64/openjdk:19-slim-bullseye"
        def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'

        inputDir = file("$projectDir/build/docker")
        platform = buildPlatform
        images.add("airbyte/$artifactName:$buildTag")
        buildArgs.put('JDK_VERSION', jdkVersion)
        buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
        buildArgs.put('ALPINE_IMAGE', alpineImage)
        buildArgs.put('NGINX_IMAGE', nginxImage)
        buildArgs.put('JDK_IMAGE', openjdkImage)
        buildArgs.put('VERSION', buildVersion)
    })
}

@SuppressWarnings('GroovyAssignabilityCheck')
def Task getPublishArtifactsTask(String buildVersion, Project subproject) {
    // generate a unique task name based on the directory name.
    return task ("publishArtifact-$subproject.name" {
        apply plugin: 'maven-publish'
        publishing {
            repositories {
                publications {
                    "$subproject.name"(MavenPublication) {
                        from subproject.components.java

                        // use the subproject group and name with the assumption there are no identical subproject
                        // names, group names or subproject group/name combination.
                        groupId = "$subproject.group"
                        artifactId = "$subproject.name"
                        version = "$buildVersion"
                        repositories.add(rootProject.repositories.getByName('cloudrepo'))
                    }
                }
            }
        }
    })
}

allprojects {
    apply plugin: 'com.bmuschko.docker-remote-api'

    task copyDocker(type: Sync) {
        from "${project.projectDir}/Dockerfile"
        into "build/docker/"
    }
}

allprojects {
    apply plugin: 'base'

    // by default gradle uses directory as the project name. That works very well in a single project environment but
    // projects clobber each other in an environments with subprojects when projects are in directories named identically.
    def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
    group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
    project.archivesBaseName = "${project.group}-${project.name}"

    version = rootProject.ext.version
}

// Java projects common configurations
subprojects {

    if (project.name == 'airbyte-webapp' || project.name == 'airbyte-webapp-e2e-tests') {
        return
    }

    apply plugin: 'java'
    apply plugin: 'jacoco'
    apply plugin: 'com.github.spotbugs'
    apply plugin: 'pmd'

    sourceCompatibility = JavaVersion.VERSION_17
    targetCompatibility = JavaVersion.VERSION_17

    repositories {
        mavenCentral()
        maven {
            // TODO(Issue-4915): Remove this when upstream is merged in.
            url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
        }
        maven {
            url 'https://jitpack.io'
        }
        maven {
            url 'https://maven.twttr.com'
        }
    }

    pmd {
        consoleOutput = true
        ignoreFailures = false
        rulesMinimumPriority = 5
        ruleSets = []
        ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml'))
        toolVersion = '6.43.0'
    }

    jacoco {
        toolVersion = "0.8.7"
    }

    jacocoTestReport {
        dependsOn test
        reports {
            html.required = true
            xml.required = true
            csv.required = false
        }
    }

    jacocoTestCoverageVerification {
        violationRules {
            failOnViolation = false
            rule {
                element = 'CLASS'
                excludes = ['**/*Test*', '**/generated*']
                limit {
                    counter = 'BRANCH'
                    minimum = 0.8
                }
                limit {
                    counter = 'INSTRUCTION'
                    minimum = 0.8
                }
            }
        }
    }

    def integrationTagName = 'platform-integration'
    def slowIntegrationTagName = 'platform-slow-integration'
    // make tag accessible in submodules.
    ext {
        cloudStorageTestTagName = 'cloud-storage'
    }

    spotbugs {
        ignoreFailures = false
        effort = 'max'
        excludeFilter = rootProject.file('spotbugs-exclude-filter-file.xml')
        reportLevel = 'high'
        showProgress = false
        toolVersion = '4.6.0'
    }

    test {
        jacoco {
            enabled = true
            excludes = ['**/*Test*', '**/generated*']
        }
        useJUnitPlatform {
            excludeTags(integrationTagName, slowIntegrationTagName, cloudStorageTestTagName)
        }
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    task newIntegrationTests(type: Test) {
        useJUnitPlatform {
            includeTags integrationTagName
        }
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
        }
        finalizedBy jacocoTestReport
    }

    task slowIntegrationTests(type: Test) {
        useJUnitPlatform {
            includeTags slowIntegrationTagName
        }
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
        }
        finalizedBy jacocoTestReport
    }

    task allTests(type: Test) {
        useJUnitPlatform()
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
        }
        finalizedBy jacocoTestReport
    }

    dependencies {
        if (project.name != 'airbyte-commons') {
            implementation project(':airbyte-commons')
        }

        implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0"))
        implementation(platform("org.glassfish.jersey:jersey-bom:2.31"))


        // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here.
        implementation libs.bundles.jackson

        implementation libs.guava

        implementation libs.commons.io

        implementation libs.bundles.apache

        implementation libs.slf4j.api


        // SLF4J as a facade over Log4j2 required dependencies
        implementation libs.bundles.log4j

        // Bridges from other logging implementations to SLF4J
        implementation libs.bundles.slf4j

        // Dependencies for logging to cloud storage, as well as the various clients used to do so.
        implementation libs.appender.log4j2
        implementation libs.aws.java.sdk.s3
        implementation libs.google.cloud.storage

        implementation libs.s3

        // Lombok dependencies
        compileOnly libs.lombok
        annotationProcessor libs.lombok

        testCompileOnly libs.lombok
        testAnnotationProcessor libs.lombok

        testRuntimeOnly libs.junit.jupiter.engine
        testImplementation libs.bundles.junit
        testImplementation libs.assertj.core

        testImplementation libs.junit.pioneer

        // adds owasp plugin
        spotbugsPlugins libs.findsecbugs.plugin
        implementation libs.spotbugs.annotations
    }

    tasks.withType(Tar) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(Zip) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(SpotBugsTask) {
        // Reports can be found under each subproject in build/spotbugs/
        reports {
            xml.required = false
            html.required = true
        }
    }

    tasks.withType(Pmd) {
        exclude '**/generated/**'
        exclude '**/jooq/**'
    }

    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
    check.dependsOn 'jacocoTestCoverageVerification'
}

task('generate') {
    dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
    dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }
}

task('format') {
    dependsOn generate
    dependsOn spotlessApply
    dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) }
}

// add licenses for python projects.
subprojects {
    def pythonFormatTask = project.tasks.findByName('blackFormat')

    if (pythonFormatTask != null) {
        apply plugin: "com.github.hierynomus.license"
        license {
            header rootProject.file("LICENSE_SHORT")
        }
        task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) {
            header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
            source = fileTree(dir: projectDir)
                    .include("**/*.py")
                    .exclude(".venv/**/*.py")
                    .exclude("**/airbyte_api_client/**/*.py")
                    .exclude("**/__init__.py")
            strictCheck = true
        }
        def licenseTask = project.tasks.findByName('licenseFormatPython')
        blackFormat.dependsOn licenseTask
        isortFormat.dependsOn licenseTask
        flakeCheck.dependsOn licenseTask

        def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles')
        if (generateFilesTask != null) {
            licenseTask.dependsOn generateFilesTask
        }
    }
}

task('generate-docker') {
    dependsOn(':airbyte-bootloader:assemble')
    dependsOn(':airbyte-workers:assemble')
    dependsOn(':airbyte-webapp:assemble')
    dependsOn(':airbyte-server:assemble')
    dependsOn(':airbyte-db:db-lib:assemble')
    dependsOn(':airbyte-config:init:assemble')
    dependsOn(':airbyte-temporal:assemble')
}

// produce reproducible archives
// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
tasks.withType(AbstractArchiveTask) {
    preserveFileTimestamps = false
    reproducibleFileOrder = true
}

// definition for publishing
catalog {
    versionCatalog {
        from(files("deps.toml"))
    }
}

publishing {
    publications {
        // This block is present for dependency catalog publishing.
        maven(MavenPublication) {
            groupId = 'io.airbyte'
            artifactId = 'oss-catalog'

            from components.versionCatalog
            // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id.
            // e.g. the subproject :airbyte-scheduler:scheduler-models is imported at io.airbyte.airbyte-config:config-persistence:<version-number>.
        }
    }
    repositories.add(rootProject.repositories.getByName('cloudrepo'))
}