From 5d13c4c76c6e27a0471880505c7c1893d6f90300 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 21 Dec 2022 14:16:35 -0800 Subject: [PATCH] Cleanup (#20459) * Cleanup * More cleanup * Disable in order to test cloud * Restore missing files * Fix test * Format and fix pmd * Add transactional * Fix version * Tentative * Cleanup the cleanup * Rm reference to the micronaut server * format * pmd * more pmd * fix build --- .env | 1 - .../airbyte/api/client/AirbyteApiClient.java | 30 ++++---- .../config/persistence/ConfigRepository.java | 8 +- .../config/persistence/ConfigWriter.java | 3 +- .../persistence/StandardSyncPersistence.java | 3 +- .../StandardSyncPersistenceTest.java | 3 +- airbyte-proxy/Dockerfile | 1 - airbyte-proxy/nginx-auth.conf.template | 68 ----------------- airbyte-proxy/nginx-no-auth.conf.template | 32 -------- airbyte-proxy/run.sh | 2 +- airbyte-proxy/test.sh | 4 +- .../java/io/airbyte/server/EventListener.java | 75 ------------------- .../java/io/airbyte/server/ServerApp.java | 18 ++++- .../server/apis/LogsApiController.java | 2 +- .../server/apis/NotFoundController.java | 30 ++++++++ .../server/apis/WebBackendApiController.java | 1 - .../server/config/ApplicationBeanFactory.java | 1 - .../server/config/DatabaseBeanFactory.java | 1 - .../config/JobErrorReportingBeanFactory.java | 3 - .../server/converters/JobConverter.java | 2 - .../errors/IdNotFoundExceptionMapper.java | 35 +++++++++ .../errors/InvalidInputExceptionMapper.java | 26 ++++--- .../errors/InvalidJsonExceptionMapper.java | 27 ++++--- .../InvalidJsonInputExceptionMapper.java | 29 ++++--- .../server/errors/KnownExceptionMapper.java | 8 +- .../errors/NotFoundExceptionMapper.java | 30 +++++--- .../errors/UncaughtExceptionMapper.java | 31 ++++---- .../DestinationDefinitionsHandler.java | 16 +--- .../airbyte/server/handlers/LogsHandler.java | 3 +- .../server/handlers/SchedulerHandler.java | 15 ++-- .../handlers/SourceDefinitionsHandler.java | 11 --- .../WebBackendConnectionsHandler.java | 3 +- .../resources/application-control-plane.yml | 5 +- .../src/main/resources/application.yml | 2 +- .../src/main/resources/banner/banner.txt | 10 --- .../src/main/resources/micronaut-banner.txt | 3 + .../DestinationDefinitionsHandlerTest.java | 4 +- .../server/handlers/SchedulerHandlerTest.java | 9 ++- .../AirbyteTestContainer.java | 4 - .../acceptance/AdvancedAcceptanceTests.java | 4 - .../test/acceptance/BasicAcceptanceTests.java | 6 +- .../test/acceptance/CdcAcceptanceTests.java | 4 - .../ContainerOrchestratorAcceptanceTests.java | 4 - .../acceptance/VersioningAcceptanceTests.java | 4 - .../workers/config/ApiClientBeanFactory.java | 28 +------ .../src/main/resources/application.yml | 1 - .../airbyte-worker/templates/deployment.yaml | 5 -- charts/airbyte/templates/env-configmap.yaml | 1 - deps.toml | 1 + docker-compose.yaml | 2 - kube/overlays/dev-integration-test/.env | 1 - kube/overlays/dev/.env | 1 - .../overlays/stable-with-resource-limits/.env | 1 - kube/overlays/stable/.env | 1 - kube/resources/worker.yaml | 5 -- tools/bin/acceptance_test.sh | 2 +- 56 files changed, 237 insertions(+), 393 deletions(-) delete mode 100644 airbyte-server/src/main/java/io/airbyte/server/EventListener.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java create mode 100644 airbyte-server/src/main/java/io/airbyte/server/errors/IdNotFoundExceptionMapper.java delete mode 100644 airbyte-server/src/main/resources/banner/banner.txt diff --git a/.env b/.env index 3db630ace868..32c5ba3c4952 100644 --- a/.env +++ b/.env @@ -58,7 +58,6 @@ CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001 ### AIRBYTE SERVICES ### TEMPORAL_HOST=airbyte-temporal:7233 INTERNAL_API_HOST=airbyte-server:8001 -INTERNAL_MICRONAUT_API_HOST=airbyte-server:8080 CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80 WEBAPP_URL=http://localhost:8000/ # Although not present as an env var, required for webapp configuration. diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java index 35e4d23d28eb..08f2c74c03b9 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java @@ -60,21 +60,21 @@ public class AirbyteApiClient { private final AttemptApi attemptApi; private final StateApi stateApi; - public AirbyteApiClient(final ApiClient apiClient, final ApiClient micronautApiClient) { - connectionApi = new ConnectionApi(micronautApiClient); - destinationDefinitionApi = new DestinationDefinitionApi(micronautApiClient); - destinationApi = new DestinationApi(micronautApiClient); - destinationSpecificationApi = new DestinationDefinitionSpecificationApi(micronautApiClient); - jobsApi = new JobsApi(micronautApiClient); - logsApi = new PatchedLogsApi(micronautApiClient); - operationApi = new OperationApi(micronautApiClient); - sourceDefinitionApi = new SourceDefinitionApi(micronautApiClient); - sourceApi = new SourceApi(micronautApiClient); - sourceDefinitionSpecificationApi = new SourceDefinitionSpecificationApi(micronautApiClient); - workspaceApi = new WorkspaceApi(micronautApiClient); - healthApi = new HealthApi(micronautApiClient); - attemptApi = new AttemptApi(micronautApiClient); - stateApi = new StateApi(micronautApiClient); + public AirbyteApiClient(final ApiClient apiClient) { + connectionApi = new ConnectionApi(apiClient); + destinationDefinitionApi = new DestinationDefinitionApi(apiClient); + destinationApi = new DestinationApi(apiClient); + destinationSpecificationApi = new DestinationDefinitionSpecificationApi(apiClient); + jobsApi = new JobsApi(apiClient); + logsApi = new PatchedLogsApi(apiClient); + operationApi = new OperationApi(apiClient); + sourceDefinitionApi = new SourceDefinitionApi(apiClient); + sourceApi = new SourceApi(apiClient); + sourceDefinitionSpecificationApi = new SourceDefinitionSpecificationApi(apiClient); + workspaceApi = new WorkspaceApi(apiClient); + healthApi = new HealthApi(apiClient); + attemptApi = new AttemptApi(apiClient); + stateApi = new StateApi(apiClient); } public ConnectionApi getConnectionApi() { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 614cd66a52a3..5aedb3c18883 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -122,7 +122,7 @@ public ConfigRepository(final Database database) { */ public boolean healthCheck() { try { - database.query(ctx -> ctx.select(WORKSPACE.ID).from(WORKSPACE).limit(1).fetch()); + database.query(ctx -> ctx.select(WORKSPACE.ID).from(WORKSPACE).limit(1).fetch()).stream().count(); } catch (final Exception e) { LOGGER.error("Health check error: ", e); return false; @@ -290,7 +290,8 @@ private Stream sourceDefQuery(final Optional sou .where(ACTOR_DEFINITION.ACTOR_TYPE.eq(ActorType.source)) .and(sourceDefId.map(ACTOR_DEFINITION.ID::eq).orElse(noCondition())) .and(includeTombstone ? noCondition() : ACTOR_DEFINITION.TOMBSTONE.notEqual(true)) - .fetchStream()) + .fetch()) + .stream() .map(DbConverter::buildStandardSourceDefinition) // Ensure version is set. Needed for connectors not upgraded since we added versioning. .map(def -> def.withProtocolVersion(AirbyteProtocolVersion.getWithDefault(def.getProtocolVersion()).serialize())); @@ -352,7 +353,8 @@ private Stream destDefQuery(final Optional .where(ACTOR_DEFINITION.ACTOR_TYPE.eq(ActorType.destination)) .and(destDefId.map(ACTOR_DEFINITION.ID::eq).orElse(noCondition())) .and(includeTombstone ? noCondition() : ACTOR_DEFINITION.TOMBSTONE.notEqual(true)) - .fetchStream()) + .fetch()) + .stream() .map(DbConverter::buildStandardDestinationDefinition) // Ensure version is set. Needed for connectors not upgraded since we added versioning. .map(def -> def.withProtocolVersion(AirbyteProtocolVersion.getWithDefault(def.getProtocolVersion()).serialize())); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java index 00640eadfd9b..8d8ea240a2f1 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java @@ -68,7 +68,8 @@ private static Stream> getActorDefiniti return ctx.select(ACTOR_DEFINITION.ID, ACTOR_DEFINITION.DOCKER_REPOSITORY, ACTOR_DEFINITION.ACTOR_TYPE, ACTOR_DEFINITION.PROTOCOL_VERSION) .from(ACTOR_DEFINITION) .join(ACTOR).on(ACTOR.ACTOR_DEFINITION_ID.equal(ACTOR_DEFINITION.ID)) - .fetchStream(); + .fetch() + .stream(); } static void writeStandardSourceDefinition(final List configs, final DSLContext ctx) { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java index 10b1d7a53a90..01f91cfe8e8a 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java @@ -289,7 +289,8 @@ private Stream findDisabledSyncs(final DSLC .where( CONNECTION.UNSUPPORTED_PROTOCOL_VERSION.eq(true).and( (actorType == ActorType.DESTINATION ? destDef : sourceDef).ID.eq(actorDefId))) - .fetchStream() + .fetch() + .stream() .map(r -> new StandardSyncIdsWithProtocolVersions( r.get(CONNECTION.ID), r.get(sourceDef.ID), diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java index 332429c88f59..cbfb4dfd8b0e 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java @@ -237,7 +237,8 @@ private Set getProtocolVersionFlagForSyncs(fina .select(CONNECTION.ID, CONNECTION.UNSUPPORTED_PROTOCOL_VERSION) .from(CONNECTION) .where(CONNECTION.ID.in(standardSync.stream().map(StandardSync::getConnectionId).toList())) - .fetchStream()) + .fetch()) + .stream() .map(r -> new StandardSyncProtocolVersionFlag(r.get(CONNECTION.ID), r.get(CONNECTION.UNSUPPORTED_PROTOCOL_VERSION))) .collect(Collectors.toSet()); } diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index b991f4144deb..b77981c68b0e 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -12,7 +12,6 @@ RUN apt-get update -y && apt-get install -y apache2-utils && rm -rf /var/lib/apt # This variable can be used to update the destination containers that Nginx proxies to. ENV PROXY_PASS_WEB "http://airbyte-webapp:80" ENV PROXY_PASS_API "http://airbyte-server:8001" -ENV PROXY_PASS_MICRONAUT_API "http://airbyte-server:8080" ENV CONNECTOR_BUILDER_SERVER_API "http://airbyte-connector-builder-server:80" # Nginx config file diff --git a/airbyte-proxy/nginx-auth.conf.template b/airbyte-proxy/nginx-auth.conf.template index c90fcc5dba77..fe817b622536 100644 --- a/airbyte-proxy/nginx-auth.conf.template +++ b/airbyte-proxy/nginx-auth.conf.template @@ -27,40 +27,6 @@ http { auth_basic off; } } - - location ~ ^/api/v1/(health|openapi) { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - auth_basic "Welcome to Airbyte"; - auth_basic_user_file /etc/nginx/.htpasswd; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - - error_page 401 /etc/nginx/401.html; - location ~ (401.html)$ { - alias /etc/nginx/$1; - auth_basic off; - } - } - - location ~ ^/api/v1/(connections|destinations|destination_definitions|destination_definition_specifications|destination_oauths|jobs|logs|notifications|operations|scheduler|source_oauths|sources|source_definitions|source_definition_specifications|state|web_backend|workspaces)/.* { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - auth_basic "Welcome to Airbyte"; - auth_basic_user_file /etc/nginx/.htpasswd; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - - error_page 401 /etc/nginx/401.html; - location ~ (401.html)$ { - alias /etc/nginx/$1; - auth_basic off; - } - } } server { @@ -89,40 +55,6 @@ http { auth_basic off; } } - - location ~ ^/api/v1/(health|openapi) { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - auth_basic "Welcome to Airbyte"; - auth_basic_user_file /etc/nginx/.htpasswd; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - - error_page 401 /etc/nginx/401.html; - location ~ (401.html)$ { - alias /etc/nginx/$1; - auth_basic off; - } - } - - location ~ ^/api/v1/(connections|destinations|destination_definitions|destination_definition_specifications|destination_oauths|jobs|logs|notifications|operations|scheduler|source_oauths|sources|source_definitions|source_definition_specifications|state|web_backend|workspaces)/.* { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - auth_basic "Welcome to Airbyte"; - auth_basic_user_file /etc/nginx/.htpasswd; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - - error_page 401 /etc/nginx/401.html; - location ~ (401.html)$ { - alias /etc/nginx/$1; - auth_basic off; - } - } } server { diff --git a/airbyte-proxy/nginx-no-auth.conf.template b/airbyte-proxy/nginx-no-auth.conf.template index bbe01b765484..35b25c934616 100644 --- a/airbyte-proxy/nginx-no-auth.conf.template +++ b/airbyte-proxy/nginx-no-auth.conf.template @@ -18,22 +18,6 @@ http { proxy_read_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; } - - location ~ ^/api/v1/(health|openapi) { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - } - - location ~ ^/api/v1/(connections|destinations|destination_definitions|destination_definition_specifications|destination_oauths|jobs|logs|notifications|operations|scheduler|source_oauths|sources|source_definitions|source_definition_specifications|state|web_backend|workspaces)/.* { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - } } server { @@ -53,22 +37,6 @@ http { proxy_read_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; } - - location ~ ^/api/v1/(health|openapi) { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - } - - location ~ ^/api/v1/(connections|destinations|destination_definitions|destination_definition_specifications|destination_oauths|jobs|logs|notifications|operations|scheduler|source_oauths|sources|source_definitions|source_definition_specifications|state|web_backend|workspaces)/.* { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_pass "${PROXY_PASS_MICRONAUT_API}"; - } } server { diff --git a/airbyte-proxy/run.sh b/airbyte-proxy/run.sh index 889111dc65a1..264026393984 100644 --- a/airbyte-proxy/run.sh +++ b/airbyte-proxy/run.sh @@ -16,7 +16,7 @@ else TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template" fi -envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${PROXY_PASS_MICRONAUT_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER} ${BASIC_AUTH_PROXY_TIMEOUT}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf +envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER} ${BASIC_AUTH_PROXY_TIMEOUT}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf echo "starting nginx..." nginx -v diff --git a/airbyte-proxy/test.sh b/airbyte-proxy/test.sh index 20075f6e1bcf..c3e66d327361 100755 --- a/airbyte-proxy/test.sh +++ b/airbyte-proxy/test.sh @@ -12,14 +12,14 @@ VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environmen echo "testing with proxy container airbyte/proxy:$VERSION" function start_container () { - CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env BASIC_AUTH_PROXY_TIMEOUT=$3 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env PROXY_PASS_MICRONAUT_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION" + CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env BASIC_AUTH_PROXY_TIMEOUT=$3 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION" echo $CMD eval $CMD wait_for_docker; } function start_container_with_proxy () { - CMD="docker run -d -p $PORT:8000 --env PROXY_PASS_WEB=$1 --env PROXY_PASS_API=$1 --env PROXY_PASS_MICRONAUT_API=$1 --name $NAME + CMD="docker run -d -p $PORT:8000 --env PROXY_PASS_WEB=$1 --env PROXY_PASS_API=$1 --name $NAME airbyte/proxy:$VERSION" echo $CMD eval $CMD diff --git a/airbyte-server/src/main/java/io/airbyte/server/EventListener.java b/airbyte-server/src/main/java/io/airbyte/server/EventListener.java deleted file mode 100644 index 096dceee0c65..000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/EventListener.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server; - -import io.airbyte.commons.lang.CloseableShutdownHook; -import io.airbyte.config.Configs; -import io.airbyte.config.EnvConfigs; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.FlywayFactory; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; -import io.micronaut.runtime.event.ApplicationStartupEvent; -import io.micronaut.scheduling.TaskExecutors; -import io.micronaut.scheduling.annotation.ExecuteOn; -import jakarta.inject.Singleton; -import java.sql.Connection; -import javax.sql.DataSource; -import lombok.extern.slf4j.Slf4j; -import org.flywaydb.core.Flyway; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DSL; - -@Singleton -@Slf4j -public class EventListener { - - private static final String DRIVER_CLASS_NAME = "org.postgresql.Driver"; - - @io.micronaut.runtime.event.annotation.EventListener - @ExecuteOn(TaskExecutors.IO) - @SuppressWarnings({"PMD.AvoidCatchingThrowable", "PMD.DoNotTerminateVM"}) - public void startEmitters(final ApplicationStartupEvent event) { - try { - /* - * In order to have a smooth transition to micronaut for the server, we are starting 2 server. One - * managed by glassfish (legacy), one by micronaut. Once all the controller are migrated, this will - * go away. - */ - final Configs configs = new EnvConfigs(); - - final DataSource configDataSource = - DataSourceFactory.create(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), DRIVER_CLASS_NAME, - configs.getConfigDatabaseUrl()); - final DataSource jobsDataSource = DataSourceFactory.create(configs.getDatabaseUser(), configs.getDatabasePassword(), DRIVER_CLASS_NAME, - configs.getDatabaseUrl()); - - // Manual configuration that will be replaced by Dependency Injection in the future - try (final Connection configsConnection = configDataSource.getConnection(); - final Connection jobsConnection = jobsDataSource.getConnection()) { - configsConnection.setAutoCommit(false); - jobsConnection.setAutoCommit(false); - - final DSLContext configsDslContext = DSL.using(configsConnection, SQLDialect.POSTGRES); - final DSLContext jobsDslContext = DSL.using(jobsConnection, SQLDialect.POSTGRES); - - // Ensure that the database resources are closed on application shutdown - CloseableShutdownHook.registerRuntimeShutdownHook(configsConnection, jobsConnection, configsDslContext, jobsDslContext); - - final Flyway configsFlyway = FlywayFactory.create(configDataSource, ServerApp.class.getSimpleName(), - ConfigsDatabaseMigrator.DB_IDENTIFIER, ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - final Flyway jobsFlyway = FlywayFactory.create(jobsDataSource, ServerApp.class.getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, - JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); - - ServerApp.getServer(new ServerFactory.Api(), configs, configsDslContext, configsFlyway, jobsDslContext, jobsFlyway).start(); - } - } catch (final Throwable e) { - log.error("Server failed", e); - System.exit(1); // so the app doesn't hang on background thread - } - } - -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 4142c1762f33..8ea6aa3e79fb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -15,6 +15,7 @@ import io.airbyte.commons.temporal.TemporalClient; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.TemporalWorkflowUtils; +import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; import io.airbyte.config.helpers.LogClientSingleton; @@ -71,6 +72,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; @@ -280,8 +282,17 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, connectionsHandler, envVariableFeatureFlags); - final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, syncSchedulerClient, - destinationHandler); + final AirbyteProtocolVersionRange airbyteProtocolVersionRange = new AirbyteProtocolVersionRange(configs.getAirbyteProtocolVersionMin(), + configs.getAirbyteProtocolVersionMax()); + + final AirbyteGithubStore airbyteGithubStore = AirbyteGithubStore.production(); + + final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, + () -> UUID.randomUUID(), + syncSchedulerClient, + airbyteGithubStore, + destinationHandler, + airbyteProtocolVersionRange); final HealthCheckHandler healthCheckHandler = new HealthCheckHandler(configRepository); @@ -295,7 +306,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, connectionsHandler); final SourceDefinitionsHandler sourceDefinitionsHandler = - new SourceDefinitionsHandler(configRepository, syncSchedulerClient, sourceHandler, configs); + new SourceDefinitionsHandler(configRepository, () -> UUID.randomUUID(), syncSchedulerClient, airbyteGithubStore, sourceHandler, + airbyteProtocolVersionRange); final JobHistoryHandler jobHistoryHandler = new JobHistoryHandler( jobPersistence, diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java index 1b879da13123..f3c43fdfdf32 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/LogsApiController.java @@ -12,7 +12,7 @@ import io.micronaut.http.annotation.Post; import java.io.File; -@Controller("/v1/logs") +@Controller("/api/v1/logs") @Context public class LogsApiController implements LogsApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java new file mode 100644 index 000000000000..b898261dac47 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/NotFoundController.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Controller; +import io.micronaut.http.annotation.Error; + +/** + * Custom controller that handles global 404 responses for unknown/unmapped paths. + */ +@Controller("/api/notfound") +public class NotFoundController { + + @Error(status = HttpStatus.NOT_FOUND, + global = true) + public HttpResponse notFound(final HttpRequest request) { + // Would like to send the id along but we don't have access to the http request anymore to fetch it + // from. TODO: Come back to this with issue #4189 + return HttpResponse.status(HttpStatus.NOT_FOUND) + .body("Object not found.") + .contentType(MediaType.APPLICATION_JSON); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java index 3be2ba02717b..6371ddccbd85 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java @@ -22,7 +22,6 @@ import io.airbyte.server.handlers.WebBackendGeographiesHandler; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; -import javax.ws.rs.Path; @Controller("/api/v1/web_backend") public class WebBackendApiController implements WebBackendApi { diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java index 75c9b8f10645..bd24592dceaf 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java @@ -14,7 +14,6 @@ import io.airbyte.commons.version.Version; import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.Configs.TrackingStrategy; -import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.persistence.job.JobPersistence; diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java index 90747dce622b..3f4533e9cddc 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java @@ -96,7 +96,6 @@ public StatePersistence statePersistence(@Named("configDatabase") final Database return new StatePersistence(configDatabase); } - @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @Named("configsDatabaseMigrationCheck") diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/JobErrorReportingBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/JobErrorReportingBeanFactory.java index 979361e3d4e1..3da39a0545f4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/JobErrorReportingBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/JobErrorReportingBeanFactory.java @@ -13,7 +13,6 @@ import io.airbyte.persistence.job.errorreporter.LoggingJobErrorReportingClient; import io.airbyte.persistence.job.errorreporter.SentryExceptionHelper; import io.airbyte.persistence.job.errorreporter.SentryJobErrorReportingClient; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; @@ -59,8 +58,6 @@ public JobErrorReporter jobErrorReporter( configRepository, deploymentMode, airbyteVersion, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME, - NormalizationRunnerFactory.NORMALIZATION_VERSION, webUrlHelper, jobErrorReportingClient.orElseGet(() -> new LoggingJobErrorReportingClient())); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index ef3630191ab7..c1a19d84ffcb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -50,9 +50,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -@Slf4j @Singleton public class JobConverter { diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/IdNotFoundExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/IdNotFoundExceptionMapper.java new file mode 100644 index 000000000000..4cd9b709cf04 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/IdNotFoundExceptionMapper.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.errors; + +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Produces; +import io.micronaut.http.server.exceptions.ExceptionHandler; +import jakarta.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Produces +@Singleton +@Requires(classes = IdNotFoundKnownException.class) +public class IdNotFoundExceptionMapper implements ExceptionHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(IdNotFoundExceptionMapper.class); + + @Override + public HttpResponse handle(final HttpRequest request, final IdNotFoundKnownException exception) { + final IdNotFoundKnownException idnf = new IdNotFoundKnownException("Id not found: " + exception.getMessage(), exception); + LOGGER.error("Not found exception", idnf.getNotFoundKnownExceptionInfo()); + + return HttpResponse.status(HttpStatus.NOT_FOUND) + .body(KnownException.infoFromThrowableWithMessage(exception, "Internal Server Error: " + exception.getMessage())) + .contentType(MediaType.APPLICATION_JSON); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidInputExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidInputExceptionMapper.java index 44629340c4ac..26a3c9ca2947 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidInputExceptionMapper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidInputExceptionMapper.java @@ -7,19 +7,26 @@ import io.airbyte.api.model.generated.InvalidInputExceptionInfo; import io.airbyte.api.model.generated.InvalidInputProperty; import io.airbyte.commons.json.Jsons; +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Produces; +import io.micronaut.http.server.exceptions.ExceptionHandler; +import jakarta.inject.Singleton; import java.util.ArrayList; import java.util.List; import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolationException; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; import org.apache.logging.log4j.core.util.Throwables; // https://www.baeldung.com/jersey-bean-validation#custom-exception-handler // handles exceptions related to the request body not matching the openapi config. -@Provider -public class InvalidInputExceptionMapper implements ExceptionMapper { +@Produces +@Singleton +@Requires(classes = ConstraintViolationException.class) +public class InvalidInputExceptionMapper implements ExceptionHandler { public static InvalidInputExceptionInfo infoFromConstraints(final ConstraintViolationException cve) { final InvalidInputExceptionInfo exceptionInfo = new InvalidInputExceptionInfo() @@ -39,11 +46,10 @@ public static InvalidInputExceptionInfo infoFromConstraints(final ConstraintViol } @Override - public Response toResponse(final ConstraintViolationException e) { - return Response.status(Response.Status.BAD_REQUEST) - .entity(Jsons.serialize(InvalidInputExceptionMapper.infoFromConstraints(e))) - .type("application/json") - .build(); + public HttpResponse handle(final HttpRequest request, final ConstraintViolationException exception) { + return HttpResponse.status(HttpStatus.BAD_REQUEST) + .body(Jsons.serialize(InvalidInputExceptionMapper.infoFromConstraints(exception))) + .contentType(MediaType.APPLICATION_JSON_TYPE); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonExceptionMapper.java index c126f1299325..c1e88694958a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonExceptionMapper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonExceptionMapper.java @@ -5,19 +5,26 @@ package io.airbyte.server.errors; import com.fasterxml.jackson.core.JsonParseException; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Produces; +import io.micronaut.http.server.exceptions.ExceptionHandler; +import jakarta.inject.Singleton; -@Provider -public class InvalidJsonExceptionMapper implements ExceptionMapper { +@Produces +@Singleton +@Requires(classes = JsonParseException.class) +public class InvalidJsonExceptionMapper implements ExceptionHandler { @Override - public Response toResponse(final JsonParseException e) { - return Response.status(422) - .entity(KnownException.infoFromThrowableWithMessage(e, "Invalid json. " + e.getMessage() + " " + e.getOriginalMessage())) - .type("application/json") - .build(); + public HttpResponse handle(final HttpRequest request, final JsonParseException exception) { + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY) + .body( + KnownException.infoFromThrowableWithMessage(exception, "Invalid json. " + exception.getMessage() + " " + exception.getOriginalMessage())) + .contentType(MediaType.APPLICATION_JSON_TYPE); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonInputExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonInputExceptionMapper.java index 90700f771782..06e917f9119c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonInputExceptionMapper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/InvalidJsonInputExceptionMapper.java @@ -5,21 +5,26 @@ package io.airbyte.server.errors; import com.fasterxml.jackson.databind.JsonMappingException; -import io.airbyte.commons.json.Jsons; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Produces; +import io.micronaut.http.server.exceptions.ExceptionHandler; +import jakarta.inject.Singleton; -@Provider -public class InvalidJsonInputExceptionMapper implements ExceptionMapper { +@Produces +@Singleton +@Requires(classes = JsonMappingException.class) +public class InvalidJsonInputExceptionMapper implements ExceptionHandler { @Override - public Response toResponse(final JsonMappingException e) { - return Response.status(422) - .entity( - Jsons.serialize(KnownException.infoFromThrowableWithMessage(e, "Invalid json input. " + e.getMessage() + " " + e.getOriginalMessage()))) - .type("application/json") - .build(); + public HttpResponse handle(final HttpRequest request, final JsonMappingException exception) { + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY) + .body(KnownException.infoFromThrowableWithMessage(exception, + "Invalid json input. " + exception.getMessage() + " " + exception.getOriginalMessage())) + .contentType(MediaType.APPLICATION_JSON_TYPE); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/KnownExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/KnownExceptionMapper.java index b849e30d5298..9d81a3fc7174 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/errors/KnownExceptionMapper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/KnownExceptionMapper.java @@ -5,13 +5,17 @@ package io.airbyte.server.errors; import io.airbyte.commons.json.Jsons; +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.annotation.Produces; +import jakarta.inject.Singleton; import javax.ws.rs.core.Response; import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Provider +@Produces +@Singleton +@Requires(classes = KnownException.class) public class KnownExceptionMapper implements ExceptionMapper { private static final Logger LOGGER = LoggerFactory.getLogger(KnownExceptionMapper.class); diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/NotFoundExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/NotFoundExceptionMapper.java index cda326f8856f..28f223d64114 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/errors/NotFoundExceptionMapper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/NotFoundExceptionMapper.java @@ -4,27 +4,33 @@ package io.airbyte.server.errors; -import io.airbyte.commons.json.Jsons; +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Produces; +import io.micronaut.http.server.exceptions.ExceptionHandler; +import jakarta.inject.Singleton; import javax.ws.rs.NotFoundException; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NotFoundExceptionMapper implements ExceptionMapper { +@Produces +@Singleton +@Requires(classes = NotFoundException.class) +public class NotFoundExceptionMapper implements ExceptionHandler { private static final Logger LOGGER = LoggerFactory.getLogger(NotFoundExceptionMapper.class); @Override - public Response toResponse(final NotFoundException e) { - // Would like to send the id along but we don't have access to the http request anymore to fetch it - // from. TODO: Come back to this with issue #4189 - final IdNotFoundKnownException idnf = new IdNotFoundKnownException("Object not found. " + e.getMessage(), e); + public HttpResponse handle(final HttpRequest request, final NotFoundException exception) { + final IdNotFoundKnownException idnf = new IdNotFoundKnownException("Object not found. " + exception.getMessage(), exception); LOGGER.error("Not found exception", idnf.getNotFoundKnownExceptionInfo()); - return Response.status(404) - .entity(Jsons.serialize(idnf.getNotFoundKnownExceptionInfo())) - .type("application/json") - .build(); + + return HttpResponse.status(HttpStatus.NOT_FOUND) + .body(KnownException.infoFromThrowableWithMessage(exception, "Internal Server Error: " + exception.getMessage())) + .contentType(MediaType.APPLICATION_JSON); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/errors/UncaughtExceptionMapper.java b/airbyte-server/src/main/java/io/airbyte/server/errors/UncaughtExceptionMapper.java index 3aac5522570e..a9e181d9a6b0 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/errors/UncaughtExceptionMapper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/errors/UncaughtExceptionMapper.java @@ -4,27 +4,30 @@ package io.airbyte.server.errors; -import io.airbyte.api.model.generated.KnownExceptionInfo; -import io.airbyte.commons.json.Jsons; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import io.micronaut.context.annotation.Requires; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Produces; +import io.micronaut.http.server.exceptions.ExceptionHandler; +import jakarta.inject.Singleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Provider -public class UncaughtExceptionMapper implements ExceptionMapper { +@Produces +@Singleton +@Requires(classes = Throwable.class) +public class UncaughtExceptionMapper implements ExceptionHandler { private static final Logger LOGGER = LoggerFactory.getLogger(UncaughtExceptionMapper.class); @Override - public Response toResponse(final Throwable e) { - LOGGER.error("Uncaught exception", e); - final KnownExceptionInfo exceptionInfo = KnownException.infoFromThrowableWithMessage(e, "Internal Server Error: " + e.getMessage()); - return Response.status(500) - .entity(Jsons.serialize(exceptionInfo)) - .type("application/json") - .build(); + public HttpResponse handle(final HttpRequest request, final Throwable exception) { + LOGGER.error("Uncaught exception", exception); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(KnownException.infoFromThrowableWithMessage(exception, "Internal Server Error: " + exception.getMessage())) + .contentType(MediaType.APPLICATION_JSON); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 8f04307e325a..6b8dfe56accb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -27,8 +27,6 @@ import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.ActorType; -import io.airbyte.config.Configs; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -64,27 +62,19 @@ public class DestinationDefinitionsHandler { private final DestinationHandler destinationHandler; private final AirbyteProtocolVersionRange protocolVersionRange; - public DestinationDefinitionsHandler(final ConfigRepository configRepository, - final SynchronousSchedulerClient schedulerSynchronousClient, - final DestinationHandler destinationHandler) { - this(configRepository, UUID::randomUUID, schedulerSynchronousClient, AirbyteGithubStore.production(), destinationHandler); - } - @VisibleForTesting public DestinationDefinitionsHandler(final ConfigRepository configRepository, final Supplier uuidSupplier, final SynchronousSchedulerClient schedulerSynchronousClient, final AirbyteGithubStore githubStore, - final DestinationHandler destinationHandler) { + final DestinationHandler destinationHandler, + final AirbyteProtocolVersionRange protocolVersionRange) { this.configRepository = configRepository; this.uuidSupplier = uuidSupplier; this.schedulerSynchronousClient = schedulerSynchronousClient; this.githubStore = githubStore; this.destinationHandler = destinationHandler; - - // TODO inject protocol min and max once this handler is being converted to micronaut - final Configs configs = new EnvConfigs(); - protocolVersionRange = new AirbyteProtocolVersionRange(configs.getAirbyteProtocolVersionMin(), configs.getAirbyteProtocolVersionMax()); + this.protocolVersionRange = protocolVersionRange; } @VisibleForTesting diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java index 77fd22b8f93e..b9ac2378aca4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/LogsHandler.java @@ -33,7 +33,8 @@ public LogsHandler(final Configs configs) { @Inject public LogsHandler(@Named("workspaceRoot") final Path workspaceRoot, - final WorkerEnvironment workerEnvironment, final LogConfigs logConfigs) { + final WorkerEnvironment workerEnvironment, + final LogConfigs logConfigs) { this.workspaceRoot = workspaceRoot; this.workerEnvironment = workerEnvironment; this.logConfigs = logConfigs; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 490208d1c725..a212e4d52dad 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -391,8 +391,8 @@ public JobInfoRead cancelJob(final JobIdRequestBody jobIdRequestBody) throws IOE // wants the connection disabled when non-breaking changes are detected. If so, disable that // connection. Modify the current discoveredSchema object to add a CatalogDiff, // containsBreakingChange paramter, and connectionStatus parameter. - private void generateCatalogDiffsAndDisableConnectionsIfNeeded(SourceDiscoverSchemaRead discoveredSchema, - SourceDiscoverSchemaRequestBody discoverSchemaRequestBody) + private void generateCatalogDiffsAndDisableConnectionsIfNeeded(final SourceDiscoverSchemaRead discoveredSchema, + final SourceDiscoverSchemaRequestBody discoverSchemaRequestBody) throws JsonValidationException, ConfigNotFoundException, IOException { final ConnectionReadList connectionsForSource = connectionsHandler.listConnectionsForSource(discoverSchemaRequestBody.getSourceId(), false); for (final ConnectionRead connectionRead : connectionsForSource.getConnections()) { @@ -400,12 +400,13 @@ private void generateCatalogDiffsAndDisableConnectionsIfNeeded(SourceDiscoverSch .getConnectionAirbyteCatalog(connectionRead.getConnectionId()); final io.airbyte.api.model.generated.@NotNull AirbyteCatalog currentAirbyteCatalog = connectionRead.getSyncCatalog(); - CatalogDiff diff = connectionsHandler.getDiff(catalogUsedToMakeConfiguredCatalog.orElse(currentAirbyteCatalog), discoveredSchema.getCatalog(), - CatalogConverter.toProtocol(currentAirbyteCatalog)); - boolean containsBreakingChange = containsBreakingChange(diff); - ConnectionUpdate updateObject = + final CatalogDiff diff = + connectionsHandler.getDiff(catalogUsedToMakeConfiguredCatalog.orElse(currentAirbyteCatalog), discoveredSchema.getCatalog(), + CatalogConverter.toProtocol(currentAirbyteCatalog)); + final boolean containsBreakingChange = containsBreakingChange(diff); + final ConnectionUpdate updateObject = new ConnectionUpdate().breakingChange(containsBreakingChange).connectionId(connectionRead.getConnectionId()); - ConnectionStatus connectionStatus; + final ConnectionStatus connectionStatus; if (shouldDisableConnection(containsBreakingChange, connectionRead.getNonBreakingChangesPreference(), diff)) { connectionStatus = ConnectionStatus.INACTIVE; } else { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java index 5273123beff8..9eb2f77df630 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java @@ -28,7 +28,6 @@ import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.ActorType; -import io.airbyte.config.Configs; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -65,16 +64,6 @@ public class SourceDefinitionsHandler { private final SourceHandler sourceHandler; private final AirbyteProtocolVersionRange protocolVersionRange; - // TODO: remove this contructor after the migration is done - @Deprecated(forRemoval = true) - public SourceDefinitionsHandler(final ConfigRepository configRepository, - final SynchronousSchedulerClient schedulerSynchronousClient, - final SourceHandler sourceHandler, - final Configs configs) { - this(configRepository, UUID::randomUUID, schedulerSynchronousClient, AirbyteGithubStore.production(), sourceHandler, - new AirbyteProtocolVersionRange(configs.getAirbyteProtocolVersionMin(), configs.getAirbyteProtocolVersionMax())); - } - @Inject public SourceDefinitionsHandler(final ConfigRepository configRepository, final Supplier uuidSupplier, diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index 01c20783846d..58441b2cb10f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -393,7 +393,8 @@ public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnecti return buildWebBackendConnectionRead(connection, currentSourceCatalogId).catalogDiff(diff); } - private AirbyteCatalog updateSchemaWithOriginalDiscoveredCatalog(final AirbyteCatalog configuredCatalog, final AirbyteCatalog originalDiscoveredCatalog) { + private AirbyteCatalog updateSchemaWithOriginalDiscoveredCatalog(final AirbyteCatalog configuredCatalog, + final AirbyteCatalog originalDiscoveredCatalog) { // We pass the original discovered catalog in as the "new" discovered catalog. return updateSchemaWithRefreshedDiscoveredCatalog(configuredCatalog, originalDiscoveredCatalog, originalDiscoveredCatalog); } diff --git a/airbyte-server/src/main/resources/application-control-plane.yml b/airbyte-server/src/main/resources/application-control-plane.yml index 52eb26eac447..9213a778a363 100644 --- a/airbyte-server/src/main/resources/application-control-plane.yml +++ b/airbyte-server/src/main/resources/application-control-plane.yml @@ -3,9 +3,8 @@ datasources: connection-test-query: SELECT 1 connection-timeout: 30000 maximum-pool-size: 10 - minimum-pool-size: 0 - minimum-idle: 1 - maximum-idle: 1000 + minimum-idle: 0 + idle-timeout: 600000 url: ${DATABASE_URL} driverClassName: org.postgresql.Driver username: ${DATABASE_USER} diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 3b6c04b20adf..10b944a4ac71 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -12,7 +12,7 @@ micronaut: access: - isAnonymous() server: - port: 8080 + port: 8001 airbyte: deployment-mode: ${DEPLOYMENT_MODE:OSS} flyway: diff --git a/airbyte-server/src/main/resources/banner/banner.txt b/airbyte-server/src/main/resources/banner/banner.txt deleted file mode 100644 index 0c432cdb9496..000000000000 --- a/airbyte-server/src/main/resources/banner/banner.txt +++ /dev/null @@ -1,10 +0,0 @@ - - ___ _ __ __ - / | (_)____/ /_ __ __/ /____ - / /| | / / ___/ __ \/ / / / __/ _ \ - / ___ |/ / / / /_/ / /_/ / /_/ __/ -/_/ |_/_/_/ /_.___/\__, /\__/\___/ - /____/ --------------------------------------- - Now ready at http://localhost:8000/ --------------------------------------- diff --git a/airbyte-server/src/main/resources/micronaut-banner.txt b/airbyte-server/src/main/resources/micronaut-banner.txt index cf7217291086..83ea519237a6 100644 --- a/airbyte-server/src/main/resources/micronaut-banner.txt +++ b/airbyte-server/src/main/resources/micronaut-banner.txt @@ -6,3 +6,6 @@ /_/ |_/_/_/ /_.___/\__, /\__/\___/ /____/ : airbyte-server : +-------------------------------------------------- + Will soon be available at http://localhost:8000/ +-------------------------------------------------- diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index 32ba33b311bd..6d61c0f44efe 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -32,6 +32,7 @@ import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.ActorType; import io.airbyte.config.Configs; @@ -92,7 +93,8 @@ void setUp() { uuidSupplier, schedulerSynchronousClient, githubStore, - destinationHandler); + destinationHandler, + new AirbyteProtocolVersionRange(new Version("0.0.0"), new Version("0.3.0"))); } private StandardDestinationDefinition generateDestinationDefinition() { diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index 957f80de8885..a0335429dabe 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -159,6 +159,11 @@ class SchedulerHandlerTest { void setup() { completedJob = mock(Job.class, RETURNS_DEEP_STUBS); jobResponse = mock(SynchronousResponse.class, RETURNS_DEEP_STUBS); + final SynchronousJobMetadata synchronousJobMetadata = mock(SynchronousJobMetadata.class); + when(synchronousJobMetadata.getConfigType()) + .thenReturn(ConfigType.SYNC); + when(jobResponse.getMetadata()) + .thenReturn(synchronousJobMetadata); configurationUpdate = mock(ConfigurationUpdate.class); jsonSchemaValidator = mock(JsonSchemaValidator.class); when(completedJob.getStatus()).thenReturn(JobStatus.SUCCEEDED); @@ -952,9 +957,9 @@ void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() throws IOExce assertEquals(expectedActorCatalog, actual.getCatalog()); assertEquals(ConnectionStatus.ACTIVE, actual.getConnectionStatus()); - ArgumentCaptor expectedArgumentCaptor = ArgumentCaptor.forClass(ConnectionUpdate.class); + final ArgumentCaptor expectedArgumentCaptor = ArgumentCaptor.forClass(ConnectionUpdate.class); verify(connectionsHandler, times(3)).updateConnection(expectedArgumentCaptor.capture()); - List connectionUpdateValues = expectedArgumentCaptor.getAllValues(); + final List connectionUpdateValues = expectedArgumentCaptor.getAllValues(); assertEquals(ConnectionStatus.ACTIVE, connectionUpdateValues.get(0).getStatus()); assertEquals(ConnectionStatus.ACTIVE, connectionUpdateValues.get(1).getStatus()); assertEquals(ConnectionStatus.INACTIVE, connectionUpdateValues.get(2).getStatus()); diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java b/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java index 54d7330573d3..321f6cdd9ad7 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java @@ -113,10 +113,6 @@ private static void waitForAirbyte() throws InterruptedException { new ApiClient().setScheme("http") .setHost("localhost") .setPort(8001) - .setBasePath("/api"), - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8080) .setBasePath("/api")); final HealthApi healthApi = apiClient.getHealthApi(); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java index 613c059ce94b..216159195f1c 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java @@ -98,10 +98,6 @@ static void init() throws URISyntaxException, IOException, InterruptedException, new ApiClient().setScheme("http") .setHost("localhost") .setPort(8001) - .setBasePath("/api"), - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8080) .setBasePath("/api")); // work in whatever default workspace is present. workspaceId = apiClient.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index d103033539f7..92f8f53ce8eb 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -174,10 +174,6 @@ static void init() throws URISyntaxException, IOException, InterruptedException, new ApiClient().setScheme("http") .setHost("localhost") .setPort(8001) - .setBasePath("/api"), - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8080) .setBasePath("/api")); webBackendApi = new WebBackendApi( new ApiClient().setScheme("http") @@ -280,7 +276,7 @@ void testDestinationCheckConnection() throws ApiException { @Test @Order(3) - void testCreateSource() throws ApiException { + void wtestCreateSource() throws ApiException { final String dbName = "acc-test-db"; final UUID postgresSourceDefinitionId = testHarness.getPostgresSourceDefinitionId(); final JsonNode sourceDbConfig = testHarness.getSourceDbConfig(); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java index 32bde1bea029..807d6a7c73a0 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java @@ -125,10 +125,6 @@ static void init() throws URISyntaxException, IOException, InterruptedException, new ApiClient().setScheme("http") .setHost("localhost") .setPort(8001) - .setBasePath("/api"), - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8080) .setBasePath("/api")); webBackendApi = new WebBackendApi( new ApiClient().setScheme("http") diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java index 4c7c15d6892c..642dda066c38 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java @@ -72,10 +72,6 @@ static void init() throws URISyntaxException, IOException, InterruptedException, new ApiClient().setScheme("http") .setHost("localhost") .setPort(8001) - .setBasePath("/api"), - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8080) .setBasePath("/api")); // work in whatever default workspace is present. workspaceId = apiClient.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java index 8e67990c5612..9147d1257d82 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java @@ -43,10 +43,6 @@ static void init() throws ApiException, URISyntaxException, IOException, Interru new ApiClient().setScheme("http") .setHost("localhost") .setPort(8001) - .setBasePath("/api"), - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8080) .setBasePath("/api")); workspaceId = apiClient.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java index f3f39242bfe3..ece74f42893e 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java @@ -60,32 +60,8 @@ public ApiClient apiClient( } @Singleton - @Named("micronautApiClient") - public ApiClient micronautApiClient( - @Value("${airbyte.internal.api.auth-header.name}") final String airbyteApiAuthHeaderName, - @Value("${airbyte.internal.api.micronaut-host}") final String airbyteMicronautApiHost, - @Named("internalApiAuthToken") final BeanProvider internalApiAuthToken, - @Named("internalApiScheme") final String internalApiScheme) { - return new io.airbyte.api.client.invoker.generated.ApiClient() - .setScheme(internalApiScheme) - .setHost(parseHostName(airbyteMicronautApiHost)) - .setPort(parsePort(airbyteMicronautApiHost)) - .setBasePath("/api") - .setHttpClientBuilder(HttpClient.newBuilder().version(Version.HTTP_1_1)) - .setRequestInterceptor(builder -> { - builder.setHeader("User-Agent", "WorkerApp"); - // internalApiAuthToken is in BeanProvider because we want to create a new token each - // time we send a request. - if (!airbyteApiAuthHeaderName.isBlank()) { - builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); - } - }); - } - - @Singleton - public AirbyteApiClient airbyteApiClient(@Named("apiClient") final ApiClient apiClient, - @Named("micronautApiClient") final ApiClient micronautApiClient) { - return new AirbyteApiClient(apiClient, micronautApiClient); + public AirbyteApiClient airbyteApiClient(@Named("apiClient") final ApiClient apiClient) { + return new AirbyteApiClient(apiClient); } @Singleton diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index cfc2ff14efd3..43708b53d805 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -81,7 +81,6 @@ airbyte: name: ${AIRBYTE_API_AUTH_HEADER_NAME:} value: ${AIRBYTE_API_AUTH_HEADER_VALUE:} host: ${INTERNAL_API_HOST} - micronaut-host: ${INTERNAL_MICRONAUT_API_HOST} local: docker-mount: ${LOCAL_DOCKER_MOUNT:} root: ${LOCAL_ROOT} diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml index fca47b894c1c..ea026012ad73 100644 --- a/charts/airbyte-worker/templates/deployment.yaml +++ b/charts/airbyte-worker/templates/deployment.yaml @@ -251,11 +251,6 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: INTERNAL_API_HOST - - name: INTERNAL_MICRONAUT_API_HOST - valueFrom: - configMapKeyRef: - name: { { .Release.Name } }-airbyte-env - key: INTERNAL_MICRONAUT_API_HOST - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION valueFrom: configMapKeyRef: diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index e8676c9b4bc6..a219395275e4 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -23,7 +23,6 @@ data: GCS_LOG_BUCKET: {{ .Values.global.logs.gcs.bucket | quote }} GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} INTERNAL_API_HOST: {{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} - INTERNAL_MICRONAUT_API_HOST: {{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} CONNECTOR_BUILDER_API_HOST: {{ .Release.Name }}-airbyte-connector-builder-server-svc:{{ .Values.connectorBuilderServer.service.port }} {{- if $.Values.global.jobs.kube.annotations }} JOB_KUBE_ANNOTATIONS: {{ $.Values.global.jobs.kube.annotations | include "airbyte.flattenMap" | quote }} diff --git a/deps.toml b/deps.toml index a1da1086ca8a..0843a03affc9 100644 --- a/deps.toml +++ b/deps.toml @@ -108,6 +108,7 @@ jakarta-inject = { module = "jakarta.annotation:jakarta.annotation-api", version javax-transaction = { module = "javax.transaction:javax.transaction-api", version = "1.3" } micronaut-bom = { module = "io.micronaut:micronaut-bom", version.ref = "micronaut" } micronaut-data-processor = { module = "io.micronaut.data:micronaut-data-processor", version = "3.8.1" } +micronaut-data-tx = { module = "io.micronaut.data:micronaut-data-tx", version = "3.8.1" } micronaut-flyway = { module = "io.micronaut.flyway:micronaut-flyway", version = "5.4.1" } micronaut-inject = { module = "io.micronaut:micronaut-inject" } micronaut-http-client = { module = "io.micronaut:micronaut-http-client" } diff --git a/docker-compose.yaml b/docker-compose.yaml index 11ae0106a9ef..4cfae8692a77 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -70,7 +70,6 @@ services: - DATABASE_USER=${DATABASE_USER} - DEPLOYMENT_MODE=${DEPLOYMENT_MODE} - INTERNAL_API_HOST=${INTERNAL_API_HOST} - - INTERNAL_MICRONAUT_API_HOST=${INTERNAL_MICRONAUT_API_HOST} - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-} - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT} - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST} @@ -153,7 +152,6 @@ services: - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA} ports: - 8001 - - 8080:8080 volumes: - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT} diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index 46b31600341c..c0d8b1754bde 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -30,7 +30,6 @@ WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 -INTERNAL_MICRONAUT_API_HOST=airbyte-server-svc:8080 CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index e39606d3f985..8762e2ea5914 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -32,7 +32,6 @@ WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 -INTERNAL_MICRONAUT_API_HOST=airbyte-server-svc:8080 CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 2ea47d5d3046..8dc6564b31ed 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -32,7 +32,6 @@ WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 -INTERNAL_MICRONAUT_API_HOST=airbyte-server-svc:8080 CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index c46883ab254d..80657585bf81 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -32,7 +32,6 @@ WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 -INTERNAL_MICRONAUT_API_HOST=airbyte-server-svc:8080 CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index 3cb7dd34cc12..d8f7be704011 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -179,11 +179,6 @@ spec: configMapKeyRef: name: airbyte-env key: INTERNAL_API_HOST - - name: INTERNAL_MICRONAUT_API_HOST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: INTERNAL_MICRONAUT_API_HOST - name: JOB_KUBE_TOLERATIONS valueFrom: configMapKeyRef: diff --git a/tools/bin/acceptance_test.sh b/tools/bin/acceptance_test.sh index 5d58326b7892..cbc9806c31f9 100755 --- a/tools/bin/acceptance_test.sh +++ b/tools/bin/acceptance_test.sh @@ -28,7 +28,7 @@ VERSION=dev TRACKING_STRATEGY=logging USE_STREAM_CAPABLE_STATE=true BASIC_AUTH_U # Sometimes source/dest containers using airbyte volumes survive shutdown, which need to be killed in order to shut down properly. shutdown_cmd="docker-compose down -v || docker kill \$(docker ps -a -f volume=airbyte_workspace -f volume=airbyte_data -f volume=airbyte_db -q) && docker-compose down -v" # Uncomment for debugging. Warning, this is verbose. -# trap "echo 'docker-compose logs:' && docker-compose logs -t --tail 1000 && $shutdown_cmd" EXIT +trap "echo 'docker-compose logs:' && docker-compose logs -t --tail 1000 && $shutdown_cmd" EXIT echo "Waiting for services to begin" starttime=`get_epoch_time`