Enable the Connector Builder server in the platform (#19269)

* init

* bad copy/paste

* move to top level

* Revert "move to top level"

This reverts commit aca3534.

* attempt to wire up connector builder frontend to server

* copy from octaviacli

* fix connection to builder server

* update

* delete

* Update

* delete python-version

* Revert "delete python-version"

This reverts commit f9258a7.

* setup python

* install python

* rename

* kube stuff

* Install python

* missing kube file

* rename

* Update files

* Update bumpversion

* install python

* try with different entrypoint

* rename container

* point to docker-compose.yaml file

* derp

* copy acceptance_test.sh

* copy from acceptance tests

* delete cruft

* update

* remove application env

* reset

* reset to master

* update

* skip comprehensive incremental tests

* Revert "skip comprehensive incremental tests"

This reverts commit 9cee657.

* reset to master

* remove cruft

* readme

* readme

* save

* buncha stuff

* it works

* improvements

* comment

* codesnippet

* delete superfluous steps

* update port to 8003

* reset to master

* Update publish docker

* move openapi spec to airbyte-connector-builder

* point to openapi spec

* dont expose the connector builder to localhost

* reset FE components to master

* Don't deploy the connector-builder

* Revert "Don't deploy the connector-builder"

This reverts commit 3d15749.

* Revert "Revert "Don't deploy the connector-builder""

This reverts commit beac3d4.

* comment out more things related to connector builder server

* more attempts at removing the connector builder

* comment out more things

* Update airbyte-connector-builder/README.md

Co-authored-by: Brian Lai <51336873+brianjlai@users.noreply.github.com>

* fixes

* rename gradle task

* save

* copyright header

* Apply suggestions from code review

Co-authored-by: Lake Mossman <lake@airbyte.io>

* Update airbyte-webapp/src/config/configProviders.ts

Co-authored-by: Lake Mossman <lake@airbyte.io>

* update

* rename

* indent

* Revert "move openapi spec to airbyte-connector-builder"

This reverts commit 57dda04.

* Revert "rename"

This reverts commit b2d802b.

* Revert "Revert "rename""

This reverts commit 91db24f.

* point to wrong file in case it fixes the build

* point to right openapi file

* Revert "Revert "move openapi spec to airbyte-connector-builder""

This reverts commit e46a837.

* point to moved file

* fix path

* Update from master

* newline

* Add failing test

* Revert "Add failing test"

This reverts commit ed9fea0.

* comment

* update commented requires

* Add a comment

* comments

* save

* formatting is for losers anyways

* pyproject

* toml

* toml

* saved

* nice

* anothaone

* fmt

* fixmypy

* fix autogen notice header location

* improvecomment

* save working branch

* fmt

* delete license and disable caching

* removeyaml

* fix merge and uncomment flakeCheck

* undo css change

* fix typo

* format

* enable CORS

* move template to backend server

* try adding connector builder server to 'generate-docker' task to see if it fixes CI

* try adding buildDockerImage task

* try adding gradle.properties

* Revert "try adding buildDockerImage task"

This reverts commit 2102009.

* Revert "try adding connector builder server to 'generate-docker' task to see if it fixes CI"

This reverts commit 40b5ae0.

* process detail in apiOverride

* fix gradle

* fix scrollbars

* Run connector builder server tests and static analysis from a docker container (#19495)

* Revert "try adding buildDockerImage task"

This reverts commit 2102009.

* Revert "try adding connector builder server to 'generate-docker' task to see if it fixes CI"

This reverts commit 40b5ae0.

* missing file

* build airbyte-connector-builder-server image

* Add .

* comment

* build connector builder server as part of octavia_cli

* build connector builder too

* patch

* missing file

* comment

* comment

* run unit tests in a container

* Run integration tests

* run

* move to platform build

* format

* rootdir

* get the path from the project

* Update

* pyproject

* delete a bunch of code

* parameterize

* run tests

* update

* rename

* comment

* reset

* uncomment

* dont format

* run everything except isort

* delete dead code

* Revert "delete dead code"

This reverts commit b11df71.

* comment

* Update module

* reset to master

* mount in home

* commit formatted files

* try running format

* Better comment

* reset

* Revert "reset"

This reverts commit 20808f8.

* Update buildSrc/src/main/groovy/airbyte-python-docker.gradle

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

* comment

* Add everything except build and hidden directories

* symlink

* Update airbyte-connector-builder-server/run_format.sh

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

* Update airbyte-connector-builder-server/run_tests.sh

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

Co-authored-by: Sherif A. Nada <snadalive@gmail.com>

Co-authored-by: Alexandre Girard <alexandre@airbyte.io>
Co-authored-by: sherifnada <snadalive@gmail.com>
Co-authored-by: Brian Lai <51336873+brianjlai@users.noreply.github.com>

.env

@@ -57,7 +57,7 @@ CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001
 ### AIRBYTE SERVICES ###
 TEMPORAL_HOST=airbyte-temporal:7233
 INTERNAL_API_HOST=airbyte-server:8001
-#CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80 #FIXME: Uncomment this when enabling the connector-builder
+CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80
 WEBAPP_URL=http://localhost:8000/
 # Although not present as an env var, required for webapp configuration.
 API_URL=/api/v1/

.env.dev

@@ -23,7 +23,7 @@ HACK_LOCAL_ROOT_PARENT=/tmp
 WEBAPP_URL=http://localhost:8000/
 API_URL=/api/v1/
 INTERNAL_API_HOST=airbyte-server:8001
-#CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:8003 #FIXME: Uncomment this when enabling the connector-builder
+CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80
 SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane

airbyte-connector-builder-server/build.gradle

@@ -2,15 +2,15 @@ import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
 
 plugins {
     id "org.openapi.generator" version "5.3.1"
-    id 'airbyte-python'
+    id 'airbyte-python-docker'
     id 'airbyte-docker'
 }
 
-airbytePython {
+airbytePythonDocker {
     moduleDirectory 'connector_builder'
 }
 
-task generateOpenApiPythonServer(type: GenerateTask){
+task generateOpenApiPythonServer(type: GenerateTask) {
     outputs.upToDateWhen { false }
 
     def generatedCodeDir = "$buildDir/airbyte_connector_builder_server"
@@ -42,4 +42,20 @@ task generateOpenApiPythonServer(type: GenerateTask){
 }
 
 project.build.dependsOn(generateOpenApiPythonServer)
-
+
+// java modules such as airbyte-server can use copyGeneratedTar to copy the files to the docker image
+// We cannot do this here because we don't generate a tar file
+// Instead, we copy the files into the build directory so they can be copied to the docker container
+task prepareBuild(type: Copy) {
+    from layout.projectDirectory.file(".")
+    exclude '.*'
+    exclude 'build'
+
+
+    into layout.buildDirectory.dir("docker")
+}
+
+tasks.named("buildDockerImage") {
+    dependsOn prepareBuild
+    dependsOn copyDocker
+}

airbyte-connector-builder-server/connector_builder/entrypoint.py

@@ -2,14 +2,24 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
 from connector_builder.generated.apis.default_api_interface import initialize_router
 from connector_builder.impl.default_api import DefaultApiImpl
-from fastapi import FastAPI
 
 app = FastAPI(
     title="Connector Builder Server API",
     description="Connector Builder Server API ",
     version="1.0.0",
 )
 
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
 app.include_router(initialize_router(DefaultApiImpl()))

airbyte-connector-builder-server/connector_builder/generated/apis/default_api_interface.py

@@ -1,6 +1,8 @@
 #
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
+# This file was auto-generated from Airbyte's custom OpenAPI templates. Do not edit it manually.
+# coding: utf-8
 
 import inspect
 from abc import ABC, abstractmethod

airbyte-connector-builder-server/connector_builder/generated/models/extra_models.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from pydantic import BaseModel

airbyte-connector-builder-server/connector_builder/generated/models/http_request.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/http_response.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/invalid_input_exception_info.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/invalid_input_property.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/known_exception_info.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/stream_read.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/stream_read_pages.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/stream_read_request_body.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/stream_read_slice_descriptor.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/stream_read_slices.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/streams_list_read.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/streams_list_read_streams.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/generated/models/streams_list_request_body.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
-#
-
 # coding: utf-8
 
 from __future__ import annotations

airbyte-connector-builder-server/connector_builder/impl/default_api.py

@@ -9,6 +9,9 @@
 from urllib.parse import parse_qs, urljoin, urlparse
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Type
+from fastapi import Body, HTTPException
+from jsonschema import ValidationError
+
 from connector_builder.generated.apis.default_api_interface import DefaultApi
 from connector_builder.generated.models.http_request import HttpRequest
 from connector_builder.generated.models.http_response import HttpResponse
@@ -20,15 +23,13 @@
 from connector_builder.generated.models.streams_list_read_streams import StreamsListReadStreams
 from connector_builder.generated.models.streams_list_request_body import StreamsListRequestBody
 from connector_builder.impl.low_code_cdk_adapter import LowCodeSourceAdapter
-from fastapi import Body, HTTPException
 
 
 class DefaultApiImpl(DefaultApi):
     logger = logging.getLogger("airbyte.connector-builder")
 
     async def get_manifest_template(self) -> str:
         return """version: "0.1.0"
-
 definitions:
   selector:
     extractor:
@@ -112,7 +113,7 @@ async def read_stream(self, stream_read_request_body: StreamReadRequestBody = Bo
         log_messages = []
         try:
             for message_group in self._get_message_groups(
-                adapter.read_stream(stream_read_request_body.stream, stream_read_request_body.config)
+                    adapter.read_stream(stream_read_request_body.stream, stream_read_request_body.config)
             ):
                 if isinstance(message_group, AirbyteLogMessage):
                     log_messages.append({"message": message_group.message})

airbyte-connector-builder-server/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=connector-builder-server

airbyte-connector-builder-server/integration_tests/test_integration_test.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+def test_test():
+    assert True

airbyte-connector-builder-server/run_format.sh

@@ -0,0 +1,21 @@
+cd $1
+
+# Pasted from https://github.com/airbytehq/airbyte/blob/master/buildSrc/src/main/groovy/airbyte-python.gradle#L85-L96
+pip install 'mccabe==0.6.1'
+pip install 'flake8==4.0.1'
+pip install 'pyproject-flake8==0.0.1a2'
+pip install 'black==22.3.0'
+pip install 'mypy==0.930'
+pip install 'isort==5.6.4'
+pip install 'pytest==6.1.2'
+pip install 'coverage[toml]==6.3.1'
+
+# Format and static analysis
+# FIXME: isort formats python files differently from gradlew format
+python -m isort --settings-file=pyproject.toml ./
+python -m isort --settings-file=pyproject.toml --diff --quiet ./
+python -m black --config pyproject.toml ./
+python -m black --config pyproject.toml ./ --diff --quiet
+python -m pflake8 --config pyproject.toml ./
+python -m pflake8 --config pyproject.toml ./ --diff --quiet
+python -m mypy --config pyproject.toml ./

airbyte-connector-builder-server/run_tests.sh

@@ -0,0 +1,10 @@
+cd $1
+
+# Install dependencies
+pip install -e .
+pip install -e '.[main]'
+pip install -e '.[tests]'
+
+# Run the tests
+python -m coverage run -m pytest unit_tests -c pytest.ini
+python -m coverage run -m pytest integration_tests -c pytest.ini

airbyte-connector-builder-server/setup.py

@@ -44,6 +44,14 @@
     install_requires=["airbyte-cdk~=0.8", "fastapi", "uvicorn"],
     python_requires=">=3.9.11",
     extras_require={
-        "tests": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "pytest-recording", "requests-mock", "pre-commit"],
+        "tests": [
+            "MyPy~=0.812",
+            "pytest~=6.2.5",
+            "pytest-cov",
+            "pytest-mock",
+            "pytest-recording",
+            "requests-mock",
+            "pre-commit",
+        ],
     },
 )

airbyte-connector-builder-server/unit_tests/connector_builder/impl/test_default_api.py

@@ -8,6 +8,8 @@
 
 import pytest
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, Level, Type
+from fastapi import HTTPException
+
 from connector_builder.generated.models.http_request import HttpRequest
 from connector_builder.generated.models.http_response import HttpResponse
 from connector_builder.generated.models.stream_read import StreamRead
@@ -17,7 +19,6 @@
 from connector_builder.generated.models.streams_list_read_streams import StreamsListReadStreams
 from connector_builder.generated.models.streams_list_request_body import StreamsListRequestBody
 from connector_builder.impl.default_api import DefaultApiImpl
-from fastapi import HTTPException
 
 MANIFEST = {
     "version": "0.1.0",

airbyte-connector-builder-server/unit_tests/connector_builder/impl/test_low_code_cdk_adapter.py

@@ -12,6 +12,7 @@
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.parsers.undefined_reference_exception import UndefinedReferenceException
 from airbyte_cdk.sources.streams.http import HttpStream
+
 from connector_builder.impl.low_code_cdk_adapter import LowCodeSourceAdapter
 
 

airbyte-connector-builder-server/unit_tests/test_unit_test.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+def test_test():
+    assert True

airbyte-webapp/.env

@@ -3,4 +3,4 @@ REACT_APP_FULL_STORY_ORG=13AXQ4
 REACT_APP_SENTRY_DSN=
 REACT_APP_INTERCOM_APP_ID=nj1oam7s
 REACT_APP_OSANO=16A0CTTE7vE8m1Qif/67beec9b-e563-4736-bdb4-4fe4adc39d48
-#REACT_APP_CONNECTOR_BUILDER_API=/connector-builder-api/ #FIXME: Uncomment this when enabling the connector builder
+REACT_APP_CONNECTOR_BUILDER_API=/connector-builder-api/

airbyte-webapp/nginx/default.conf.template

@@ -2,9 +2,9 @@ upstream api-server {
     server $INTERNAL_API_HOST;
 }
 
-# upstream connector-builder-server { #FIXME: Uncomment this block when enabling the connector-builder
-#    server $CONNECTOR_BUILDER_API_HOST;
-# }
+upstream connector-builder-server {
+   server $CONNECTOR_BUILDER_API_HOST;
+}
 
 server {
     listen       80;
@@ -45,10 +45,10 @@ server {
         proxy_pass http://api-server/api/;
     }
 
-    #location /connector-builder-api/ { #FIXME: Uncomment this block when enabling the connector-builder
-    #    fastcgi_read_timeout 1h;
-    #    proxy_read_timeout 1h;
-    #    client_max_body_size 200M;
-    #    proxy_pass http://connector-builder-server/;
-    #}
+    location /connector-builder-api/ {
+        fastcgi_read_timeout 1h;
+        proxy_read_timeout 1h;
+        client_max_body_size 200M;
+        proxy_pass http://connector-builder-server/;
+    }
 }

airbyte-webapp/src/components/StreamTestingPanel/LogsDisplay.module.scss

@@ -6,6 +6,7 @@
   display: flex;
   flex-direction: column;
   height: 100%;
+  overflow-y: hidden;
 }
 
 .header {

airbyte-webapp/src/components/StreamTestingPanel/StreamTester.module.scss

@@ -7,6 +7,7 @@
   flex-direction: column;
   align-items: center;
   gap: variables.$spacing-lg;
+  min-height: 0;
 }
 
 .resizablePanelsContainer {

airbyte-webapp/src/config/configProviders.ts

@@ -22,7 +22,7 @@ const windowConfigProvider: ConfigProvider = async () => {
 const envConfigProvider: ConfigProvider = async () => {
   return {
     apiUrl: process.env.REACT_APP_API_URL,
-    // connectorBuilderUrl: process.env.REACT_APP_CONNECTOR_BUILDER_API, #FIXME: Uncomment this when enabling the connector-builder
+    connectorBuilderUrl: process.env.REACT_APP_CONNECTOR_BUILDER_API,
     integrationUrl: process.env.REACT_APP_INTEGRATION_DOCS_URLS,
     segment: {
       token: process.env.REACT_APP_SEGMENT_TOKEN,