Merge branch 'master' into gosusnp/19853-out-of-date-definitions-endp…

…oint

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
           ).?$
 
   - repo: https://github.com/csachs/pyproject-flake8
-    rev: v0.0.1a2.post1
+    rev: v6.0.0
     hooks:
       - id: pyproject-flake8
         args: ["--config", "pyproject.toml"]

airbyte-api/src/main/openapi/config.yaml

@@ -277,26 +277,7 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
-  /v1/source_definitions/create:
-    post:
-      tags:
-        - source_definition
-      summary: Creates a sourceDefinition
-      operationId: createSourceDefinition
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/SourceDefinitionCreate"
-      responses:
-        "200":
-          description: Successful operation
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/SourceDefinitionRead"
-        "422":
-          $ref: "#/components/responses/InvalidInputResponse"
+
   /v1/source_definitions/update:
     post:
       tags:
@@ -469,47 +450,6 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
-  /v1/source_definitions/update_custom:
-    post:
-      tags:
-        - source_definition
-      summary: Update a custom sourceDefinition for the given workspace
-      operationId: updateCustomSourceDefinition
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/CustomSourceDefinitionUpdate"
-      responses:
-        "200":
-          description: Successful operation
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/SourceDefinitionRead"
-        "404":
-          $ref: "#/components/responses/NotFoundResponse"
-        "422":
-          $ref: "#/components/responses/InvalidInputResponse"
-  /v1/source_definitions/delete_custom:
-    post:
-      tags:
-        - source_definition
-      summary: Delete a custom source definition for the given workspace
-      operationId: deleteCustomSourceDefinition
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/SourceDefinitionIdWithWorkspaceId"
-        required: true
-      responses:
-        "204":
-          description: The resource was deleted successfully.
-        "404":
-          $ref: "#/components/responses/NotFoundResponse"
-        "422":
-          $ref: "#/components/responses/InvalidInputResponse"
   /v1/source_definitions/grant_definition:
     post:
       tags:
@@ -798,26 +738,6 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
-  /v1/destination_definitions/create:
-    post:
-      tags:
-        - destination_definition
-      summary: Creates a destinationsDefinition
-      operationId: createDestinationDefinition
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/DestinationDefinitionCreate"
-      responses:
-        "200":
-          description: Successful operation
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/DestinationDefinitionRead"
-        "422":
-          $ref: "#/components/responses/InvalidInputResponse"
   /v1/destination_definitions/update:
     post:
       tags:
@@ -991,47 +911,6 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
-  /v1/destination_definitions/update_custom:
-    post:
-      tags:
-        - destination_definition
-      summary: Update a custom destinationDefinition for the given workspace
-      operationId: updateCustomDestinationDefinition
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/CustomDestinationDefinitionUpdate"
-      responses:
-        "200":
-          description: Successful operation
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/DestinationDefinitionRead"
-        "404":
-          $ref: "#/components/responses/NotFoundResponse"
-        "422":
-          $ref: "#/components/responses/InvalidInputResponse"
-  /v1/destination_definitions/delete_custom:
-    post:
-      tags:
-        - destination_definition
-      summary: Delete a custom destination definition for the given workspace
-      operationId: deleteCustomDestinationDefinition
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/DestinationDefinitionIdWithWorkspaceId"
-        required: true
-      responses:
-        "204":
-          description: The destination was deleted successfully.
-        "404":
-          $ref: "#/components/responses/NotFoundResponse"
-        "422":
-          $ref: "#/components/responses/InvalidInputResponse"
   /v1/destination_definitions/grant_definition:
     post:
       tags:
@@ -2693,16 +2572,6 @@ components:
           $ref: "#/components/schemas/WorkspaceId"
         sourceDefinition:
           $ref: "#/components/schemas/SourceDefinitionCreate"
-    CustomSourceDefinitionUpdate:
-      type: object
-      required:
-        - workspaceId
-        - sourceDefinition
-      properties:
-        workspaceId:
-          $ref: "#/components/schemas/WorkspaceId"
-        sourceDefinition:
-          $ref: "#/components/schemas/SourceDefinitionUpdate"
     SourceDefinitionIdWithWorkspaceId:
       type: object
       required:
@@ -3063,16 +2932,6 @@ components:
           $ref: "#/components/schemas/WorkspaceId"
         destinationDefinition:
           $ref: "#/components/schemas/DestinationDefinitionCreate"
-    CustomDestinationDefinitionUpdate:
-      type: object
-      required:
-        - workspaceId
-        - destinationDefinition
-      properties:
-        workspaceId:
-          $ref: "#/components/schemas/WorkspaceId"
-        destinationDefinition:
-          $ref: "#/components/schemas/DestinationDefinitionUpdate"
     DestinationDefinitionIdWithWorkspaceId:
       type: object
       required:
@@ -4192,6 +4051,7 @@ components:
         - normalization
         - dbt
         - airbyte_platform
+        - unknown
     AttemptFailureType:
       description: Categorizes well known errors into types for programmatic handling. If not set, the type of error is not well known.
       type: string

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.11.2
+Low-code: Log response error message on failure
+
 ## 0.11.2
 Low-code: Include the HTTP method used by the request in logging output of the `airbyte-cdk`
 

airbyte-cdk/python/README.md

@@ -70,6 +70,61 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies
 
 All tests are located in the `unit_tests` directory. Run `pytest --cov=airbyte_cdk unit_tests/` to run them. This also presents a test coverage report.
 
+#### Building a connector with your local CDK
+
+When developing a new feature in the CDK, you may find it helpful to run a connector that uses that new feature. You can test this in one of two ways:
+* Running a connector locally
+* Building and running a source via Docker
+
+##### Installing your local CDK into a local Python connector
+
+In order to get a local Python connector running your local CDK, do the following.
+
+First, make sure you have your connector's virtual environment active:
+```bash
+# from the `airbyte/airbyte-integrations/connectors/<connector-directory>` directory
+source .venv/bin/activate
+
+# if you haven't installed dependencies for your connector already
+pip install -e .
+```
+
+Then, navigate to the CDK and install it in editable mode:
+```bash
+cd ../../../airbyte-cdk/python
+pip install -e .
+```
+
+You should see that `pip` has uninstalled the version of `airbyte-cdk` defined by your connector's `setup.py` and installed your local CDK. Any changes you make will be immediately reflected in your editor, so long as your editor's interpreter is set to your connector's virtual environment.
+
+##### Building a Python connector in Docker with your local CDK installed
+
+Create a symlink that connects `<connector-directory>/airbyte-cdk` to your local CDK installation:
+```bash
+# from the `airbyte/airbyte-integrations/connectors/<connector-directory>` directory
+ln -s ../../../airbyte-cdk/python airbyte-cdk
+```
+
+Add the following lines to your connector's `Dockerfile`, before the line that installs dependencies via `pip install -e .`:
+```Dockerfile
+COPY airbyte-cdk airbyte-cdk
+RUN pip install -e ./airbyte-cdk
+```
+
+Add the following to your connectors `build.gradle` file:
+```java
+airbyteDocker {
+    followSymlinks = true
+}
+```
+
+You should be able to build your connector with
+```bash
+# from the airbytehq/airbyte base directory
+./gradlew build :airbyte-integrations:connectors:<connector-directory>
+```
+and the installation should use your local CDK. Note that the local CDK is injected at build time, so if you make changes, you will have to run the build command again to see them reflected.
+**Note:** if your connector uses a `.dockerignore` file, it cannot have `exclude-all` or `exclude-except` patterns, i.e. the `.dockerignore` must specifically say which files to ignore without using any regex. 
 #### Publishing a new version to PyPi
 
 1. Bump the package version in `setup.py`

airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -319,7 +319,10 @@ def parse_response(
         # else -> delegate to record selector
         response_status = self.requester.interpret_response_status(response)
         if response_status.action == ResponseAction.FAIL:
-            error_message = response_status.error_message or f"Request {response.request} failed with response {response}"
+            error_message = (
+                response_status.error_message
+                or f"Request to {response.request.url} failed with status code {response.status_code} and error message {HttpStream.parse_response_error_message(response)}"
+            )
             raise ReadException(error_message)
         elif response_status.action == ResponseAction.IGNORE:
             self.logger.info(f"Ignoring response for failed request with error message {HttpStream.parse_response_error_message(response)}")

airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py

@@ -377,6 +377,7 @@ def _try_get_error(value):
                     or value.get("errors")
                     or value.get("failures")
                     or value.get("failure")
+                    or value.get("detail")
                 )
                 return _try_get_error(new_value)
             return None

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.11.2",
+    version="0.11.3",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py

@@ -221,14 +221,15 @@ def test_parse_response(test_name, status_code, response_status, len_expected_re
         name="stream_name", primary_key=primary_key, requester=requester, record_selector=record_selector, options={}, config={}
     )
     response = requests.Response()
+    response.request = requests.Request()
     response.status_code = status_code
     requester.interpret_response_status.return_value = response_status
     if len_expected_records is None:
         try:
             retriever.parse_response(response, stream_state={})
             assert False
         except ReadException as actual_exception:
-            assert type(expected_error) is type(actual_exception) and expected_error.args == actual_exception.args
+            assert type(expected_error) is type(actual_exception)
     else:
         records = retriever.parse_response(response, stream_state={})
         assert len(records) == len_expected_records

airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java

@@ -38,7 +38,7 @@
 import io.airbyte.workers.internal.AirbyteDestination;
 import io.airbyte.workers.internal.AirbyteMapper;
 import io.airbyte.workers.internal.AirbyteSource;
-import io.airbyte.workers.internal.MessageTracker;
+import io.airbyte.workers.internal.book_keeping.MessageTracker;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;