Revive REST API Source (#1103)

airbytehq · Nov 28, 2020 · 83fc3e5 · 83fc3e5
1 parent b98aeb7
commit 83fc3e5
Show file tree

Hide file tree

Showing 23 changed files with 107 additions and 72 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json
@@ -0,0 +1,7 @@
+{
+  "sourceDefinitionId": "9845d17a-45f1-4070-8a60-50914b1c8e2b",
+  "name": "HTTP Request",
+  "dockerRepository": "airbyte/source-http-request",
+  "dockerImageTag": "0.1.0",
+  "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-http-request"
+}
diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteProtocolConverters.java b/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteProtocolConverters.java
@@ -32,6 +32,7 @@
 import io.airbyte.protocol.models.AirbyteStream;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 import io.airbyte.protocol.models.ConfiguredAirbyteStream;
+import io.airbyte.protocol.models.SyncMode;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -45,6 +46,7 @@ public class AirbyteProtocolConverters {
 
   public static ConfiguredAirbyteCatalog toConfiguredCatalog(Schema schema) {
     final List<ConfiguredAirbyteStream> airbyteStreams = schema.getStreams().stream()
+        .filter(s -> s.getSelected() != null && s.getSelected())
         .map(s -> new ConfiguredAirbyteStream()
             // immutable
             // todo (cgardens) - not great that we just trust the API to not mutate these.
@@ -53,16 +55,13 @@ public static ConfiguredAirbyteCatalog toConfiguredCatalog(Schema schema) {
                 .withJsonSchema(toJson(s.getFields()))
                 .withSupportedSyncModes(s.getSupportedSyncModes()
                     .stream()
-                    .map(e -> Enums.convertTo(e, io.airbyte.protocol.models.SyncMode.class))
+                    .map(e -> Enums.convertTo(e, SyncMode.class))
                     .collect(Collectors.toList()))
                 .withSourceDefinedCursor(s.getSourceDefinedCursor())
                 .withDefaultCursorField(s.getDefaultCursorField()))
             // configurable
-            .withSyncMode(Enums.convertTo(s.getSyncMode(), io.airbyte.protocol.models.SyncMode.class))
+            .withSyncMode(Enums.convertTo(s.getSyncMode(), SyncMode.class))
             .withCursorField(s.getCursorField()))
-
-        // perform selection based on the output of toJson, which keeps properties if selected=true
-        .filter(s -> !s.getStream().getJsonSchema().get("properties").isEmpty())
         .collect(Collectors.toList());
     return new ConfiguredAirbyteCatalog().withStreams(airbyteStreams);
   }
@@ -81,7 +80,6 @@ public static JsonNode toJson(List<Field> fields) {
         .put("type", "object")
         .put("properties", fields
             .stream()
-            .filter(Field::getSelected)
             .collect(Collectors.toMap(
                 Field::getName,
                 field -> ImmutableMap.of("type", field.getDataType().toString().toLowerCase()))))
@@ -100,7 +98,8 @@ public static Schema toSchema(AirbyteCatalog catalog) {
                 .map(e -> Enums.convertTo(e, StandardSync.SyncMode.class))
                 .collect(Collectors.toList()))
             .withSourceDefinedCursor(airbyteStream.getSourceDefinedCursor())
-            .withDefaultCursorField(airbyteStream.getDefaultCursorField()))
+            .withDefaultCursorField(airbyteStream.getDefaultCursorField())
+            .withSelected(true)) // by default all discovered streams are treated as default.
         // configurable fields syncMode and cursorField are not set since they will never be defined in an
         // AirbyteCatalog.
         .collect(Collectors.toList()));
@@ -115,10 +114,13 @@ private static List<Field> toFields(JsonNode jsonSchemaPropertiesObject) {
       list.add(iterator.next());
     }
 
-    return list.stream().map(item -> new Field()
-        .withName(item.getKey())
-        .withDataType(getDataType(item.getValue()))
-        .withSelected(true)).collect(Collectors.toList());
+    return list
+        .stream()
+        .map(item -> new Field()
+            .withName(item.getKey())
+            .withDataType(getDataType(item.getValue()))
+            .withSelected(true))
+        .collect(Collectors.toList());
   }
 
   // todo (cgardens) - add more robust handling for jsonschema types.

diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/AirbyteProtocolConvertersTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/AirbyteProtocolConvertersTest.java
@@ -27,6 +27,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.json.Jsons;
@@ -38,6 +39,7 @@
 import io.airbyte.protocol.models.Field;
 import io.airbyte.protocol.models.Field.JsonSchemaPrimitive;
 import io.airbyte.protocol.models.SyncMode;
+import java.util.Collections;
 import org.junit.jupiter.api.Test;
 
 class AirbyteProtocolConvertersTest {
@@ -66,32 +68,30 @@ class AirbyteProtocolConvertersTest {
 
   private static final Schema SCHEMA = new Schema()
       .withStreams(Lists.newArrayList(new Stream()
+          .withSelected(true)
           .withName(STREAM)
           .withFields(Lists.newArrayList(
               new io.airbyte.config.Field()
                   .withName(COLUMN_NAME)
-                  .withDataType(DataType.STRING)
-                  .withSelected(true),
+                  .withDataType(DataType.STRING),
               new io.airbyte.config.Field()
                   .withName(COLUMN_AGE)
-                  .withDataType(DataType.NUMBER)
-                  .withSelected(true)))
+                  .withDataType(DataType.NUMBER)))
           .withSourceDefinedCursor(false)
           .withSupportedSyncModes(Lists.newArrayList(StandardSync.SyncMode.FULL_REFRESH, StandardSync.SyncMode.INCREMENTAL))
           .withDefaultCursorField(Lists.newArrayList(COLUMN_AGE))));
 
   private static final Schema SCHEMA_WITH_UNSELECTED = new Schema()
       .withStreams(Lists.newArrayList(new Stream()
+          .withSelected(true)
           .withName(STREAM)
           .withFields(Lists.newArrayList(
               new io.airbyte.config.Field()
                   .withName(COLUMN_NAME)
-                  .withDataType(DataType.STRING)
-                  .withSelected(true),
+                  .withDataType(DataType.STRING),
               new io.airbyte.config.Field()
                   .withName(COLUMN_AGE)
-                  .withDataType(DataType.NUMBER)
-                  .withSelected(true)))
+                  .withDataType(DataType.NUMBER)))
           .withSourceDefinedCursor(false)
           .withSupportedSyncModes(Lists.newArrayList(StandardSync.SyncMode.FULL_REFRESH, StandardSync.SyncMode.INCREMENTAL))
           .withDefaultCursorField(Lists.newArrayList(COLUMN_AGE)),
@@ -100,12 +100,10 @@ class AirbyteProtocolConvertersTest {
               .withFields(Lists.newArrayList(
                   new io.airbyte.config.Field()
                       .withName(COLUMN_NAME)
-                      .withDataType(DataType.STRING)
-                      .withSelected(false),
+                      .withDataType(DataType.STRING),
                   new io.airbyte.config.Field()
                       .withName(COLUMN_AGE)
-                      .withDataType(DataType.NUMBER)
-                      .withSelected(false)))));
+                      .withDataType(DataType.NUMBER)))));
 
   @Test
   void testToConfiguredCatalog() {
@@ -127,7 +125,10 @@ void testToConfiguredCatalogWithUnselectedStream() {
 
   @Test
   void testToSchema() {
-    assertEquals(SCHEMA, AirbyteProtocolConverters.toSchema(CATALOG));
+    final Schema schema = Jsons.clone(SCHEMA);
+    schema.getStreams().forEach(table -> table.getFields().forEach(c -> c.setSelected(true))); // select all fields
+
+    assertEquals(schema, AirbyteProtocolConverters.toSchema(CATALOG));
   }
 
   @Test
@@ -145,7 +146,8 @@ void testToSchemaWithMultipleJsonSchemaTypesAndFormats() {
                 new io.airbyte.config.Field()
                     .withName(COLUMN_AGE)
                     .withDataType(DataType.NUMBER)
-                    .withSelected(true)))));
+                    .withSelected(true)))
+            .withSelected(true)));
 
     assertEquals(schema, AirbyteProtocolConverters.toSchema(catalog));
   }
@@ -162,12 +164,27 @@ void testAnyOfAsObject() {
                 new io.airbyte.config.Field()
                     .withName("date")
                     .withDataType(DataType.OBJECT)
-                    .withSelected(true)))));
+                    .withSelected(true)))
+            .withSelected(true)));
 
     final AirbyteCatalog catalog = Jsons.deserialize(testString, AirbyteCatalog.class);
     assertEquals(schema, AirbyteProtocolConverters.toSchema(catalog));
   }
 
+  @Test
+  void testStreamWithNoFields() {
+    final Schema schema = Jsons.clone(SCHEMA);
+    schema.getStreams().get(0).withCursorField(Lists.newArrayList(COLUMN_NAME));
+    schema.getStreams().get(0).withSyncMode(StandardSync.SyncMode.INCREMENTAL);
+    schema.getStreams().get(0).setFields(Lists.newArrayList());
+    final ConfiguredAirbyteCatalog actualCatalog = AirbyteProtocolConverters.toConfiguredCatalog(schema);
+
+    final ConfiguredAirbyteCatalog expectedCatalog = Jsons.clone(CONFIGURED_CATALOG);
+    ((ObjectNode) expectedCatalog.getStreams().get(0).getStream().getJsonSchema()).set("properties", Jsons.jsonNode(Collections.emptyMap()));
+
+    assertEquals(expectedCatalog, actualCatalog);
+  }
+
   @Test
   void testEnumConversion() {
     assertTrue(Enums.isCompatible(io.airbyte.protocol.models.SyncMode.class, io.airbyte.config.StandardSync.SyncMode.class));

diff --git a/...ons/connectors/source-rest-api/Dockerfile → ...connectors/source-http-request/Dockerfile b/...ons/connectors/source-rest-api/Dockerfile → ...connectors/source-http-request/Dockerfile
@@ -2,14 +2,14 @@ FROM airbyte/integration-base-python:dev
 
 RUN apt-get update && apt-get install -y jq curl bash && rm -rf /var/lib/apt/lists/*
 
-ENV CODE_PATH="source_rest_api"
-ENV AIRBYTE_IMPL_MODULE="source_rest_api"
-ENV AIRBYTE_IMPL_PATH="SourceRestApi"
+ENV CODE_PATH="source_http_request"
+ENV AIRBYTE_IMPL_MODULE="source_http_request"
+ENV AIRBYTE_IMPL_PATH="SourceHttpRequest"
 
 WORKDIR /airbyte/integration_code
 COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
 RUN pip install ".[main]"
 
 LABEL io.airbyte.version=0.1.0
-LABEL io.airbyte.name=airbyte/source-rest-api
+LABEL io.airbyte.name=airbyte/source-http-request
diff --git a/...onnectors/source-rest-api/Dockerfile.test → ...ctors/source-http-request/Dockerfile.test b/...onnectors/source-rest-api/Dockerfile.test → ...ctors/source-http-request/Dockerfile.test
@@ -4,16 +4,16 @@ RUN apt-get update && rm -rf /var/lib/apt/lists/*
 
 ENV CODE_PATH="integration_tests"
 ENV AIRBYTE_TEST_MODULE="integration_tests"
-ENV AIRBYTE_TEST_PATH="SourceRestApiStandardTest"
+ENV AIRBYTE_TEST_PATH="SourceHttpRequestStandardTest"
 ENV AIRBYTE_TEST_CASE=true
 
 LABEL io.airbyte.version=0.1.0
-LABEL io.airbyte.name=airbyte/source-rest-api-standard-test
+LABEL io.airbyte.name=airbyte/source-http-request-standard-test
 
 WORKDIR /airbyte/integration_code
-COPY source_rest_api source_rest_api
+COPY source_http_request source_http_request
 COPY $CODE_PATH $CODE_PATH
-COPY source_rest_api/*.json $CODE_PATH
+COPY source_http_request/*.json $CODE_PATH
 COPY setup.py ./
 
 RUN pip install ".[tests]"

diff --git a/...ions/connectors/source-rest-api/README.md → .../connectors/source-http-request/README.md b/...ions/connectors/source-rest-api/README.md → .../connectors/source-http-request/README.md
@@ -1,19 +1,19 @@
-# Rest Api Source 
+# HTTP Request Source 
 
-This is the repository for the Rest Api source connector, written in Python. 
+This is the repository for the HTTP Request source connector, written in Python. 
 For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rest-api).
 
 ## Local development
 ### Build
 First, build the module by running the following from the `airbyte` project root directory: 
 ```
-./gradlew :airbyte-integrations:connectors:source-rest-api:build
+./gradlew :airbyte-integrations:connectors:source-http-request:build
 ```
 
-This should generate a virtualenv for this module in `source-rest-api/.venv`. Make sure this venv is active in your
-development environment of choice. If you are on the terminal, run the following from the `source-rest-api` directory:
+This should generate a virtualenv for this module in `source-http-request/.venv`. Make sure this venv is active in your
+development environment of choice. If you are on the terminal, run the following from the `source-http-request` directory:
 ```
-cd airbyte-integrations/connectors/source-rest-api # cd into the connector directory
+cd airbyte-integrations/connectors/source-http-request # cd into the connector directory
 source .venv/bin/activate
 ```
 If you are in an IDE, follow your IDE's instructions to activate the virtualenv. 
@@ -37,16 +37,16 @@ pytest unit_tests
 ### Locally running the connector docker image
 ```
 # in airbyte root directory
-./gradlew :airbyte-integrations:connectors:source-rest-api:airbyteDocker
-docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev spec
-docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev check --config /sample_files/sample_files/config.json
-docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev discover --config /sample_files/sample_files/config.json
-docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev read --config /sample_files/sample_files/config.json --catalog /sample_files/integration_tests/catalog.json
+./gradlew :airbyte-integrations:connectors:source-http-request:airbyteDocker
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev spec
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev check --config /sample_files/sample_files/config.json
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev discover --config /sample_files/sample_files/config.json
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev read --config /sample_files/sample_files/config.json --catalog /sample_files/integration_tests/catalog.json
 ```
 
 ### Integration Tests 
 1. Configure credentials as appropriate, described below.
-1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-rest-api:standardSourceTestPython` to run the standard integration test suite.
+1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-http-request:standardSourceTestPython` to run the standard integration test suite.
 1. To run additional integration tests, place your integration tests in the `integration_tests` directory and run them with `pytest integration_tests`.
    Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 

diff --git a/...nnectors/source-rest-api/airbyte_protocol → ...tors/source-http-request/airbyte_protocol b/...nnectors/source-rest-api/airbyte_protocol → ...tors/source-http-request/airbyte_protocol
diff --git a/...ns/connectors/source-rest-api/base_python → ...onnectors/source-http-request/base_python b/...ns/connectors/source-rest-api/base_python → ...onnectors/source-http-request/base_python
diff --git a/...s/connectors/source-rest-api/build.gradle → ...nnectors/source-http-request/build.gradle b/...s/connectors/source-rest-api/build.gradle → ...nnectors/source-http-request/build.gradle
@@ -2,12 +2,11 @@ plugins {
     id 'java'
     id 'airbyte-python'
     id 'airbyte-docker'
-    // todo (cgardens) - bring back when we re-add this to the product.
-    // id 'airbyte-source-test'
+    id 'airbyte-source-test'
 }
 
 airbytePython {
-    moduleDirectory 'source_rest_api'
+    moduleDirectory 'source_http_request'
 }
 
 dependencies {

diff --git a/...ce-rest-api/integration_tests/__init__.py → ...ttp-request/integration_tests/__init__.py b/...ce-rest-api/integration_tests/__init__.py → ...ttp-request/integration_tests/__init__.py
@@ -22,6 +22,6 @@
 SOFTWARE.
 """
 
-from .standard_source_test import SourceRestApiStandardTest
+from .standard_source_test import SourceHttpRequestStandardTest
 
-__all__ = ["SourceRestApiStandardTest"]
+__all__ = ["SourceHttpRequestStandardTest"]
diff --git a/...e-rest-api/integration_tests/catalog.json → ...tp-request/integration_tests/catalog.json b/...e-rest-api/integration_tests/catalog.json → ...tp-request/integration_tests/catalog.json
diff --git a/...integration_tests/standard_source_test.py → ...integration_tests/standard_source_test.py b/...integration_tests/standard_source_test.py → ...integration_tests/standard_source_test.py
@@ -29,7 +29,7 @@
 from base_python_test import StandardSourceTestIface
 
 
-class SourceRestApiStandardTest(StandardSourceTestIface):
+class SourceHttpRequestStandardTest(StandardSourceTestIface):
     def __init__(self):
         pass
 

diff --git a/...ns/connectors/source-rest-api/main_dev.py → ...onnectors/source-http-request/main_dev.py b/...ns/connectors/source-rest-api/main_dev.py → ...onnectors/source-http-request/main_dev.py
@@ -25,8 +25,8 @@
 import sys
 
 from base_python.entrypoint import launch
-from source_rest_api import SourceRestApi
+from source_http_request import SourceHttpRequest
 
 if __name__ == "__main__":
-    source = SourceRestApi()
+    source = SourceHttpRequest()
     launch(source, sys.argv[1:])
diff --git a/...nnectors/source-rest-api/requirements.txt → ...tors/source-http-request/requirements.txt b/...nnectors/source-rest-api/requirements.txt → ...tors/source-http-request/requirements.txt
diff --git a/.../source-rest-api/sample_files/config.json → ...rce-http-request/sample_files/config.json b/.../source-rest-api/sample_files/config.json → ...rce-http-request/sample_files/config.json
diff --git a/...tions/connectors/source-rest-api/setup.py → ...s/connectors/source-http-request/setup.py b/...tions/connectors/source-rest-api/setup.py → ...s/connectors/source-http-request/setup.py
@@ -25,8 +25,8 @@
 from setuptools import find_packages, setup
 
 setup(
-    name="source_rest_api",
-    description="Source implementation for Rest Api.",
+    name="source_http_request",
+    description="Source implementation for HTTP Request.",
     author="Airbyte",
     author_email="contact@airbyte.io",
     packages=find_packages(),

diff --git a/...urce-rest-api/source_rest_api/__init__.py → ...p-request/source_http_request/__init__.py b/...urce-rest-api/source_rest_api/__init__.py → ...p-request/source_http_request/__init__.py
@@ -22,6 +22,6 @@
 SOFTWARE.
 """
 
-from .source import SourceRestApi
+from .source import SourceHttpRequest
 
-__all__ = ["SourceRestApi"]
+__all__ = ["SourceHttpRequest"]
diff --git a/...source-rest-api/source_rest_api/source.py → ...ttp-request/source_http_request/source.py b/...source-rest-api/source_rest_api/source.py → ...ttp-request/source_http_request/source.py
@@ -31,7 +31,7 @@
 from base_python import AirbyteLogger, Source
 
 
-class SourceRestApi(Source):
+class SourceHttpRequest(Source):
     STREAM_NAME = "data"
 
     def __init__(self):
@@ -49,11 +49,13 @@ def discover(self, logger: AirbyteLogger, config_container) -> AirbyteCatalog:
             "$schema": "http://json-schema.org/draft-07/schema#",
             "additionalProperties": True,
             "type": "object",
-            "properties": {},
+            # todo (cgardens) - remove data column. added to handle UI bug where streams without fields cannot be selected.
+            # issue: https://github.com/airbytehq/airbyte/issues/1104
+            "properties": {"data": {"type": "object"}},
         }
 
         # json body will be returned as the "data" stream". we can't know its schema ahead of time, so we assume it's object (i.e. valid json).
-        return AirbyteCatalog(streams=[AirbyteStream(name=SourceRestApi.STREAM_NAME, json_schema=json_schema)])
+        return AirbyteCatalog(streams=[AirbyteStream(name=SourceHttpRequest.STREAM_NAME, json_schema=json_schema)])
 
     def read(self, logger: AirbyteLogger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
         r = self._make_request(config_container.rendered_config)
@@ -63,8 +65,11 @@ def read(self, logger: AirbyteLogger, config_container, catalog_path, state=None
         # need to eagerly fetch the json.
         message = AirbyteMessage(
             type=Type.RECORD,
-            record=AirbyteRecordMessage(stream=SourceRestApi.STREAM_NAME, data=r.json(), emitted_at=int(datetime.now().timestamp()) * 1000),
+            record=AirbyteRecordMessage(
+                stream=SourceHttpRequest.STREAM_NAME, data=r.json(), emitted_at=int(datetime.now().timestamp()) * 1000
+            ),
         )
+
         return (m for m in [message])
 
     def _make_request(self, config):

diff --git a/...source-rest-api/source_rest_api/spec.json → ...ttp-request/source_http_request/spec.json b/...source-rest-api/source_rest_api/spec.json → ...ttp-request/source_http_request/spec.json