diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json new file mode 100644 index 000000000000..ecabb046eb08 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9845d17a-45f1-4070-8a60-50914b1c8e2b.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "9845d17a-45f1-4070-8a60-50914b1c8e2b", + "name": "HTTP Request", + "dockerRepository": "airbyte/source-http-request", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-http-request" +} diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteProtocolConverters.java b/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteProtocolConverters.java index 53c92a3e5ab0..2fce87672458 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteProtocolConverters.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/AirbyteProtocolConverters.java @@ -32,6 +32,7 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.SyncMode; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -45,6 +46,7 @@ public class AirbyteProtocolConverters { public static ConfiguredAirbyteCatalog toConfiguredCatalog(Schema schema) { final List airbyteStreams = schema.getStreams().stream() + .filter(s -> s.getSelected() != null && s.getSelected()) .map(s -> new ConfiguredAirbyteStream() // immutable // todo (cgardens) - not great that we just trust the API to not mutate these. @@ -53,16 +55,13 @@ public static ConfiguredAirbyteCatalog toConfiguredCatalog(Schema schema) { .withJsonSchema(toJson(s.getFields())) .withSupportedSyncModes(s.getSupportedSyncModes() .stream() - .map(e -> Enums.convertTo(e, io.airbyte.protocol.models.SyncMode.class)) + .map(e -> Enums.convertTo(e, SyncMode.class)) .collect(Collectors.toList())) .withSourceDefinedCursor(s.getSourceDefinedCursor()) .withDefaultCursorField(s.getDefaultCursorField())) // configurable - .withSyncMode(Enums.convertTo(s.getSyncMode(), io.airbyte.protocol.models.SyncMode.class)) + .withSyncMode(Enums.convertTo(s.getSyncMode(), SyncMode.class)) .withCursorField(s.getCursorField())) - - // perform selection based on the output of toJson, which keeps properties if selected=true - .filter(s -> !s.getStream().getJsonSchema().get("properties").isEmpty()) .collect(Collectors.toList()); return new ConfiguredAirbyteCatalog().withStreams(airbyteStreams); } @@ -81,7 +80,6 @@ public static JsonNode toJson(List fields) { .put("type", "object") .put("properties", fields .stream() - .filter(Field::getSelected) .collect(Collectors.toMap( Field::getName, field -> ImmutableMap.of("type", field.getDataType().toString().toLowerCase())))) @@ -100,7 +98,8 @@ public static Schema toSchema(AirbyteCatalog catalog) { .map(e -> Enums.convertTo(e, StandardSync.SyncMode.class)) .collect(Collectors.toList())) .withSourceDefinedCursor(airbyteStream.getSourceDefinedCursor()) - .withDefaultCursorField(airbyteStream.getDefaultCursorField())) + .withDefaultCursorField(airbyteStream.getDefaultCursorField()) + .withSelected(true)) // by default all discovered streams are treated as default. // configurable fields syncMode and cursorField are not set since they will never be defined in an // AirbyteCatalog. .collect(Collectors.toList())); @@ -115,10 +114,13 @@ private static List toFields(JsonNode jsonSchemaPropertiesObject) { list.add(iterator.next()); } - return list.stream().map(item -> new Field() - .withName(item.getKey()) - .withDataType(getDataType(item.getValue())) - .withSelected(true)).collect(Collectors.toList()); + return list + .stream() + .map(item -> new Field() + .withName(item.getKey()) + .withDataType(getDataType(item.getValue())) + .withSelected(true)) + .collect(Collectors.toList()); } // todo (cgardens) - add more robust handling for jsonschema types. diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/AirbyteProtocolConvertersTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/AirbyteProtocolConvertersTest.java index bb0a038a48c5..8938b09bf98a 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/AirbyteProtocolConvertersTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/AirbyteProtocolConvertersTest.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; @@ -38,6 +39,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; +import java.util.Collections; import org.junit.jupiter.api.Test; class AirbyteProtocolConvertersTest { @@ -66,32 +68,30 @@ class AirbyteProtocolConvertersTest { private static final Schema SCHEMA = new Schema() .withStreams(Lists.newArrayList(new Stream() + .withSelected(true) .withName(STREAM) .withFields(Lists.newArrayList( new io.airbyte.config.Field() .withName(COLUMN_NAME) - .withDataType(DataType.STRING) - .withSelected(true), + .withDataType(DataType.STRING), new io.airbyte.config.Field() .withName(COLUMN_AGE) - .withDataType(DataType.NUMBER) - .withSelected(true))) + .withDataType(DataType.NUMBER))) .withSourceDefinedCursor(false) .withSupportedSyncModes(Lists.newArrayList(StandardSync.SyncMode.FULL_REFRESH, StandardSync.SyncMode.INCREMENTAL)) .withDefaultCursorField(Lists.newArrayList(COLUMN_AGE)))); private static final Schema SCHEMA_WITH_UNSELECTED = new Schema() .withStreams(Lists.newArrayList(new Stream() + .withSelected(true) .withName(STREAM) .withFields(Lists.newArrayList( new io.airbyte.config.Field() .withName(COLUMN_NAME) - .withDataType(DataType.STRING) - .withSelected(true), + .withDataType(DataType.STRING), new io.airbyte.config.Field() .withName(COLUMN_AGE) - .withDataType(DataType.NUMBER) - .withSelected(true))) + .withDataType(DataType.NUMBER))) .withSourceDefinedCursor(false) .withSupportedSyncModes(Lists.newArrayList(StandardSync.SyncMode.FULL_REFRESH, StandardSync.SyncMode.INCREMENTAL)) .withDefaultCursorField(Lists.newArrayList(COLUMN_AGE)), @@ -100,12 +100,10 @@ class AirbyteProtocolConvertersTest { .withFields(Lists.newArrayList( new io.airbyte.config.Field() .withName(COLUMN_NAME) - .withDataType(DataType.STRING) - .withSelected(false), + .withDataType(DataType.STRING), new io.airbyte.config.Field() .withName(COLUMN_AGE) - .withDataType(DataType.NUMBER) - .withSelected(false))))); + .withDataType(DataType.NUMBER))))); @Test void testToConfiguredCatalog() { @@ -127,7 +125,10 @@ void testToConfiguredCatalogWithUnselectedStream() { @Test void testToSchema() { - assertEquals(SCHEMA, AirbyteProtocolConverters.toSchema(CATALOG)); + final Schema schema = Jsons.clone(SCHEMA); + schema.getStreams().forEach(table -> table.getFields().forEach(c -> c.setSelected(true))); // select all fields + + assertEquals(schema, AirbyteProtocolConverters.toSchema(CATALOG)); } @Test @@ -145,7 +146,8 @@ void testToSchemaWithMultipleJsonSchemaTypesAndFormats() { new io.airbyte.config.Field() .withName(COLUMN_AGE) .withDataType(DataType.NUMBER) - .withSelected(true))))); + .withSelected(true))) + .withSelected(true))); assertEquals(schema, AirbyteProtocolConverters.toSchema(catalog)); } @@ -162,12 +164,27 @@ void testAnyOfAsObject() { new io.airbyte.config.Field() .withName("date") .withDataType(DataType.OBJECT) - .withSelected(true))))); + .withSelected(true))) + .withSelected(true))); final AirbyteCatalog catalog = Jsons.deserialize(testString, AirbyteCatalog.class); assertEquals(schema, AirbyteProtocolConverters.toSchema(catalog)); } + @Test + void testStreamWithNoFields() { + final Schema schema = Jsons.clone(SCHEMA); + schema.getStreams().get(0).withCursorField(Lists.newArrayList(COLUMN_NAME)); + schema.getStreams().get(0).withSyncMode(StandardSync.SyncMode.INCREMENTAL); + schema.getStreams().get(0).setFields(Lists.newArrayList()); + final ConfiguredAirbyteCatalog actualCatalog = AirbyteProtocolConverters.toConfiguredCatalog(schema); + + final ConfiguredAirbyteCatalog expectedCatalog = Jsons.clone(CONFIGURED_CATALOG); + ((ObjectNode) expectedCatalog.getStreams().get(0).getStream().getJsonSchema()).set("properties", Jsons.jsonNode(Collections.emptyMap())); + + assertEquals(expectedCatalog, actualCatalog); + } + @Test void testEnumConversion() { assertTrue(Enums.isCompatible(io.airbyte.protocol.models.SyncMode.class, io.airbyte.config.StandardSync.SyncMode.class)); diff --git a/airbyte-integrations/connectors/source-rest-api/Dockerfile b/airbyte-integrations/connectors/source-http-request/Dockerfile similarity index 60% rename from airbyte-integrations/connectors/source-rest-api/Dockerfile rename to airbyte-integrations/connectors/source-http-request/Dockerfile index 2f86e8bf6d14..3f73a612d6dc 100644 --- a/airbyte-integrations/connectors/source-rest-api/Dockerfile +++ b/airbyte-integrations/connectors/source-http-request/Dockerfile @@ -2,9 +2,9 @@ FROM airbyte/integration-base-python:dev RUN apt-get update && apt-get install -y jq curl bash && rm -rf /var/lib/apt/lists/* -ENV CODE_PATH="source_rest_api" -ENV AIRBYTE_IMPL_MODULE="source_rest_api" -ENV AIRBYTE_IMPL_PATH="SourceRestApi" +ENV CODE_PATH="source_http_request" +ENV AIRBYTE_IMPL_MODULE="source_http_request" +ENV AIRBYTE_IMPL_PATH="SourceHttpRequest" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH @@ -12,4 +12,4 @@ COPY setup.py ./ RUN pip install ".[main]" LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-rest-api +LABEL io.airbyte.name=airbyte/source-http-request diff --git a/airbyte-integrations/connectors/source-rest-api/Dockerfile.test b/airbyte-integrations/connectors/source-http-request/Dockerfile.test similarity index 62% rename from airbyte-integrations/connectors/source-rest-api/Dockerfile.test rename to airbyte-integrations/connectors/source-http-request/Dockerfile.test index 524f55e6e7d1..5e5b4e3f663c 100644 --- a/airbyte-integrations/connectors/source-rest-api/Dockerfile.test +++ b/airbyte-integrations/connectors/source-http-request/Dockerfile.test @@ -4,16 +4,16 @@ RUN apt-get update && rm -rf /var/lib/apt/lists/* ENV CODE_PATH="integration_tests" ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceRestApiStandardTest" +ENV AIRBYTE_TEST_PATH="SourceHttpRequestStandardTest" ENV AIRBYTE_TEST_CASE=true LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-rest-api-standard-test +LABEL io.airbyte.name=airbyte/source-http-request-standard-test WORKDIR /airbyte/integration_code -COPY source_rest_api source_rest_api +COPY source_http_request source_http_request COPY $CODE_PATH $CODE_PATH -COPY source_rest_api/*.json $CODE_PATH +COPY source_http_request/*.json $CODE_PATH COPY setup.py ./ RUN pip install ".[tests]" diff --git a/airbyte-integrations/connectors/source-rest-api/README.md b/airbyte-integrations/connectors/source-http-request/README.md similarity index 64% rename from airbyte-integrations/connectors/source-rest-api/README.md rename to airbyte-integrations/connectors/source-http-request/README.md index b56c74e8a6bb..712af1dcf1e6 100644 --- a/airbyte-integrations/connectors/source-rest-api/README.md +++ b/airbyte-integrations/connectors/source-http-request/README.md @@ -1,19 +1,19 @@ -# Rest Api Source +# HTTP Request Source -This is the repository for the Rest Api source connector, written in Python. +This is the repository for the HTTP Request source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rest-api). ## Local development ### Build First, build the module by running the following from the `airbyte` project root directory: ``` -./gradlew :airbyte-integrations:connectors:source-rest-api:build +./gradlew :airbyte-integrations:connectors:source-http-request:build ``` -This should generate a virtualenv for this module in `source-rest-api/.venv`. Make sure this venv is active in your -development environment of choice. If you are on the terminal, run the following from the `source-rest-api` directory: +This should generate a virtualenv for this module in `source-http-request/.venv`. Make sure this venv is active in your +development environment of choice. If you are on the terminal, run the following from the `source-http-request` directory: ``` -cd airbyte-integrations/connectors/source-rest-api # cd into the connector directory +cd airbyte-integrations/connectors/source-http-request # cd into the connector directory source .venv/bin/activate ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -37,16 +37,16 @@ pytest unit_tests ### Locally running the connector docker image ``` # in airbyte root directory -./gradlew :airbyte-integrations:connectors:source-rest-api:airbyteDocker -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev spec -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev check --config /sample_files/sample_files/config.json -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev discover --config /sample_files/sample_files/config.json -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-rest-api:/sample_files airbyte/source-rest-api:dev read --config /sample_files/sample_files/config.json --catalog /sample_files/integration_tests/catalog.json +./gradlew :airbyte-integrations:connectors:source-http-request:airbyteDocker +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev spec +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev check --config /sample_files/sample_files/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev discover --config /sample_files/sample_files/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-http-request:/sample_files airbyte/source-http-request:dev read --config /sample_files/sample_files/config.json --catalog /sample_files/integration_tests/catalog.json ``` ### Integration Tests 1. Configure credentials as appropriate, described below. -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-rest-api:standardSourceTestPython` to run the standard integration test suite. +1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-http-request:standardSourceTestPython` to run the standard integration test suite. 1. To run additional integration tests, place your integration tests in the `integration_tests` directory and run them with `pytest integration_tests`. Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. diff --git a/airbyte-integrations/connectors/source-rest-api/airbyte_protocol b/airbyte-integrations/connectors/source-http-request/airbyte_protocol similarity index 100% rename from airbyte-integrations/connectors/source-rest-api/airbyte_protocol rename to airbyte-integrations/connectors/source-http-request/airbyte_protocol diff --git a/airbyte-integrations/connectors/source-rest-api/base_python b/airbyte-integrations/connectors/source-http-request/base_python similarity index 100% rename from airbyte-integrations/connectors/source-rest-api/base_python rename to airbyte-integrations/connectors/source-http-request/base_python diff --git a/airbyte-integrations/connectors/source-rest-api/build.gradle b/airbyte-integrations/connectors/source-http-request/build.gradle similarity index 59% rename from airbyte-integrations/connectors/source-rest-api/build.gradle rename to airbyte-integrations/connectors/source-http-request/build.gradle index b29f61b27d76..30efbc4c8ae4 100644 --- a/airbyte-integrations/connectors/source-rest-api/build.gradle +++ b/airbyte-integrations/connectors/source-http-request/build.gradle @@ -2,12 +2,11 @@ plugins { id 'java' id 'airbyte-python' id 'airbyte-docker' - // todo (cgardens) - bring back when we re-add this to the product. - // id 'airbyte-source-test' + id 'airbyte-source-test' } airbytePython { - moduleDirectory 'source_rest_api' + moduleDirectory 'source_http_request' } dependencies { diff --git a/airbyte-integrations/connectors/source-rest-api/integration_tests/__init__.py b/airbyte-integrations/connectors/source-http-request/integration_tests/__init__.py similarity index 90% rename from airbyte-integrations/connectors/source-rest-api/integration_tests/__init__.py rename to airbyte-integrations/connectors/source-http-request/integration_tests/__init__.py index 33c3deb193e8..260939f2b294 100644 --- a/airbyte-integrations/connectors/source-rest-api/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-http-request/integration_tests/__init__.py @@ -22,6 +22,6 @@ SOFTWARE. """ -from .standard_source_test import SourceRestApiStandardTest +from .standard_source_test import SourceHttpRequestStandardTest -__all__ = ["SourceRestApiStandardTest"] +__all__ = ["SourceHttpRequestStandardTest"] diff --git a/airbyte-integrations/connectors/source-rest-api/integration_tests/catalog.json b/airbyte-integrations/connectors/source-http-request/integration_tests/catalog.json similarity index 100% rename from airbyte-integrations/connectors/source-rest-api/integration_tests/catalog.json rename to airbyte-integrations/connectors/source-http-request/integration_tests/catalog.json diff --git a/airbyte-integrations/connectors/source-rest-api/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-http-request/integration_tests/standard_source_test.py similarity index 97% rename from airbyte-integrations/connectors/source-rest-api/integration_tests/standard_source_test.py rename to airbyte-integrations/connectors/source-http-request/integration_tests/standard_source_test.py index fd0e54300db5..2cf70fe8454e 100644 --- a/airbyte-integrations/connectors/source-rest-api/integration_tests/standard_source_test.py +++ b/airbyte-integrations/connectors/source-http-request/integration_tests/standard_source_test.py @@ -29,7 +29,7 @@ from base_python_test import StandardSourceTestIface -class SourceRestApiStandardTest(StandardSourceTestIface): +class SourceHttpRequestStandardTest(StandardSourceTestIface): def __init__(self): pass diff --git a/airbyte-integrations/connectors/source-rest-api/main_dev.py b/airbyte-integrations/connectors/source-http-request/main_dev.py similarity index 93% rename from airbyte-integrations/connectors/source-rest-api/main_dev.py rename to airbyte-integrations/connectors/source-http-request/main_dev.py index f602ba80815c..050b79046da5 100644 --- a/airbyte-integrations/connectors/source-rest-api/main_dev.py +++ b/airbyte-integrations/connectors/source-http-request/main_dev.py @@ -25,8 +25,8 @@ import sys from base_python.entrypoint import launch -from source_rest_api import SourceRestApi +from source_http_request import SourceHttpRequest if __name__ == "__main__": - source = SourceRestApi() + source = SourceHttpRequest() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rest-api/requirements.txt b/airbyte-integrations/connectors/source-http-request/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-rest-api/requirements.txt rename to airbyte-integrations/connectors/source-http-request/requirements.txt diff --git a/airbyte-integrations/connectors/source-rest-api/sample_files/config.json b/airbyte-integrations/connectors/source-http-request/sample_files/config.json similarity index 100% rename from airbyte-integrations/connectors/source-rest-api/sample_files/config.json rename to airbyte-integrations/connectors/source-http-request/sample_files/config.json diff --git a/airbyte-integrations/connectors/source-rest-api/setup.py b/airbyte-integrations/connectors/source-http-request/setup.py similarity index 94% rename from airbyte-integrations/connectors/source-rest-api/setup.py rename to airbyte-integrations/connectors/source-http-request/setup.py index b84a056f891d..e79c6799ea8b 100644 --- a/airbyte-integrations/connectors/source-rest-api/setup.py +++ b/airbyte-integrations/connectors/source-http-request/setup.py @@ -25,8 +25,8 @@ from setuptools import find_packages, setup setup( - name="source_rest_api", - description="Source implementation for Rest Api.", + name="source_http_request", + description="Source implementation for HTTP Request.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), diff --git a/airbyte-integrations/connectors/source-rest-api/source_rest_api/__init__.py b/airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py similarity index 93% rename from airbyte-integrations/connectors/source-rest-api/source_rest_api/__init__.py rename to airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py index 3e1eed2ebfe8..6a969389f8ec 100644 --- a/airbyte-integrations/connectors/source-rest-api/source_rest_api/__init__.py +++ b/airbyte-integrations/connectors/source-http-request/source_http_request/__init__.py @@ -22,6 +22,6 @@ SOFTWARE. """ -from .source import SourceRestApi +from .source import SourceHttpRequest -__all__ = ["SourceRestApi"] +__all__ = ["SourceHttpRequest"] diff --git a/airbyte-integrations/connectors/source-rest-api/source_rest_api/source.py b/airbyte-integrations/connectors/source-http-request/source_http_request/source.py similarity index 87% rename from airbyte-integrations/connectors/source-rest-api/source_rest_api/source.py rename to airbyte-integrations/connectors/source-http-request/source_http_request/source.py index 27c2f0679077..a299cbbdffbe 100644 --- a/airbyte-integrations/connectors/source-rest-api/source_rest_api/source.py +++ b/airbyte-integrations/connectors/source-http-request/source_http_request/source.py @@ -31,7 +31,7 @@ from base_python import AirbyteLogger, Source -class SourceRestApi(Source): +class SourceHttpRequest(Source): STREAM_NAME = "data" def __init__(self): @@ -49,11 +49,13 @@ def discover(self, logger: AirbyteLogger, config_container) -> AirbyteCatalog: "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": True, "type": "object", - "properties": {}, + # todo (cgardens) - remove data column. added to handle UI bug where streams without fields cannot be selected. + # issue: https://github.com/airbytehq/airbyte/issues/1104 + "properties": {"data": {"type": "object"}}, } # json body will be returned as the "data" stream". we can't know its schema ahead of time, so we assume it's object (i.e. valid json). - return AirbyteCatalog(streams=[AirbyteStream(name=SourceRestApi.STREAM_NAME, json_schema=json_schema)]) + return AirbyteCatalog(streams=[AirbyteStream(name=SourceHttpRequest.STREAM_NAME, json_schema=json_schema)]) def read(self, logger: AirbyteLogger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: r = self._make_request(config_container.rendered_config) @@ -63,8 +65,11 @@ def read(self, logger: AirbyteLogger, config_container, catalog_path, state=None # need to eagerly fetch the json. message = AirbyteMessage( type=Type.RECORD, - record=AirbyteRecordMessage(stream=SourceRestApi.STREAM_NAME, data=r.json(), emitted_at=int(datetime.now().timestamp()) * 1000), + record=AirbyteRecordMessage( + stream=SourceHttpRequest.STREAM_NAME, data=r.json(), emitted_at=int(datetime.now().timestamp()) * 1000 + ), ) + return (m for m in [message]) def _make_request(self, config): diff --git a/airbyte-integrations/connectors/source-rest-api/source_rest_api/spec.json b/airbyte-integrations/connectors/source-http-request/source_http_request/spec.json similarity index 100% rename from airbyte-integrations/connectors/source-rest-api/source_rest_api/spec.json rename to airbyte-integrations/connectors/source-http-request/source_http_request/spec.json diff --git a/airbyte-integrations/connectors/source-rest-api/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py similarity index 92% rename from airbyte-integrations/connectors/source-rest-api/unit_tests/unit_test.py rename to airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py index ab0f643c16aa..08d36e7f0366 100644 --- a/airbyte-integrations/connectors/source-rest-api/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py @@ -24,10 +24,10 @@ import unittest -from source_rest_api import SourceRestApi +from source_http_request import SourceHttpRequest -class TestSourceRestApi(unittest.TestCase): +class TestSourceHttpRequest(unittest.TestCase): def test_parse_config(self): config = { "http_method": "get", @@ -36,7 +36,7 @@ def test_parse_config(self): "body": '{"something": "good"}', } - source = SourceRestApi() + source = SourceHttpRequest() actual = source._parse_config(config) expected = { "http_method": "get", diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java index 34501d910e6f..ff0edfab3560 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java @@ -98,7 +98,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class AcceptanceTests { - private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME = "public.id_and_name"; private static final String COLUMN_ID = "id"; private static final String COLUMN_NAME = "name"; @@ -371,8 +371,9 @@ private void assertSourceAndTargetDbInSync(PostgreSQLContainer sourceDb) throws final Database database = getDatabase(sourceDb); final Set sourceStreams = listStreams(database); - final Set targetStreams = listCsvStreams(); - assertEquals(sourceStreams, targetStreams); + final Set destinationStreams = listCsvStreams(); + assertEquals(sourceStreams, destinationStreams, + String.format("streams did not match.\n source stream names: %s\n destination stream names: %s\n", sourceStreams, destinationStreams)); for (String table : sourceStreams) { assertStreamsEquivalent(database, table); @@ -388,9 +389,13 @@ private Set listStreams(Database database) throws SQLException { context -> { Result fetch = context.fetch( - "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema'"); + "SELECT tablename, schemaname FROM pg_catalog.pg_tables WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema'"); return fetch.stream() - .map(record -> (String) record.get("tablename")) + .map(record -> { + final String schemaName = (String) record.get("schemaname"); + final String tableName = (String) record.get("tablename"); + return schemaName + "." + tableName; + }) .collect(Collectors.toSet()); }); } diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 43541cb3b276..fa6fdb29475f 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -18,13 +18,13 @@ * [Google Analytics](integrations/sources/googleanalytics.md) * [Google Sheets](integrations/sources/google-sheets.md) * [Hubspot](integrations/sources/hubspot.md) + * [HTTP Request](integrations/sources/http-request.md) * [Mailchimp](integrations/sources/mailchimp.md) * [Marketo](integrations/sources/marketo.md) * [MySQL](integrations/sources/mysql.md) * [Microsoft SQL Server \(MSSQL\)](integrations/sources/mssql.md) * [Postgres](integrations/sources/postgres.md) * [Recurly](integrations/sources/recurly.md) - * [Rest API](integrations/sources/rest-api.md) * [Salesforce](integrations/sources/salesforce.md) * [Sendgrid](integrations/sources/sendgrid.md) * [Shopify](integrations/sources/shopify.md) diff --git a/docs/integrations/sources/rest-api.md b/docs/integrations/sources/http-request.md similarity index 97% rename from docs/integrations/sources/rest-api.md rename to docs/integrations/sources/http-request.md index 1572f2a72aeb..166212569687 100644 --- a/docs/integrations/sources/rest-api.md +++ b/docs/integrations/sources/http-request.md @@ -1,4 +1,4 @@ -# Rest API +# HTTP Request ## Overview