diff --git a/.env b/.env index a8f8f95d6482..8efe0cda076a 100644 --- a/.env +++ b/.env @@ -67,6 +67,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= ### LOGGING/MONITORING/TRACKING ### TRACKING_STRATEGY=segment diff --git a/.github/workflows/shared-pulls.yml b/.github/workflows/shared-pulls.yml index df9c11d6b49a..d5635e2769a8 100644 --- a/.github/workflows/shared-pulls.yml +++ b/.github/workflows/shared-pulls.yml @@ -4,35 +4,13 @@ on: types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened] jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.OCTAVIA_PAT }} \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} - shared-pulls: - name: "Label github issues for tracking" - needs: - - find_valid_pat runs-on: ubuntu-latest steps: - uses: nick-fields/private-action-loader@v3 with: - pal-repo-token: ${{ needs.find_valid_pat.outputs.pat }} + pal-repo-token: "${{ secrets.OCTAVIA_PAT }}" pal-repo-name: airbytehq/workflow-actions@production # the following input gets passed to the private action - token: ${{ needs.find_valid_pat.outputs.pat }} + token: "${{ secrets.OCTAVIA_PAT }}" command: "pull" diff --git a/.vscode/frontend.code-workspace b/.vscode/frontend.code-workspace index 8ec2e7d144b2..3e2645deb6f5 100644 --- a/.vscode/frontend.code-workspace +++ b/.vscode/frontend.code-workspace @@ -5,6 +5,9 @@ }, { "path": "../airbyte-webapp-e2e-tests" + }, + { + "path": "../docs" } ], "extensions": { diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index f9314d1b0c64..10a36e9fb2ae 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -7,6 +7,49 @@ plugins { def specFile = "$projectDir/src/main/openapi/config.yaml" +// Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer +task generateApiServerLegacy(type: GenerateTask) { + def serverOutputDir = "$buildDir/generated/api/server" + + inputs.file specFile + outputs.dir serverOutputDir + + generatorName = "jaxrs-spec" + inputSpec = specFile + outputDir = serverOutputDir + + apiPackage = "io.airbyte.api.generated" + invokerPackage = "io.airbyte.api.invoker.generated" + modelPackage = "io.airbyte.api.model.generated" + + importMappings = [ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', + ] + + generateApiDocumentation = false + + configOptions = [ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true", + /* + JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. + It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. + The below Jackson annotation is made to only keep non null values in serialized json. + We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. + Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. + */ + additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + ] +} + task generateApiServer(type: GenerateTask) { def serverOutputDir = "$buildDir/generated/api/server" @@ -45,10 +88,14 @@ task generateApiServer(type: GenerateTask) { We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)" + additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + + // Generate separate classes for each endpoint "domain" + useTags: "true" ] } -compileJava.dependsOn tasks.generateApiServer + +compileJava.dependsOn tasks.generateApiServerLegacy, tasks.generateApiServer task generateApiClient(type: GenerateTask) { def clientOutputDir = "$buildDir/generated/api/client" diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index da65057ffdba..c13f98405731 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3218,6 +3218,10 @@ components: $ref: "#/components/schemas/AirbyteCatalog" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" resourceRequirements: @@ -3257,6 +3261,10 @@ components: $ref: "#/components/schemas/AirbyteCatalog" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" resourceRequirements: @@ -3298,6 +3306,10 @@ components: $ref: "#/components/schemas/AirbyteCatalog" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" resourceRequirements: @@ -3335,6 +3347,10 @@ components: $ref: "#/components/schemas/AirbyteCatalog" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" resourceRequirements: @@ -3386,6 +3402,10 @@ components: $ref: "#/components/schemas/AirbyteCatalog" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" resourceRequirements: @@ -3416,6 +3436,10 @@ components: $ref: "#/components/schemas/DestinationId" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" source: @@ -3445,6 +3469,10 @@ components: $ref: "#/components/schemas/DestinationId" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" source: @@ -3467,6 +3495,8 @@ components: - active - inactive - deprecated + # TODO(https://github.com/airbytehq/airbyte/issues/11432): remove. + # Prefer the ConnectionScheduleType and ConnectionScheduleData properties. ConnectionSchedule: description: if null, then no schedule is set. type: object @@ -3485,6 +3515,46 @@ components: - days - weeks - months + ConnectionScheduleType: + description: determine how the schedule data should be interpreted + type: string + enum: + - manual + - basic + - cron + ConnectionScheduleData: + description: schedule for when the the connection should run, per the schedule type + type: object + properties: + # This should be populated when schedule type is basic. + basicSchedule: + type: object + required: + - timeUnit + - units + properties: + timeUnit: + type: string + enum: + - minutes + - hours + - days + - weeks + - months + units: + type: integer + format: int64 + # This should be populated when schedule type is cron. + cron: + type: object + required: + - cronExpression + - cronTimeZone + properties: + cronExpression: + type: string + cronTimeZone: + type: string NamespaceDefinitionType: type: string description: Method used for computing final namespace in destination @@ -4564,6 +4634,10 @@ components: $ref: "#/components/schemas/AirbyteCatalog" schedule: $ref: "#/components/schemas/ConnectionSchedule" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" status: $ref: "#/components/schemas/ConnectionStatus" operationIds: diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 7f0fa5574a95..05f9deed11e5 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,8 +1,14 @@ # Changelog -## 0.1.72 +## 0.1.74 +- Replace JelloRecordExtractor with DpathRecordExtractor + +## 0.1.73 - Bugfix: Fix bug in DatetimeStreamSlicer's parsing method +## 0.1.72 +- Bugfix: Fix bug in DatetimeStreamSlicer's format method + ## 0.1.71 - Refactor declarative package to dataclasses - Bugfix: Requester header always converted to string diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py new file mode 100644 index 000000000000..f3ed27da3a46 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from typing import Union + + +class DatetimeParser: + """ + Parses and formats datetime objects according to a specified format. + + This class mainly acts as a wrapper to properly handling timestamp formatting through the "%s" directive. + + %s is part of the list of format codes required by the 1989 C standard, but it is unreliable because it always return a datetime in the system's timezone. + Instead of using the directive directly, we can use datetime.fromtimestamp and dt.timestamp() + """ + + def parse(self, date: Union[str, int], format: str, timezone): + # "%s" is a valid (but unreliable) directive for formatting, but not for parsing + # It is defined as + # The number of seconds since the Epoch, 1970-01-01 00:00:00+0000 (UTC). https://man7.org/linux/man-pages/man3/strptime.3.html + # + # The recommended way to parse a date from its timestamp representation is to use datetime.fromtimestamp + # See https://stackoverflow.com/a/4974930 + if format == "%s": + return datetime.datetime.fromtimestamp(int(date), tz=timezone) + else: + return datetime.datetime.strptime(str(date), format).replace(tzinfo=timezone) + + def format(self, dt: datetime.datetime, format: str) -> str: + # strftime("%s") is unreliable because it ignores the time zone information and assumes the time zone of the system it's running on + # It's safer to use the timestamp() method than the %s directive + # See https://stackoverflow.com/a/4974930 + if format == "%s": + return str(int(dt.timestamp())) + else: + return dt.strftime(format) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py index 0c4b5232cf69..c7b3b498b28a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py @@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field from typing import Any, Mapping, Union +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from dataclasses_jsonschema import JsonSchemaMixin @@ -40,6 +41,7 @@ class MinMaxDatetime(JsonSchemaMixin): def __post_init__(self, options: Mapping[str, Any]): self.datetime = InterpolatedString.create(self.datetime, options=options or {}) self.timezone = dt.timezone.utc + self._parser = DatetimeParser() self.min_datetime = InterpolatedString.create(self.min_datetime, options=options) if self.min_datetime else None self.max_datetime = InterpolatedString.create(self.max_datetime, options=options) if self.max_datetime else None @@ -57,17 +59,13 @@ def get_datetime(self, config, **additional_options) -> dt.datetime: if not datetime_format: datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z" - time = dt.datetime.strptime(str(self.datetime.eval(config, **additional_options)), datetime_format).replace(tzinfo=self._timezone) + time = self._parser.parse(str(self.datetime.eval(config, **additional_options)), datetime_format, self.timezone) if self.min_datetime: - min_time = dt.datetime.strptime(str(self.min_datetime.eval(config, **additional_options)), datetime_format).replace( - tzinfo=self._timezone - ) + min_time = self._parser.parse(str(self.min_datetime.eval(config, **additional_options)), datetime_format, self.timezone) time = max(time, min_time) if self.max_datetime: - max_time = dt.datetime.strptime(str(self.max_datetime.eval(config, **additional_options)), datetime_format).replace( - tzinfo=self._timezone - ) + max_time = self._parser.parse(str(self.max_datetime.eval(config, **additional_options)), datetime_format, self.timezone) time = min(time, max_time) return time diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index feae3fa4d51a..100a76a1035f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -20,8 +20,8 @@ class DeclarativeStream(Stream, JsonSchemaMixin): DeclarativeStream is a Stream that delegates most of its logic to its schema_load and retriever Attributes: - stream_name (str): stream name - stream_primary_key (Optional[Union[str, List[str], List[List[str]]]]): the primary key of the stream + name (str): stream name + primary_key (Optional[Union[str, List[str], List[List[str]]]]): the primary key of the stream schema_loader (SchemaLoader): The schema loader retriever (Retriever): The retriever config (Config): The user-provided configuration as specified by the source's spec diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/read_exception.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/exceptions.py similarity index 58% rename from airbyte-cdk/python/airbyte_cdk/sources/declarative/read_exception.py rename to airbyte-cdk/python/airbyte_cdk/sources/declarative/exceptions.py index 160cdcb43f0c..185a833b0def 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/read_exception.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/exceptions.py @@ -7,3 +7,9 @@ class ReadException(Exception): """ Raise when there is an error reading data from an API Source """ + + +class InvalidConnectorDefinitionException(Exception): + """ + Raise when the connector definition is invalid + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py index 897f382ea0de..5f76dfa5e796 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/__init__.py @@ -2,9 +2,9 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector -__all__ = ["HttpSelector", "JelloExtractor", "RecordFilter", "RecordSelector"] +__all__ = ["HttpSelector", "DpathExtractor", "RecordFilter", "RecordSelector"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py new file mode 100644 index 000000000000..164161de63e6 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Union + +import dpath.util +import requests +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config, Record +from dataclasses_jsonschema import JsonSchemaMixin + + +@dataclass +class DpathExtractor(RecordExtractor, JsonSchemaMixin): + """ + Record extractor that searches a decoded response over a path defined as an array of fields. + + If the field pointer points to an array, that array is returned. + If the field pointer points to an object, that object is returned wrapped as an array. + If the field pointer points to an empty object, an empty array is returned. + If the field pointer points to a non-existing path, an empty array is returned. + + Examples of instantiating this transform: + ``` + extractor: + type: DpathExtractor + field_pointer: + - "root" + - "data" + ``` + + ``` + extractor: + type: DpathExtractor + field_pointer: + - "root" + - "{{ options['field'] }}" + ``` + + ``` + extractor: + type: DpathExtractor + field_pointer: [] + ``` + + Attributes: + transform (Union[InterpolatedString, str]): Pointer to the field that should be extracted + config (Config): The user-provided configuration as specified by the source's spec + decoder (Decoder): The decoder responsible to transfom the response in a Mapping + """ + + field_pointer: List[Union[InterpolatedString, str]] + config: Config + options: InitVar[Mapping[str, Any]] + decoder: Decoder = JsonDecoder(options={}) + + def __post_init__(self, options: Mapping[str, Any]): + for pointer_index in range(len(self.field_pointer)): + if isinstance(self.field_pointer[pointer_index], str): + self.field_pointer[pointer_index] = InterpolatedString.create(self.field_pointer[pointer_index], options=options) + + def extract_records(self, response: requests.Response) -> List[Record]: + response_body = self.decoder.decode(response) + if len(self.field_pointer) == 0: + extracted = response_body + else: + pointer = [pointer.eval(self.config) for pointer in self.field_pointer] + extracted = dpath.util.get(response_body, pointer, default=[]) + if isinstance(extracted, list): + return extracted + elif extracted: + return [extracted] + else: + return [] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py deleted file mode 100644 index f36613e2a56e..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from dataclasses import InitVar, dataclass -from typing import Any, List, Mapping, Union - -import requests -from airbyte_cdk.sources.declarative.decoders.decoder import Decoder -from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Config, Record -from dataclasses_jsonschema import JsonSchemaMixin -from jello import lib as jello_lib - - -@dataclass -class JelloExtractor(JsonSchemaMixin): - """ - Record extractor that evaluates a Jello query to extract records from a decoded response. - - More information on Jello can be found at https://github.com/kellyjonbrazil/jello - - Attributes: - transform (Union[InterpolatedString, str]): The Jello query to evaluate on the decoded response - config (Config): The user-provided configuration as specified by the source's spec - decoder (Decoder): The decoder responsible to transfom the response in a Mapping - """ - - default_transform = "_" - transform: Union[InterpolatedString, str] - config: Config - options: InitVar[Mapping[str, Any]] - decoder: Decoder = JsonDecoder(options={}) - - def __post_init__(self, options: Mapping[str, Any]): - if isinstance(self.transform, str): - self.transform = InterpolatedString(string=self.transform, default=self.default_transform, options=options or {}) - - def extract_records(self, response: requests.Response) -> List[Record]: - response_body = self.decoder.decode(response) - script = self.transform.eval(self.config) - return jello_lib.pyquery(response_body, script) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py new file mode 100644 index 000000000000..5e2b865156eb --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List + +import requests +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class RecordExtractor(ABC): + """ + Responsible for translating an HTTP response into a list of records by extracting records from the response. + """ + + @abstractmethod + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + """ + Selects records from the response + :param response: The response to extract the records from + :return: List of Records extracted from the response + """ + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index dd738a69015d..34d93c13763d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -7,7 +7,7 @@ import requests from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from dataclasses_jsonschema import JsonSchemaMixin @@ -20,11 +20,11 @@ class RecordSelector(HttpSelector, JsonSchemaMixin): records based on a heuristic. Attributes: - extractor (JelloExtractor): The record extractor responsible for extracting records from a response + extractor (RecordExtractor): The record extractor responsible for extracting records from a response record_filter (RecordFilter): The record filter responsible for filtering extracted records """ - extractor: JelloExtractor + extractor: RecordExtractor options: InitVar[Mapping[str, Any]] record_filter: RecordFilter = None @@ -39,9 +39,6 @@ def select_records( next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: all_records = self.extractor.extract_records(response) - # Some APIs don't wrap single records in a list - if not isinstance(all_records, list): - all_records = [all_records] if self.record_filter: return self.record_filter.filter_records( all_records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index ad0c268e1ac1..602176f36e9e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -7,7 +7,7 @@ from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -46,11 +46,11 @@ "DatetimeStreamSlicer": DatetimeStreamSlicer, "DeclarativeStream": DeclarativeStream, "DefaultErrorHandler": DefaultErrorHandler, + "DpathExtractor": DpathExtractor, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, "InterpolatedBoolean": InterpolatedBoolean, "InterpolatedString": InterpolatedString, - "JelloExtractor": JelloExtractor, "JsonSchema": JsonSchema, "LimitPaginator": LimitPaginator, "ListStreamSlicer": ListStreamSlicer, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py index f09c00d954e8..234504b2de0b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py @@ -10,7 +10,9 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -50,11 +52,12 @@ InterpolatedString: InterpolatedString, MinMaxDatetime: MinMaxDatetime, Paginator: NoPagination, + ParentStreamConfig: ParentStreamConfig, + RecordExtractor: DpathExtractor, RequestOption: RequestOption, RequestOptionsProvider: InterpolatedRequestOptionsProvider, Requester: HttpRequester, Retriever: SimpleRetriever, - ParentStreamConfig: ParentStreamConfig, SchemaLoader: JsonSchema, Stream: DeclarativeStream, StreamSlicer: SingleSlice, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 8443026161d5..d019c87b95c5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -7,8 +7,8 @@ import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector -from airbyte_cdk.sources.declarative.read_exception import ReadException from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py index e3a42dd04f17..acdf92e6a35f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py @@ -25,7 +25,6 @@ class JsonSchema(SchemaLoader, JsonSchemaMixin): """ file_path: Union[InterpolatedString, str] - name: str config: Config options: InitVar[Mapping[str, Any]] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index c81d11e85129..ff08da789638 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -5,9 +5,10 @@ import datetime import re from dataclasses import InitVar, dataclass, field -from typing import Any, Iterable, Mapping, Optional, Union +from typing import Any, Iterable, Mapping, Optional from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation @@ -77,6 +78,7 @@ def __post_init__(self, options: Mapping[str, Any]): self.cursor_field = InterpolatedString.create(self.cursor_field, options=options) self.stream_slice_field_start = InterpolatedString.create(self.stream_state_field_start or "start_time", options=options) self.stream_slice_field_end = InterpolatedString.create(self.stream_state_field_end or "end_time", options=options) + self._parser = DatetimeParser() # If datetime format is not specified then start/end datetime should inherit it from the stream slicer if not self.start_datetime.datetime_format: @@ -142,7 +144,12 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> start_datetime = max(cursor_datetime, start_datetime) - state_date = self.parse_date(stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state))) + state_cursor_value = stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state)) + + if state_cursor_value: + state_date = self.parse_date(state_cursor_value) + else: + state_date = None if state_date: # If the input_state's date is greater than start_datetime, the start of the time window is the state's next day next_date = state_date + datetime.timedelta(days=1) @@ -151,13 +158,7 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> return dates def _format_datetime(self, dt: datetime.datetime): - # strftime("%s") is unreliable because it ignores the time zone information and assumes the time zone of the system it's running on - # It's safer to use the timestamp() method than the %s directive - # See https://stackoverflow.com/a/4974930 - if self.datetime_format == "%s": - return str(int(dt.timestamp())) - else: - return dt.strftime(self.datetime_format) + return self._parser.format(dt, self.datetime_format) def _partition_daterange(self, start, end, step: datetime.timedelta): start_field = self.stream_slice_field_start.eval(self.config) @@ -170,14 +171,11 @@ def _partition_daterange(self, start, end, step: datetime.timedelta): return dates def _get_date(self, cursor_value, default_date: datetime.datetime, comparator) -> datetime.datetime: - cursor_date = self.parse_date(cursor_value or default_date) + cursor_date = cursor_value or default_date return comparator(cursor_date, default_date) - def parse_date(self, date: Union[str, datetime.datetime]) -> datetime.datetime: - if isinstance(date, str): - return datetime.datetime.strptime(str(date), self.datetime_format).replace(tzinfo=self._timezone) - else: - return date + def parse_date(self, date: str) -> datetime.datetime: + return self._parser.parse(date, self.datetime_format, self._timezone) @classmethod def _parse_timedelta(cls, time_str): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index bebecdfa2e2a..f07537d2ed32 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource +from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException from airbyte_cdk.sources.declarative.parsers.factory import DeclarativeComponentFactory from airbyte_cdk.sources.declarative.parsers.yaml_parser import YamlParser from airbyte_cdk.sources.streams import Stream @@ -16,6 +17,8 @@ class YamlDeclarativeSource(DeclarativeSource): """Declarative source defined by a yaml file""" + VALID_TOP_LEVEL_FIELDS = {"definitions", "streams", "check", "version"} + def __init__(self, path_to_yaml): """ :param path_to_yaml: Path to the yaml file describing the source @@ -25,6 +28,11 @@ def __init__(self, path_to_yaml): self._path_to_yaml = path_to_yaml self._source_config = self._read_and_parse_yaml_file(path_to_yaml) + # Stopgap to protect the top-level namespace until it's validated through the schema + unknown_fields = [key for key in self._source_config.keys() if key not in self.VALID_TOP_LEVEL_FIELDS] + if unknown_fields: + raise InvalidConnectorDefinitionException(f"Found unknown top-level fields: {unknown_fields}") + @property def connection_checker(self) -> ConnectionChecker: check = self._source_config["check"] diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.datetime.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.datetime.rst index 7cd9ebae47ca..f523d1b1736a 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.datetime.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.datetime.rst @@ -2,6 +2,14 @@ Submodules ---------- +airbyte\_cdk.sources.declarative.datetime.datetime\_parser module +----------------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.declarative.datetime.datetime_parser + :members: + :undoc-members: + :show-inheritance: + airbyte\_cdk.sources.declarative.datetime.min\_max\_datetime module ------------------------------------------------------------------- diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.extractors.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.extractors.rst index 507b25296fe4..3b901d5c9f1e 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.extractors.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.extractors.rst @@ -2,6 +2,14 @@ Submodules ---------- +airbyte\_cdk.sources.declarative.extractors.dpath\_extractor module +------------------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.declarative.extractors.dpath_extractor + :members: + :undoc-members: + :show-inheritance: + airbyte\_cdk.sources.declarative.extractors.http\_selector module ----------------------------------------------------------------- @@ -10,10 +18,10 @@ airbyte\_cdk.sources.declarative.extractors.http\_selector module :undoc-members: :show-inheritance: -airbyte\_cdk.sources.declarative.extractors.jello module --------------------------------------------------------- +airbyte\_cdk.sources.declarative.extractors.record\_extractor module +-------------------------------------------------------------------- -.. automodule:: airbyte_cdk.sources.declarative.extractors.jello +.. automodule:: airbyte_cdk.sources.declarative.extractors.record_extractor :members: :undoc-members: :show-inheritance: diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst index 983bada8ee01..8fc8f41b57b7 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst @@ -26,14 +26,6 @@ airbyte\_cdk.sources.declarative.requesters.paginators.no\_pagination module :undoc-members: :show-inheritance: -airbyte\_cdk.sources.declarative.requesters.paginators.pagination\_strategy module ----------------------------------------------------------------------------------- - -.. automodule:: airbyte_cdk.sources.declarative.requesters.paginators.pagination_strategy - :members: - :undoc-members: - :show-inheritance: - airbyte\_cdk.sources.declarative.requesters.paginators.paginator module ----------------------------------------------------------------------- diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.strategies.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.strategies.rst index 98c5c0218225..86f929120e33 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.strategies.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.strategies.rst @@ -26,6 +26,14 @@ airbyte\_cdk.sources.declarative.requesters.paginators.strategies.page\_incremen :undoc-members: :show-inheritance: +airbyte\_cdk.sources.declarative.requesters.paginators.strategies.pagination\_strategy module +--------------------------------------------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.request_options.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.request_options.rst index 454e6c2af4bb..ff8eb074f6d1 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.request_options.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.request_options.rst @@ -2,6 +2,14 @@ Submodules ---------- +airbyte\_cdk.sources.declarative.requesters.request\_options.interpolated\_request\_input\_provider module +---------------------------------------------------------------------------------------------------------- + +.. automodule:: airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider + :members: + :undoc-members: + :show-inheritance: + airbyte\_cdk.sources.declarative.requesters.request\_options.interpolated\_request\_options\_provider module ------------------------------------------------------------------------------------------------------------ diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.rst index 7ed1b2691130..63a9dc689e6e 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.rst @@ -20,14 +20,6 @@ airbyte\_cdk.sources.declarative.requesters.http\_requester module :undoc-members: :show-inheritance: -airbyte\_cdk.sources.declarative.requesters.interpolated\_request\_input\_provider module ------------------------------------------------------------------------------------------ - -.. automodule:: airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider - :members: - :undoc-members: - :show-inheritance: - airbyte\_cdk.sources.declarative.requesters.request\_option module ------------------------------------------------------------------ diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.rst index 97b718996eba..0ffe29a4ae01 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.rst @@ -45,10 +45,10 @@ airbyte\_cdk.sources.declarative.declarative\_stream module :undoc-members: :show-inheritance: -airbyte\_cdk.sources.declarative.read\_exception module -------------------------------------------------------- +airbyte\_cdk.sources.declarative.exceptions module +-------------------------------------------------- -.. automodule:: airbyte_cdk.sources.declarative.read_exception +.. automodule:: airbyte_cdk.sources.declarative.exceptions :members: :undoc-members: :show-inheritance: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index ff3a17f01051..0b0114ea5cb2 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.72", + version="0.1.74", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -55,7 +55,6 @@ "vcrpy", "Deprecated~=1.2", "Jinja2~=3.1.2", - "jello~=1.5.2", ], python_requires=">=3.9", extras_require={ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py new file mode 100644 index 000000000000..e4d701fc3e7a --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime + +import pytest +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser + + +@pytest.mark.parametrize( + "test_name, input_date, date_format, expected_output_date", + [ + ( + "test_parse_date_iso", + "2021-01-01T00:00:00.000000+0000", + "%Y-%m-%dT%H:%M:%S.%f%z", + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + ), + ( + "test_parse_timestamp", + "1609459200", + "%s", + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + ), + ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), + ], +) +def test_parse_date(test_name, input_date, date_format, expected_output_date): + parser = DatetimeParser() + output_date = parser.parse(input_date, date_format, datetime.timezone.utc) + assert expected_output_date == output_date + + +@pytest.mark.parametrize( + "test_name, input_dt, datetimeformat, expected_output", + [ + ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), + ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), + ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), + ], +) +def test_format_datetime(test_name, input_dt, datetimeformat, expected_output): + parser = DatetimeParser() + output_date = parser.format(input_dt, datetimeformat) + assert expected_output == output_date diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py new file mode 100644 index 000000000000..ca94f57a41bd --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +import pytest +import requests +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor + +config = {"field": "record_array"} +options = {"options_field": "record_array"} + +decoder = JsonDecoder(options={}) + + +@pytest.mark.parametrize( + "test_name, field_pointer, body, expected_records", + [ + ("test_extract_from_array", ["data"], {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_extract_single_record", ["data"], {"data": {"id": 1}}, [{"id": 1}]), + ("test_extract_from_root_array", [], [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), + ("test_nested_field", ["data", "records"], {"data": {"records": [{"id": 1}, {"id": 2}]}}, [{"id": 1}, {"id": 2}]), + ("test_field_in_config", ["{{ config['field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_field_in_options", ["{{ options['options_field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_field_does_not_exist", ["record"], {"id": 1}, []), + ], +) +def test_dpath_extractor(test_name, field_pointer, body, expected_records): + extractor = DpathExtractor(field_pointer=field_pointer, config=config, decoder=decoder, options=options) + + response = create_response(body) + actual_records = extractor.extract_records(response) + + assert actual_records == expected_records + + +def create_response(body): + response = requests.Response() + response._content = json.dumps(body).encode("utf-8") + return response diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py deleted file mode 100644 index b9a1ec25322d..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import json - -import pytest -import requests -from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor - -config = {"field": "record_array"} -options = {"options_field": "record_array"} - -decoder = JsonDecoder(options={}) - - -@pytest.mark.parametrize( - "test_name, transform, body, expected_records", - [ - ("test_extract_from_array", "_.data", {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), - ("test_field_in_config", "_.{{ config['field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), - ("test_field_in_options", "_.{{ options['options_field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), - ("test_default", "_{{kwargs['field']}}", [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), - ( - "test_remove_fields_from_records", - "[{k:v for k,v in d.items() if k != 'value_to_remove'} for d in _.data]", - {"data": [{"id": 1, "value": "HELLO", "value_to_remove": "fail"}, {"id": 2, "value": "WORLD", "value_to_remove": "fail"}]}, - [{"id": 1, "value": "HELLO"}, {"id": 2, "value": "WORLD"}], - ), - ( - "test_add_fields_from_records", - "[{**{k:v for k,v in d.items()}, **{'project_id': d['project']['id']}} for d in _.data]", - {"data": [{"id": 1, "value": "HELLO", "project": {"id": 8}}, {"id": 2, "value": "WORLD", "project": {"id": 9}}]}, - [ - {"id": 1, "value": "HELLO", "project_id": 8, "project": {"id": 8}}, - {"id": 2, "value": "WORLD", "project_id": 9, "project": {"id": 9}}, - ], - ), - ], -) -def test(test_name, transform, body, expected_records): - extractor = JelloExtractor(transform=transform, config=config, decoder=decoder, options=options) - - response = create_response(body) - actual_records = extractor.extract_records(response) - - assert actual_records == expected_records - - -def create_response(body): - response = requests.Response() - response._content = json.dumps(body).encode("utf-8") - return response diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index fa2bbfdcd7ce..cc68b567b9db 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -7,45 +7,59 @@ import pytest import requests from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector @pytest.mark.parametrize( - "test_name, transform_template, filter_template, body, expected_records", + "test_name, field_pointer, filter_template, body, expected_records", [ ( "test_with_extractor_and_filter", - "_.data", + ["data"], "{{ record['created_at'] > stream_state['created_at'] }}", {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}]}, [{"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}], ), ( "test_no_record_filter_returns_all_records", - "_.data", + ["data"], None, {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}]}, [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}], ), ( "test_with_extractor_and_filter_with_options", - "_.{{ options['options_field'] }}", + ["{{ options['options_field'] }}"], "{{ record['created_at'] > options['created_at'] }}", {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}, {"id": 3, "created_at": "06-08-21"}]}, [{"id": 3, "created_at": "06-08-21"}], ), ( "test_read_single_record", - "_.data", + ["data"], None, {"data": {"id": 1, "created_at": "06-06-21"}}, [{"id": 1, "created_at": "06-06-21"}], ), + ( + "test_no_record", + ["data"], + None, + {"data": []}, + [], + ), + ( + "test_no_record_from_root", + [], + None, + [], + [], + ), ], ) -def test_record_filter(test_name, transform_template, filter_template, body, expected_records): +def test_record_filter(test_name, field_pointer, filter_template, body, expected_records): config = {"response_override": "stop_if_you_see_me"} options = {"options_field": "data", "created_at": "06-07-21"} stream_state = {"created_at": "06-06-21"} @@ -54,7 +68,7 @@ def test_record_filter(test_name, transform_template, filter_template, body, exp response = create_response(body) decoder = JsonDecoder(options={}) - extractor = JelloExtractor(transform=transform_template, decoder=decoder, config=config, options=options) + extractor = DpathExtractor(field_pointer=field_pointer, decoder=decoder, config=config, options=options) if filter_template is None: record_filter = None else: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 6639aa6ec807..064bef0f8786 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -8,7 +8,7 @@ import pytest import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.read_exception import ReadException +from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py index e2321ad607f2..ea83a06ad449 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py @@ -454,13 +454,13 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, "%Y-%m-%dT%H:%M:%S.%f%z", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), - ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ( - "test_parse_date_datetime", - datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), - "%Y%m%d", + "test_parse_timestamp", + "1609459200", + "%s", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), + ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ], ) def test_parse_date(test_name, input_date, date_format, expected_output_date): @@ -483,6 +483,7 @@ def test_parse_date(test_name, input_date, date_format, expected_output_date): [ ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), + ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), ], ) def test_format_datetime(test_name, input_dt, datetimeformat, expected_output): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 190c44846047..48d6373f31d8 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -8,7 +8,7 @@ from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation import InterpolatedString @@ -126,7 +126,7 @@ def test_create_substream_slicer(): path: "/v3" record_selector: extractor: - transform: "_" + field_pointer: [] stream_A: type: DeclarativeStream $options: @@ -242,7 +242,7 @@ def test_full_config(): decoder: class_name: "airbyte_cdk.sources.declarative.decoders.json_decoder.JsonDecoder" extractor: - class_name: airbyte_cdk.sources.declarative.extractors.jello.JelloExtractor + class_name: airbyte_cdk.sources.declarative.extractors.dpath_extractor.DpathExtractor decoder: "*ref(decoder)" selector: class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector @@ -298,7 +298,7 @@ def test_full_config(): primary_key: "id" extractor: $ref: "*ref(extractor)" - transform: "_.result" + field_pointer: ["result"] retriever: $ref: "*ref(retriever)" requester: @@ -321,7 +321,7 @@ def test_full_config(): assert stream_config["cursor_field"] == [] stream = factory.create_component(stream_config, input_config)() - assert isinstance(stream.retriever.record_selector.extractor, JelloExtractor) + assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor) assert type(stream) == DeclarativeStream assert stream.primary_key == "id" @@ -333,7 +333,7 @@ def test_full_config(): assert type(stream.retriever.record_selector) == RecordSelector assert type(stream.retriever.record_selector.extractor.decoder) == JsonDecoder - assert stream.retriever.record_selector.extractor.transform.eval(input_config) == "_.result" + assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["result"] assert type(stream.retriever.record_selector.record_filter) == RecordFilter assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}" assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json" @@ -349,8 +349,7 @@ def test_full_config(): def test_create_record_selector(): content = """ extractor: - type: JelloExtractor - transform: "_.result" + type: DpathExtractor selector: class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector record_filter: @@ -358,13 +357,13 @@ def test_create_record_selector(): condition: "{{ record['id'] > stream_state['id'] }}" extractor: $ref: "*ref(extractor)" - transform: "_.result" + field_pointer: ["result"] """ config = parser.parse(content) selector = factory.create_component(config["selector"], input_config)() assert isinstance(selector, RecordSelector) - assert isinstance(selector.extractor, JelloExtractor) - assert selector.extractor.transform.eval(input_config) == "_.result" + assert isinstance(selector.extractor, DpathExtractor) + assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == ["result"] assert isinstance(selector.record_filter, RecordFilter) @@ -455,7 +454,7 @@ def test_config_with_defaults(): page_size: 10 record_selector: extractor: - transform: "_.result" + field_pointer: ["result"] streams: - "*ref(lists_stream)" """ @@ -471,7 +470,7 @@ def test_config_with_defaults(): assert stream.retriever.requester.http_method == HttpMethod.GET assert stream.retriever.requester.authenticator._token.eval(input_config) == "verysecrettoken" - assert stream.retriever.record_selector.extractor.transform.eval(input_config) == "_.result" + assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["result"] assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.yaml" assert isinstance(stream.retriever.paginator, LimitPaginator) @@ -521,7 +520,7 @@ class TestCreateTransformations: page_size: 10 record_selector: extractor: - transform: "_.result" + field_pointer: ["result"] """ def test_no_transformations(self): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py new file mode 100644 index 000000000000..91e3f710cb09 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import os +import tempfile +import unittest + +from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + + +class TestYamlDeclarativeSource(unittest.TestCase): + def test_source_is_created_if_toplevel_fields_are_known(self): + content = """ + version: "version" + streams: "streams" + check: "check" + """ + temporary_file = TestFileContent(content) + YamlDeclarativeSource(temporary_file.filename) + + def test_source_is_not_created_if_toplevel_fields_are_unknown(self): + content = """ + version: "version" + streams: "streams" + check: "check" + not_a_valid_field: "error" + """ + temporary_file = TestFileContent(content) + with self.assertRaises(InvalidConnectorDefinitionException): + YamlDeclarativeSource(temporary_file.filename) + + +class TestFileContent: + def __init__(self, content): + self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) + + with self.file as f: + f.write(content) + + @property + def filename(self): + return self.file.name + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + os.unlink(self.filename) diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index f00605241179..692c396afdfe 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -332,17 +332,41 @@ public interface Configs { String getCheckJobMainContainerCpuLimit(); /** - * Define the job container's minimum RAM usage. Defaults to + * Define the check job container's minimum RAM usage. Defaults to * {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only. */ String getCheckJobMainContainerMemoryRequest(); /** - * Define the job container's maximum RAM usage. Defaults to + * Define the check job container's maximum RAM usage. Defaults to * {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only. */ String getCheckJobMainContainerMemoryLimit(); + /** + * Define the normalization job container's minimum CPU request. Defaults to + * {@link #getJobMainContainerCpuRequest()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerCpuRequest(); + + /** + * Define the normalization job container's maximum CPU usage. Defaults to + * {@link #getJobMainContainerCpuLimit()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerCpuLimit(); + + /** + * Define the normalization job container's minimum RAM usage. Defaults to + * {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerMemoryRequest(); + + /** + * Define the normalization job container's maximum RAM usage. Defaults to + * {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerMemoryLimit(); + /** * Define one or more Job pod tolerations. Tolerations are separated by ';'. Each toleration * contains k=v pairs mentioning some/all of key, effect, operator and value and separated by `,`. diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 76e6990230cd..9af20b09d7bd 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -153,6 +153,11 @@ public class EnvConfigs implements Configs { static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST"; static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST = "NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT = "NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT"; + // defaults private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache"; public static final String DEFAULT_JOB_KUBE_NAMESPACE = "default"; @@ -766,6 +771,26 @@ public String getCheckJobMainContainerMemoryLimit() { return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit()); } + @Override + public String getNormalizationJobMainContainerCpuRequest() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST, getJobMainContainerCpuRequest()); + } + + @Override + public String getNormalizationJobMainContainerCpuLimit() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT, getJobMainContainerCpuLimit()); + } + + @Override + public String getNormalizationJobMainContainerMemoryRequest() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST, getJobMainContainerMemoryRequest()); + } + + @Override + public String getNormalizationJobMainContainerMemoryLimit() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit()); + } + @Override public LogConfigs getLogConfigs() { return logConfigs; diff --git a/airbyte-config/config-models/src/main/resources/types/SyncStats.yaml b/airbyte-config/config-models/src/main/resources/types/SyncStats.yaml index 5c38885e6dc2..6616996cb33d 100644 --- a/airbyte-config/config-models/src/main/resources/types/SyncStats.yaml +++ b/airbyte-config/config-models/src/main/resources/types/SyncStats.yaml @@ -7,13 +7,17 @@ type: object required: - recordsEmitted - bytesEmitted -additionalProperties: false +additionalProperties: true properties: recordsEmitted: type: integer bytesEmitted: type: integer - stateMessagesEmitted: # TODO make required once per-stream state messages are supported in V2 + sourceStateMessagesEmitted: + description: Number of State messages emitted by the Source Connector + type: integer + destinationStateMessagesEmitted: + description: Number of State messages emitted by the Destination Connector type: integer recordsCommitted: type: integer # if unset, committed records could not be computed diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 63f6fe788372..0fa910b2a198 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -250,7 +250,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.3.12 + dockerImageTag: 0.3.13 documentationUrl: https://docs.airbyte.io/integrations/destinations/s3 icon: s3.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 2ddd01880f89..ad360c84386f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3974,7 +3974,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.3.12" +- dockerImage: "airbyte/destination-s3:0.3.13" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e37555642c0b..26156ca6950c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -271,11 +271,23 @@ - name: File sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 dockerRepository: airbyte/source-file - dockerImageTag: 0.2.15 + dockerImageTag: 0.2.17 documentationUrl: https://docs.airbyte.io/integrations/sources/file icon: file.svg sourceType: file releaseStage: alpha +- name: Freshcaller + sourceDefinitionId: 8a5d48f6-03bb-4038-a942-a8d3f175cca3 + dockerRepository: airbyte/source-freshcaller + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/freshcaller +- name: Flexport + sourceDefinitionId: f95337f1-2ad1-4baf-922f-2ca9152de630 + dockerRepository: airbyte/source-flexport + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/flexport + sourceType: api + releaseStage: alpha - name: Freshdesk sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 dockerRepository: airbyte/source-freshdesk @@ -303,7 +315,7 @@ - name: GitHub sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e dockerRepository: airbyte/source-github - dockerImageTag: 0.2.44 + dockerImageTag: 0.2.45 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg sourceType: api @@ -383,7 +395,7 @@ - name: Greenhouse sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 dockerRepository: airbyte/source-greenhouse - dockerImageTag: 0.2.7 + dockerImageTag: 0.2.8 documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse icon: greenhouse.svg sourceType: api @@ -399,7 +411,7 @@ - name: Harvest sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 dockerRepository: airbyte/source-harvest - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/harvest icon: harvest.svg sourceType: api @@ -411,6 +423,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/hellobaton sourceType: api releaseStage: alpha +- name: Hubplanner + sourceDefinitionId: 8097ceb9-383f-42f6-9f92-d3fd4bcc7689 + dockerRepository: airbyte/source-hubplanner + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/hubplanner + sourceType: api + releaseStage: alpha - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot @@ -513,6 +532,14 @@ icon: linkedin.svg sourceType: api releaseStage: generally_available +- name: LinkedIn Pages + sourceDefinitionId: af54297c-e8f8-4d63-a00d-a94695acc9d3 + dockerRepository: airbyte/source-linkedin-pages + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-pages + icon: linkedin.svg + sourceType: api + releaseStage: alpha - name: Linnworks sourceDefinitionId: 7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e dockerRepository: airbyte/source-linnworks @@ -564,7 +591,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.13 + dockerImageTag: 0.4.15 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -612,7 +639,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.6.1 + dockerImageTag: 0.6.2 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -762,7 +789,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.2 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -802,7 +829,7 @@ - name: Recurly sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01 dockerRepository: airbyte/source-recurly - dockerImageTag: 0.4.0 + dockerImageTag: 0.4.1 documentationUrl: https://docs.airbyte.io/integrations/sources/recurly icon: recurly.svg sourceType: api @@ -849,7 +876,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 1.0.11 + dockerImageTag: 1.0.12 documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce icon: salesforce.svg sourceType: api @@ -865,7 +892,7 @@ - name: Sendgrid sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 dockerRepository: airbyte/source-sendgrid - dockerImageTag: 0.2.8 + dockerImageTag: 0.2.9 documentationUrl: https://docs.airbyte.io/integrations/sources/sendgrid icon: sendgrid.svg sourceType: api @@ -937,7 +964,7 @@ - name: Stripe sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de dockerRepository: airbyte/source-stripe - dockerImageTag: 0.1.35 + dockerImageTag: 0.1.36 documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api @@ -1088,7 +1115,7 @@ - sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 name: Sentry dockerRepository: airbyte/source-sentry - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/sentry icon: sentry.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d55013002ca3..b12665f70214 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2255,7 +2255,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-file:0.2.15" +- dockerImage: "airbyte/source-file:0.2.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/file" connectionSpecification: @@ -2487,6 +2487,78 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-freshcaller:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/freshcaller" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Freshcaller Spec" + type: "object" + required: + - "domain" + - "api_key" + - "start_date" + additionalProperties: true + properties: + domain: + type: "string" + title: "Domain for Freshcaller account" + description: "Used to construct Base URL for the Freshcaller APIs" + examples: + - "snaptravel" + api_key: + type: "string" + title: "API Key" + description: "Freshcaller API Key. See the docs for more information on how to obtain this key." + airbyte_secret: true + requests_per_minute: + title: "Requests per minute" + type: "integer" + description: "The number of requests per minute that this source allowed\ + \ to use. There is a rate limit of 50 requests per minute per app per\ + \ account." + start_date: + title: "Start Date" + description: "UTC date and time. Any data created after this date will be\ + \ replicated." + format: "date-time" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2022-01-01T12:00:00Z" + sync_lag_minutes: + title: "Lag in minutes for each sync" + type: "integer" + description: "Lag in minutes for each sync, i.e., at time T, data for the\ + \ time range [prev_sync_time, T-30] will be fetched" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-flexport:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/flexport" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Flexport Spec" + additionalProperties: true + type: "object" + required: + - "api_key" + - "start_date" + properties: + api_key: + order: 0 + type: "string" + title: "API Key" + airbyte_secret: true + start_date: + order: 1 + title: "Start Date" + type: "string" + format: "date-time" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-freshdesk:0.3.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk" @@ -2593,7 +2665,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.2.44" +- dockerImage: "airbyte/source-github:0.2.45" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/github" connectionSpecification: @@ -3484,7 +3556,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-greenhouse:0.2.7" +- dockerImage: "airbyte/source-greenhouse:0.2.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/greenhouse" connectionSpecification: @@ -3493,13 +3565,15 @@ type: "object" required: - "api_key" - additionalProperties: false + additionalProperties: true properties: api_key: + title: "API Key" type: "string" description: "Greenhouse API Key. See the docs for more information on how to generate this key." airbyte_secret: true + order: 0 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] @@ -3547,7 +3621,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-harvest:0.1.9" +- dockerImage: "airbyte/source-harvest:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/harvest" connectionSpecification: @@ -3703,6 +3777,25 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-hubplanner:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/hubplanner" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Hubplanner Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "Hubplanner API key. See https://github.com/hubplanner/API#authentication\ + \ for more details." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-hubspot:0.1.81" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" @@ -4699,6 +4792,84 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" +- dockerImage: "airbyte/source-linkedin-pages:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/linkedin-pages/" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Linkedin Pages Spec" + type: "object" + required: + - "org_id" + additionalProperties: true + properties: + org_id: + title: "Organization ID" + type: "integer" + airbyte_secret: true + description: "Specify the Organization ID" + examples: + - "123456789" + credentials: + title: "Authentication *" + type: "object" + oneOf: + - type: "object" + title: "OAuth2.0" + required: + - "client_id" + - "client_secret" + - "refresh_token" + properties: + auth_method: + type: "string" + const: "oAuth2.0" + client_id: + type: "string" + title: "Client ID" + description: "The client ID of the LinkedIn developer application." + airbyte_secret: true + client_secret: + type: "string" + title: "Client secret" + description: "The client secret of the LinkedIn developer application." + airbyte_secret: true + refresh_token: + type: "string" + title: "Refresh token" + description: "The token value generated using the LinkedIn Developers\ + \ OAuth Token Tools. See the docs to obtain yours." + airbyte_secret: true + - title: "Access token" + type: "object" + required: + - "access_token" + properties: + auth_method: + type: "string" + const: "access_token" + access_token: + type: "string" + title: "Access token" + description: "The token value generated using the LinkedIn Developers\ + \ OAuth Token Tools. See the docs to obtain yours." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "credentials" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" - dockerImage: "airbyte/source-linnworks:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/linnworks" @@ -4995,7 +5166,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.13" +- dockerImage: "airbyte/source-mssql:0.4.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -5770,7 +5941,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.6.1" +- dockerImage: "airbyte/source-mysql:0.6.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: @@ -7140,7 +7311,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.0" +- dockerImage: "airbyte/source-postgres:1.0.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -7703,7 +7874,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-recurly:0.4.0" +- dockerImage: "airbyte/source-recurly:0.4.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/recurly" connectionSpecification: @@ -8272,7 +8443,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:1.0.11" +- dockerImage: "airbyte/source-salesforce:1.0.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: @@ -8312,7 +8483,7 @@ title: "Refresh Token" description: "Enter your application's Salesforce Refresh Token used for Airbyte to access your Salesforce\ - \ account. " + \ account." type: "string" airbyte_secret: true order: 4 @@ -8475,7 +8646,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-sendgrid:0.2.8" +- dockerImage: "airbyte/source-sendgrid:0.2.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/sendgrid" connectionSpecification: @@ -8484,7 +8655,7 @@ type: "object" required: - "apikey" - additionalProperties: false + additionalProperties: true properties: apikey: title: "Sendgrid API key" @@ -9326,7 +9497,7 @@ type: "string" path_in_connector_config: - "client_secret" -- dockerImage: "airbyte/source-stripe:0.1.35" +- dockerImage: "airbyte/source-stripe:0.1.36" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/stripe" connectionSpecification: @@ -9370,6 +9541,23 @@ \ is frequently updated after creation. More info here" order: 3 + slice_range: + type: "integer" + title: "Data request time increment in days (Optional)" + default: 365 + minimum: 1 + examples: + - 1 + - 3 + - 10 + - 30 + - 180 + - 360 + description: "The time increment used by the connector when requesting data\ + \ from the Stripe API. The bigger the value is, the less requests will\ + \ be made and faster the sync will be. On the other hand, the more seldom\ + \ the state is persisted." + order: 4 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] @@ -10580,7 +10768,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-sentry:0.1.1" +- dockerImage: "airbyte/source-sentry:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/sentry" connectionSpecification: @@ -10591,7 +10779,7 @@ - "auth_token" - "organization" - "project" - additionalProperties: false + additionalProperties: true properties: auth_token: type: "string" diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index fb726c0402bd..cd0570bfc535 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -196,7 +196,6 @@ private static JobOrchestrator getJobOrchestrator(final Configs configs, final ProcessFactory processFactory, final String application, final FeatureFlags featureFlags) { - return switch (application) { case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, workerConfigs, processFactory, featureFlags); case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(configs, workerConfigs, processFactory); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java index 692d4ea050f1..6e82d77a8fc4 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java @@ -16,6 +16,10 @@ import java.time.format.DateTimeFormatter; import java.util.function.Function; +/** + * TODO : Replace all the DateTime related logic of this class with + * {@link io.airbyte.db.jdbc.DateTimeConverter} + */ public class DataTypeUtils { public static final String DATE_FORMAT_PATTERN = "yyyy-MM-dd'T'HH:mm:ss'Z'"; @@ -27,6 +31,7 @@ public class DataTypeUtils { public static final DateTimeFormatter TIMETZ_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSSXXX"); public static final DateTimeFormatter TIMESTAMPTZ_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX"); public static final DateTimeFormatter OFFSETDATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSS XXX"); + public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); // wrap SimpleDateFormat in a function because SimpleDateFormat is not threadsafe as a static final. public static DateFormat getDateFormat() { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 8d4560cf4335..eea2286e8f34 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -258,18 +258,19 @@ public String getFullyQualifiedTableNameWithQuoting(final Connection connection, return schemaName != null ? enquoteIdentifier(connection, schemaName) + "." + quotedTableName : quotedTableName; } - protected ObjectType getObject(ResultSet resultSet, int index, Class clazz) throws SQLException { + protected ObjectType getObject(final ResultSet resultSet, final int index, final Class clazz) throws SQLException { return resultSet.getObject(index, clazz); } - protected void putTimeWithTimezone(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { - OffsetTime timetz = getObject(resultSet, index, OffsetTime.class); + protected void putTimeWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + final OffsetTime timetz = getObject(resultSet, index, OffsetTime.class); node.put(columnName, timetz.format(TIMETZ_FORMATTER)); } - protected void putTimestampWithTimezone(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { - OffsetDateTime timestamptz = getObject(resultSet, index, OffsetDateTime.class); - LocalDate localDate = timestamptz.toLocalDate(); + protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) + throws SQLException { + final OffsetDateTime timestamptz = getObject(resultSet, index, OffsetDateTime.class); + final LocalDate localDate = timestamptz.toLocalDate(); node.put(columnName, resolveEra(localDate, timestamptz.format(TIMESTAMPTZ_FORMATTER))); } @@ -283,7 +284,7 @@ protected void putTimestampWithTimezone(ObjectNode node, String columnName, Resu * * You most likely would prefer to call one of the overloaded methods, which accept temporal types. */ - public static String resolveEra(boolean isBce, String value) { + public static String resolveEra(final boolean isBce, final String value) { String mangledValue = value; if (isBce) { if (mangledValue.startsWith("-")) { @@ -296,11 +297,11 @@ public static String resolveEra(boolean isBce, String value) { return mangledValue; } - public static boolean isBce(LocalDate date) { + public static boolean isBce(final LocalDate date) { return date.getEra().equals(IsoEra.BCE); } - public static String resolveEra(LocalDate date, String value) { + public static String resolveEra(final LocalDate date, final String value) { return resolveEra(isBce(date), value); } @@ -311,14 +312,14 @@ public static String resolveEra(LocalDate date, String value) { * This is technically kind of sketchy due to ancient timestamps being weird (leap years, etc.), but * my understanding is that {@link #ONE_CE} has the same weirdness, so it cancels out. */ - public static String resolveEra(Date date, String value) { + public static String resolveEra(final Date date, final String value) { return resolveEra(date.before(ONE_CE), value); } /** * See {@link #resolveEra(Date, String)} for explanation. */ - public static String resolveEra(Timestamp timestamp, String value) { + public static String resolveEra(final Timestamp timestamp, final String value) { return resolveEra(timestamp.before(ONE_CE), value); } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java new file mode 100644 index 000000000000..68e8208a4f07 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.jdbc; + +import static io.airbyte.db.DataTypeUtils.DATE_FORMATTER; +import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; +import static io.airbyte.db.DataTypeUtils.TIMESTAMP_FORMATTER; +import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; +import static io.airbyte.db.DataTypeUtils.TIME_FORMATTER; +import static io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations.resolveEra; +import static java.time.ZoneOffset.UTC; + +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.OffsetTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DateTimeConverter { + + private static final Logger LOGGER = LoggerFactory.getLogger(DateTimeConverter.class); + public static final DateTimeFormatter TIME_WITH_TIMEZONE_FORMATTER = DateTimeFormatter.ofPattern( + "HH:mm:ss[.][SSSSSSSSS][SSSSSSS][SSSSSS][SSSSS][SSSS][SSS][SS][S][''][XXX][XX][X]"); + + public static String convertToTimeWithTimezone(final Object time) { + if (time instanceof final java.time.OffsetTime timetz) { + return timetz.format(TIMETZ_FORMATTER); + } + final OffsetTime timetz = OffsetTime.parse(time.toString(), TIME_WITH_TIMEZONE_FORMATTER); + return timetz.format(TIMETZ_FORMATTER); + } + + public static String convertToTimestampWithTimezone(final Object timestamp) { + if (timestamp instanceof final Timestamp t) { + // In snapshot mode, debezium produces a java.sql.Timestamp object for the TIMESTAMPTZ type. + // Conceptually, a timestamp with timezone is an Instant. But t.toInstant() actually mangles the + // value for ancient dates, because leap years weren't applied consistently in ye olden days. + // Additionally, toInstant() (and toLocalDateTime()) actually lose the era indicator, so we can't + // rely on their getEra() methods. + // So we have special handling for this case, which sidesteps the toInstant conversion. + final ZonedDateTime timestamptz = t.toLocalDateTime().atZone(UTC); + final String value = timestamptz.format(TIMESTAMPTZ_FORMATTER); + return resolveEra(t, value); + } else if (timestamp instanceof final OffsetDateTime t) { + return resolveEra(t.toLocalDate(), t.format(TIMESTAMPTZ_FORMATTER)); + } else if (timestamp instanceof final ZonedDateTime timestamptz) { + return resolveEra(timestamptz.toLocalDate(), timestamptz.format(TIMESTAMPTZ_FORMATTER)); + } else { + // This case probably isn't strictly necessary, but I'm leaving it just in case there's some weird + // situation that I'm not aware of. + final Instant instant = Instant.parse(timestamp.toString()); + final OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(instant, UTC); + final ZonedDateTime timestamptz = ZonedDateTime.from(offsetDateTime); + final LocalDate localDate = timestamptz.toLocalDate(); + final String value = timestamptz.format(TIMESTAMPTZ_FORMATTER); + return resolveEra(localDate, value); + } + } + + /** + * See {@link #convertToTimestampWithTimezone(Object)} for explanation of the weird things happening + * here. + */ + public static String convertToTimestamp(final Object timestamp) { + if (timestamp instanceof final Timestamp t) { + // Snapshot mode + final LocalDateTime localDateTime = t.toLocalDateTime(); + final String value = localDateTime.format(TIMESTAMP_FORMATTER); + return resolveEra(t, value); + } else if (timestamp instanceof final Instant i) { + // Incremental mode + return resolveEra(i.atZone(UTC).toLocalDate(), i.atOffset(UTC).toLocalDateTime().format(TIMESTAMP_FORMATTER)); + } else { + final LocalDateTime localDateTime = LocalDateTime.parse(timestamp.toString()); + final LocalDate date = localDateTime.toLocalDate(); + final String value = localDateTime.format(TIMESTAMP_FORMATTER); + return resolveEra(date, value); + } + } + + /** + * See {@link #convertToTimestampWithTimezone(Object)} for explanation of the weird things happening + * here. + */ + public static String convertToDate(final Object date) { + if (date instanceof final Date d) { + // Snapshot mode + final LocalDate localDate = ((Date) date).toLocalDate(); + return resolveEra(d, localDate.format(DATE_FORMATTER)); + } else if (date instanceof LocalDate d) { + // Incremental mode + return resolveEra(d, d.format(DATE_FORMATTER)); + } else { + final LocalDate localDate = LocalDate.parse(date.toString()); + return resolveEra(localDate, localDate.format(DATE_FORMATTER)); + } + } + + public static String convertToTime(final Object time) { + if (time instanceof final Time sqlTime) { + return sqlTime.toLocalTime().format(TIME_FORMATTER); + } else if (time instanceof final LocalTime localTime) { + return localTime.format(TIME_FORMATTER); + } else if (time instanceof java.time.Duration) { + long value = ((Duration) time).toNanos(); + if (value >= 0 && value <= TimeUnit.DAYS.toNanos(1)) { + return LocalTime.ofNanoOfDay(value).format(TIME_FORMATTER); + } else { + final long updatedValue = 0 > value ? Math.abs(value) : TimeUnit.DAYS.toNanos(1); + LOGGER.debug("Time values must use number of milliseconds greater than 0 and less than 86400000000000 but its {}, converting to {} ", value, + updatedValue); + return LocalTime.ofNanoOfDay(updatedValue).format(TIME_FORMATTER); + } + } else { + return LocalTime.parse(time.toString()).format(TIME_FORMATTER); + } + } + +} diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 23656c080c42..edf20177631a 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.17 +LABEL io.airbyte.version=0.2.18 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index a3c1a77bbc4d..0522565c5e3c 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -177,7 +177,7 @@ def transform_postgres(config: Dict[str, Any]): ssl = config.get("ssl") if ssl: - ssl_mode = config.get("ssl_mode", "allow") + ssl_mode = config.get("ssl_mode", {"mode": "allow"}) dbt_config["sslmode"] = ssl_mode.get("mode") if ssl_mode["mode"] == "verify-ca": TransformConfig.create_file("ca.crt", ssl_mode["ca_certificate"]) diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DateTimeConverter.java b/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DateTimeConverter.java deleted file mode 100644 index 719eda0995fc..000000000000 --- a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DateTimeConverter.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMESTAMP_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIME_FORMATTER; -import static io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations.isBce; -import static io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations.resolveEra; - -import java.sql.Date; -import java.sql.Timestamp; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.OffsetTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; - -public class DateTimeConverter { - - public static final DateTimeFormatter TIME_WITH_TIMEZONE_FORMATTER = DateTimeFormatter.ofPattern( - "HH:mm:ss[.][SSSSSSSSS][SSSSSSS][SSSSSS][SSSSS][SSSS][SSS][SS][S][''][XXX][XX][X]"); - - public static String convertToTimeWithTimezone(Object time) { - OffsetTime timetz = OffsetTime.parse(time.toString(), TIME_WITH_TIMEZONE_FORMATTER); - return timetz.format(TIMETZ_FORMATTER); - } - - public static String convertToTimestampWithTimezone(Object timestamp) { - if (timestamp instanceof Timestamp t) { - // In snapshot mode, debezium produces a java.sql.Timestamp object for the TIMESTAMPTZ type. - // Conceptually, a timestamp with timezone is an Instant. But t.toInstant() actually mangles the - // value for ancient dates, because leap years weren't applied consistently in ye olden days. - // Additionally, toInstant() (and toLocalDateTime()) actually lose the era indicator, so we can't - // rely on their getEra() methods. - // So we have special handling for this case, which sidesteps the toInstant conversion. - ZonedDateTime timestamptz = t.toLocalDateTime().atZone(ZoneOffset.UTC); - String value = timestamptz.format(TIMESTAMPTZ_FORMATTER); - return resolveEra(t, value); - } else if (timestamp instanceof OffsetDateTime t) { - // In incremental mode, debezium emits java.time.OffsetDateTime objects. - // java.time classes have a year 0, but the standard AD/BC system does not. For example, - // "0001-01-01 BC" is represented as LocalDate("0000-01-01"). - // We just subtract one year to hack around this difference. - LocalDate localDate = t.toLocalDate(); - if (isBce(localDate)) { - t = t.minusYears(1); - } - return resolveEra(localDate, t.toString()); - } else { - // This case probably isn't strictly necessary, but I'm leaving it just in case there's some weird - // situation that I'm not aware of. - Instant instant = Instant.parse(timestamp.toString()); - OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(instant, ZoneOffset.UTC); - ZonedDateTime timestamptz = ZonedDateTime.from(offsetDateTime); - LocalDate localDate = timestamptz.toLocalDate(); - String value = timestamptz.format(TIMESTAMPTZ_FORMATTER); - return resolveEra(localDate, value); - } - } - - /** - * See {@link #convertToTimestampWithTimezone(Object)} for explanation of the weird things happening - * here. - */ - public static String convertToTimestamp(Object timestamp) { - if (timestamp instanceof Timestamp t) { - // Snapshot mode - LocalDateTime localDateTime = t.toLocalDateTime(); - String value = localDateTime.format(TIMESTAMP_FORMATTER); - return resolveEra(t, value); - } else if (timestamp instanceof Instant i) { - // Incremental mode - LocalDate localDate = i.atZone(ZoneOffset.UTC).toLocalDate(); - if (isBce(localDate)) { - // i.minus(1, ChronoUnit.YEARS) would be nice, but it throws an exception because you can't subtract - // YEARS from an Instant - i = i.atZone(ZoneOffset.UTC).minusYears(1).toInstant(); - } - return resolveEra(localDate, i.toString()); - } else { - LocalDateTime localDateTime = LocalDateTime.parse(timestamp.toString()); - final LocalDate date = localDateTime.toLocalDate(); - String value = localDateTime.format(TIMESTAMP_FORMATTER); - return resolveEra(date, value); - } - } - - /** - * See {@link #convertToTimestampWithTimezone(Object)} for explanation of the weird things happening - * here. - */ - public static Object convertToDate(Object date) { - if (date instanceof Date d) { - // Snapshot mode - LocalDate localDate = ((Date) date).toLocalDate(); - return resolveEra(d, localDate.toString()); - } else if (date instanceof LocalDate d) { - // Incremental mode - if (isBce(d)) { - d = d.minusYears(1); - } - return resolveEra(d, d.toString()); - } else { - LocalDate localDate = LocalDate.parse(date.toString()); - return resolveEra(localDate, localDate.toString()); - } - } - - public static String convertToTime(Object time) { - LocalTime localTime = LocalTime.parse(time.toString()); - return localTime.format(TIME_FORMATTER); - } - -} diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java b/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java index ab0a9e6cde16..e5499cb2fe4c 100644 --- a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java +++ b/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java @@ -23,6 +23,9 @@ private DebeziumConverterUtils() { throw new UnsupportedOperationException(); } + /** + * TODO : Replace usage of this method with {@link io.airbyte.db.jdbc.DateTimeConverter} + */ public static String convertDate(final Object input) { /** * While building this custom converter we were not sure what type debezium could return cause there diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java b/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java index e3b7889a2c84..c985b09cf420 100644 --- a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java +++ b/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.debezium.internals; +import io.airbyte.db.jdbc.DateTimeConverter; import io.debezium.spi.converter.CustomConverter; import io.debezium.spi.converter.RelationalColumn; import java.math.BigDecimal; @@ -21,7 +22,7 @@ public class PostgresConverter implements CustomConverter DateTimeConverter.convertToDate(x); case "TIME" -> DateTimeConverter.convertToTime(x); case "INTERVAL" -> convertInterval((PGInterval) x); - default -> DebeziumConverterUtils.convertDate(x); + default -> throw new IllegalArgumentException("Unknown field type " + fieldType.toUpperCase(Locale.ROOT)); }; }); } diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index 010ea7e9045d..e401fb627dc6 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.2.1 +Don't fail on updating `additionalProperties`: fix IndexError [#15532](https://github.com/airbytehq/airbyte/pull/15532/) + +## 0.2.0 +Finish backward compatibility syntactic tests implementation: check that cursor fields were not changed. [#15520](https://github.com/airbytehq/airbyte/pull/15520/) + +## 0.1.62 +Backward compatibility tests: add syntactic validation of catalogs [#15486](https://github.com/airbytehq/airbyte/pull/15486/) + ## 0.1.61 Add unit tests coverage computation [#15443](https://github.com/airbytehq/airbyte/pull/15443/). diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index fdffec159d68..a885306672eb 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.1.61 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/README.md b/airbyte-integrations/bases/source-acceptance-test/README.md index e5783d9fd47e..2a1b2d0f206c 100644 --- a/airbyte-integrations/bases/source-acceptance-test/README.md +++ b/airbyte-integrations/bases/source-acceptance-test/README.md @@ -48,7 +48,7 @@ These iterations are more conveniently achieved by remaining in the current dire * Existing test modules are defined in `./source_acceptance_test/tests` * `acceptance-test-config.yaml` structure is defined in `./source_acceptance_test/config.py` 6. Unit test your changes by adding tests to `./unit_tests` -7. Run the unit tests on SAT again: `python -m pytest unit_tests`, make sure the coverage did not decrease. +7. Run the unit tests on SAT again: `python -m pytest unit_tests`, make sure the coverage did not decrease. You can bypass slow tests by using the `slow` marker: `python -m pytest unit_tests -m "not slow"`. 8. Manually test the changes you made by running SAT on a specific connector. e.g. `python -m pytest -p source_acceptance_test.plugin --acceptance-test-config=../../connectors/source-pokeapi` 9. Make sure you updated `docs/connector-development/testing-connectors/source-acceptance-tests-reference.md` according to your changes 10. Bump the SAT version in `airbyte-integrations/bases/source-acceptance-test/Dockerfile` diff --git a/airbyte-integrations/bases/source-acceptance-test/pytest.ini b/airbyte-integrations/bases/source-acceptance-test/pytest.ini index c5ee5ea27018..2531c1f41463 100644 --- a/airbyte-integrations/bases/source-acceptance-test/pytest.ini +++ b/airbyte-integrations/bases/source-acceptance-test/pytest.ini @@ -6,4 +6,4 @@ testpaths = markers = default_timeout - backward_compatibility + slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py index dfcbc8ba33db..1e1e6f8ee4da 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py @@ -57,6 +57,9 @@ class Status(Enum): class DiscoveryTestConfig(BaseConfig): config_path: str = config_path timeout_seconds: int = timeout_seconds + backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field( + description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig() + ) class ExpectedRecordsConfig(BaseModel): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py index 301267c0329a..8c71b02db7fb 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py @@ -178,6 +178,12 @@ def cached_schemas_fixture() -> MutableMapping[str, AirbyteStream]: return {} +@pytest.fixture(name="previous_cached_schemas", scope="session") +def previous_cached_schemas_fixture() -> MutableMapping[str, AirbyteStream]: + """Simple cache for discovered catalog of previous connector: stream_name -> json_schema""" + return {} + + @pytest.fixture(name="discovered_catalog") def discovered_catalog_fixture(connector_config, docker_runner: ConnectorRunner, cached_schemas) -> MutableMapping[str, AirbyteStream]: """JSON schemas for each stream""" @@ -190,6 +196,19 @@ def discovered_catalog_fixture(connector_config, docker_runner: ConnectorRunner, return cached_schemas +@pytest.fixture(name="previous_discovered_catalog") +def previous_discovered_catalog_fixture( + connector_config, previous_connector_docker_runner: ConnectorRunner, previous_cached_schemas +) -> MutableMapping[str, AirbyteStream]: + """JSON schemas for each stream""" + if not previous_cached_schemas: + output = previous_connector_docker_runner.call_discover(config=connector_config) + catalogs = [message.catalog for message in output if message.type == Type.CATALOG] + for stream in catalogs[-1].streams: + previous_cached_schemas[stream.name] = stream + return previous_cached_schemas + + @pytest.fixture def detailed_logger() -> Logger: """ diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index 7e7abef51440..d53594855323 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -24,13 +24,12 @@ TraceType, Type, ) -from deepdiff import DeepDiff from docker.errors import ContainerError from jsonschema._utils import flatten from source_acceptance_test.base import BaseTest -from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig, SpecTestConfig +from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig, DiscoveryTestConfig, SpecTestConfig from source_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, make_hashable, verify_records_schema -from source_acceptance_test.utils.backward_compatibility import SpecDiffChecker, validate_previous_configs +from source_acceptance_test.utils.backward_compatibility import CatalogDiffChecker, SpecDiffChecker, validate_previous_configs from source_acceptance_test.utils.common import find_all_values_for_key_in_schema, find_keyword_schema from source_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure @@ -46,15 +45,6 @@ class TestSpec(BaseTest): spec_cache: ConnectorSpecification = None previous_spec_cache: ConnectorSpecification = None - @staticmethod - def compute_spec_diff(actual_connector_spec: ConnectorSpecification, previous_connector_spec: ConnectorSpecification): - return DeepDiff( - previous_connector_spec.dict()["connectionSpecification"], - actual_connector_spec.dict()["connectionSpecification"], - view="tree", - ignore_order=True, - ) - @pytest.fixture(name="skip_backward_compatibility_tests") def skip_backward_compatibility_tests_fixture(self, inputs: SpecTestConfig, previous_connector_docker_runner: ConnectorRunner) -> bool: if previous_connector_docker_runner is None: @@ -185,14 +175,10 @@ def test_backward_compatibility( previous_connector_spec: ConnectorSpecification, number_of_configs_to_generate: int = 100, ): - """Check if the current spec is backward_compatible: - 1. Perform multiple hardcoded syntactic checks with SpecDiffChecker. - 2. Validate fake generated previous configs against the actual connector specification with validate_previous_configs. - """ + """Check if the current spec is backward_compatible with the previous one""" assert isinstance(actual_connector_spec, ConnectorSpecification) and isinstance(previous_connector_spec, ConnectorSpecification) - spec_diff = self.compute_spec_diff(actual_connector_spec, previous_connector_spec) - checker = SpecDiffChecker(spec_diff) - checker.assert_spec_is_backward_compatible() + checker = SpecDiffChecker(previous=previous_connector_spec.dict(), current=actual_connector_spec.dict()) + checker.assert_is_backward_compatible() validate_previous_configs(previous_connector_spec, actual_connector_spec, number_of_configs_to_generate) def test_additional_properties_is_true(self, actual_connector_spec: ConnectorSpecification): @@ -235,6 +221,20 @@ def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runn @pytest.mark.default_timeout(30) class TestDiscovery(BaseTest): + @pytest.fixture(name="skip_backward_compatibility_tests") + def skip_backward_compatibility_tests_fixture( + self, inputs: DiscoveryTestConfig, previous_connector_docker_runner: ConnectorRunner + ) -> bool: + if previous_connector_docker_runner is None: + pytest.skip("The previous connector image could not be retrieved.") + + # Get the real connector version in case 'latest' is used in the config: + previous_connector_version = previous_connector_docker_runner._image.labels.get("io.airbyte.version") + + if previous_connector_version == inputs.backward_compatibility_tests_config.disable_for_version: + pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") + return False + def test_discover(self, connector_config, docker_runner: ConnectorRunner): """Verify that discover produce correct schema.""" output = docker_runner.call_discover(config=connector_config) @@ -307,6 +307,19 @@ def test_additional_properties_is_true(self, discovered_catalog: Mapping[str, An [additional_properties_value is True for additional_properties_value in additional_properties_values] ), "When set, additionalProperties field value must be true for backward compatibility." + @pytest.mark.default_timeout(60) + @pytest.mark.backward_compatibility + def test_backward_compatibility( + self, + skip_backward_compatibility_tests: bool, + discovered_catalog: MutableMapping[str, AirbyteStream], + previous_discovered_catalog: MutableMapping[str, AirbyteStream], + ): + """Check if the current catalog is backward_compatible with the previous one.""" + assert isinstance(discovered_catalog, MutableMapping) and isinstance(previous_discovered_catalog, MutableMapping) + checker = CatalogDiffChecker(previous_discovered_catalog, discovered_catalog) + checker.assert_is_backward_compatible() + def primary_keys_for_records(streams, records): streams_with_primary_key = [stream for stream in streams if stream.stream.source_defined_primary_key] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/backward_compatibility.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/backward_compatibility.py index c228dcdf3c31..84485c08bb77 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/backward_compatibility.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/backward_compatibility.py @@ -2,79 +2,77 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from abc import ABC, abstractmethod +from enum import Enum + import jsonschema from airbyte_cdk.models import ConnectorSpecification from deepdiff import DeepDiff -from hypothesis import given, settings +from hypothesis import HealthCheck, Verbosity, given, settings from hypothesis_jsonschema import from_schema from source_acceptance_test.utils import SecretDict -class NonBackwardCompatibleSpecError(Exception): - pass +class BackwardIncompatibilityContext(Enum): + SPEC = 1 + DISCOVER = 2 -class SpecDiffChecker: - """A class to perform multiple backward compatible checks on a spec diff""" +class NonBackwardCompatibleError(Exception): + def __init__(self, error_message: str, context: BackwardIncompatibilityContext) -> None: + self.error_message = error_message + self.context = context + super().__init__(error_message) - def __init__(self, diff: DeepDiff) -> None: - self._diff = diff + def __str__(self): + return f"{self.context} - {self.error_message}" - def assert_spec_is_backward_compatible(self): - self.check_if_declared_new_required_field() - self.check_if_added_a_new_required_property() - self.check_if_value_of_type_field_changed() - # self.check_if_new_type_was_added() We want to allow type expansion atm - self.check_if_type_of_type_field_changed() - self.check_if_field_was_made_not_nullable() - self.check_if_enum_was_narrowed() - self.check_if_declared_new_enum_field() - def _raise_error(self, message: str): - raise NonBackwardCompatibleSpecError(f"{message}: {self._diff.pretty()}") +class BaseDiffChecker(ABC): + def __init__(self, previous: dict, current: dict) -> None: + self._previous = previous + self._current = current + self.compute_diffs() - def check_if_declared_new_required_field(self): - """Check if the new spec declared a 'required' field.""" - added_required_fields = [ - addition for addition in self._diff.get("dictionary_item_added", []) if addition.path(output_format="list")[-1] == "required" - ] - if added_required_fields: - self._raise_error("The current spec declared a new 'required' field") + def _raise_error(self, message: str, diff: DeepDiff): + raise NonBackwardCompatibleError(f"{message}. Diff: {diff.pretty()}", self.context) - def check_if_added_a_new_required_property(self): - """Check if the new spec added a property to the 'required' list.""" - added_required_properties = [ - addition for addition in self._diff.get("iterable_item_added", []) if addition.up.path(output_format="list")[-1] == "required" - ] - if added_required_properties: - self._raise_error("A new property was added to 'required'") + @property + @abstractmethod + def context(self): # pragma: no cover + pass - def check_if_value_of_type_field_changed(self): - """Check if a type was changed""" - # Detect type value change in case type field is declared as a string (e.g "str" -> "int"): - type_values_changed = [change for change in self._diff.get("values_changed", []) if change.path(output_format="list")[-1] == "type"] + @abstractmethod + def compute_diffs(self): # pragma: no cover + pass - # Detect type value change in case type field is declared as a single item list (e.g ["str"] -> ["int"]): - type_values_changed_in_list = [ - change for change in self._diff.get("values_changed", []) if change.path(output_format="list")[-2] == "type" + @abstractmethod + def assert_is_backward_compatible(self): # pragma: no cover + pass + + def check_if_value_of_type_field_changed(self, diff: DeepDiff): + """Check if a type was changed on a property""" + # Detect type value change in case type field is declared as a string (e.g "str" -> "int"): + changes_on_property_type = [ + change for change in diff.get("values_changed", []) if {"properties", "type"}.issubset(change.path(output_format="list")) ] - if type_values_changed or type_values_changed_in_list: - self._raise_error("The current spec changed the value of a 'type' field") + if changes_on_property_type: + self._raise_error("The'type' field value was changed.", diff) - def check_if_new_type_was_added(self): + def check_if_new_type_was_added(self, diff: DeepDiff): # pragma: no cover """Detect type value added to type list if new type value is not None (e.g ["str"] -> ["str", "int"])""" new_values_in_type_list = [ change - for change in self._diff.get("iterable_item_added", []) + for change in diff.get("iterable_item_added", []) if change.path(output_format="list")[-2] == "type" if change.t2 != "null" ] if new_values_in_type_list: - self._raise_error("The current spec changed the value of a 'type' field") + self._raise_error("A new value was added to a 'type' field") - def check_if_type_of_type_field_changed(self): + def check_if_type_of_type_field_changed(self, diff: DeepDiff): """ - Detect the change of type of a type field + Detect the change of type of a type field on a property e.g: - "str" -> ["str"] VALID - "str" -> ["str", "null"] VALID @@ -84,55 +82,93 @@ def check_if_type_of_type_field_changed(self): - ["str"] -> "int" INVALID - ["str"] -> 1 INVALID """ - type_changes = [change for change in self._diff.get("type_changes", []) if change.path(output_format="list")[-1] == "type"] + type_changes = [ + change for change in diff.get("type_changes", []) if {"properties", "type"}.issubset(change.path(output_format="list")) + ] for change in type_changes: # We only accept change on the type field if the new type for this field is list or string # This might be something already guaranteed by JSON schema validation. if isinstance(change.t1, str): if not isinstance(change.t2, list): - self._raise_error("The current spec change a type field from string to an invalid value.") - if not 0 < len(change.t2) <= 2: - self._raise_error( - "The current spec change a type field from string to an invalid value. The type list length should not be empty and have a maximum of two items." - ) + self._raise_error("A 'type' field was changed from string to an invalid value.", diff) # If the new type field is a list we want to make sure it only has the original type (t1) and null: e.g. "str" -> ["str", "null"] # We want to raise an error otherwise. t2_not_null_types = [_type for _type in change.t2 if _type != "null"] if not (len(t2_not_null_types) == 1 and t2_not_null_types[0] == change.t1): - self._raise_error("The current spec change a type field to a list with multiple invalid values.") + self._raise_error("The 'type' field was changed to a list with multiple invalid values", diff) if isinstance(change.t1, list): if not isinstance(change.t2, str): - self._raise_error("The current spec change a type field from list to an invalid value.") + self._raise_error("The 'type' field was changed from a list to an invalid value", diff) if not (len(change.t1) == 1 and change.t2 == change.t1[0]): - self._raise_error("The current spec narrowed a field type.") + self._raise_error("An element was removed from the list of 'type'", diff) + + +class SpecDiffChecker(BaseDiffChecker): + """A class to perform backward compatibility checks on a connector specification diff""" - def check_if_field_was_made_not_nullable(self): + context = BackwardIncompatibilityContext.SPEC + + def compute_diffs(self): + self.connection_specification_diff = DeepDiff( + self._previous["connectionSpecification"], + self._current["connectionSpecification"], + view="tree", + ignore_order=True, + ) + + def assert_is_backward_compatible(self): + self.check_if_declared_new_required_field(self.connection_specification_diff) + self.check_if_added_a_new_required_property(self.connection_specification_diff) + self.check_if_value_of_type_field_changed(self.connection_specification_diff) + # self.check_if_new_type_was_added(self.connection_specification_diff) We want to allow type expansion atm + self.check_if_type_of_type_field_changed(self.connection_specification_diff) + self.check_if_field_was_made_not_nullable(self.connection_specification_diff) + self.check_if_enum_was_narrowed(self.connection_specification_diff) + self.check_if_declared_new_enum_field(self.connection_specification_diff) + + def check_if_declared_new_required_field(self, diff: DeepDiff): + """Check if the new spec declared a 'required' field.""" + added_required_fields = [ + addition for addition in diff.get("dictionary_item_added", []) if addition.path(output_format="list")[-1] == "required" + ] + if added_required_fields: + self._raise_error("A new 'required' field was declared.", diff) + + def check_if_added_a_new_required_property(self, diff: DeepDiff): + """Check if the new spec added a property to the 'required' list""" + added_required_properties = [ + addition for addition in diff.get("iterable_item_added", []) if addition.up.path(output_format="list")[-1] == "required" + ] + if added_required_properties: + self._raise_error("A new property was added to 'required'", diff) + + def check_if_field_was_made_not_nullable(self, diff: DeepDiff): """Detect when field was made not nullable but is still a list: e.g ["string", "null"] -> ["string"]""" removed_nullable = [ - change for change in self._diff.get("iterable_item_removed", []) if change.path(output_format="list")[-2] == "type" + change for change in diff.get("iterable_item_removed", []) if {"properties", "type"}.issubset(change.path(output_format="list")) ] if removed_nullable: - self._raise_error("The current spec narrowed a field type or made a field not nullable.") + self._raise_error("A field type was narrowed or made a field not nullable", diff) - def check_if_enum_was_narrowed(self): + def check_if_enum_was_narrowed(self, diff: DeepDiff): """Check if the list of values in a enum was shortened in a spec.""" enum_removals = [ enum_removal - for enum_removal in self._diff.get("iterable_item_removed", []) + for enum_removal in diff.get("iterable_item_removed", []) if enum_removal.up.path(output_format="list")[-1] == "enum" ] if enum_removals: - self._raise_error("The current spec narrowed an enum field.") + self._raise_error("An enum field was narrowed.", diff) - def check_if_declared_new_enum_field(self): + def check_if_declared_new_enum_field(self, diff: DeepDiff): """Check if an 'enum' field was added to the spec.""" enum_additions = [ enum_addition - for enum_addition in self._diff.get("dictionary_item_added", []) + for enum_addition in diff.get("dictionary_item_added", []) if enum_addition.path(output_format="list")[-1] == "enum" ] if enum_additions: - self._raise_error("An 'enum' field was declared on an existing property of the spec.") + self._raise_error("An 'enum' field was declared on an existing property", diff) def validate_previous_configs( @@ -143,13 +179,53 @@ def validate_previous_configs( 2. Validate a fake previous config against the actual connector specification json schema.""" @given(from_schema(previous_connector_spec.dict()["connectionSpecification"])) - @settings(max_examples=number_of_configs_to_generate) + @settings(max_examples=number_of_configs_to_generate, verbosity=Verbosity.quiet, suppress_health_check=(HealthCheck.too_slow,)) def check_fake_previous_config_against_actual_spec(fake_previous_config): - fake_previous_config = SecretDict(fake_previous_config) - filtered_fake_previous_config = {key: value for key, value in fake_previous_config.data.items() if not key.startswith("_")} - try: - jsonschema.validate(instance=filtered_fake_previous_config, schema=actual_connector_spec.connectionSpecification) - except jsonschema.exceptions.ValidationError as err: - raise NonBackwardCompatibleSpecError(err) + if isinstance(fake_previous_config, dict): # Looks like hypothesis-jsonschema not only generate dict objects... + fake_previous_config = SecretDict(fake_previous_config) + filtered_fake_previous_config = {key: value for key, value in fake_previous_config.data.items() if not key.startswith("_")} + try: + jsonschema.validate(instance=filtered_fake_previous_config, schema=actual_connector_spec.connectionSpecification) + except jsonschema.exceptions.ValidationError as err: + raise NonBackwardCompatibleError(err, BackwardIncompatibilityContext.SPEC) check_fake_previous_config_against_actual_spec() + + +class CatalogDiffChecker(BaseDiffChecker): + """A class to perform backward compatibility checks on a discoverd catalog diff""" + + context = BackwardIncompatibilityContext.DISCOVER + + def compute_diffs(self): + self.streams_json_schemas_diff = DeepDiff( + {stream_name: airbyte_stream.dict().pop("json_schema") for stream_name, airbyte_stream in self._previous.items()}, + {stream_name: airbyte_stream.dict().pop("json_schema") for stream_name, airbyte_stream in self._current.items()}, + view="tree", + ignore_order=True, + ) + self.streams_cursor_fields_diff = DeepDiff( + {stream_name: airbyte_stream.dict().pop("default_cursor_field") for stream_name, airbyte_stream in self._previous.items()}, + {stream_name: airbyte_stream.dict().pop("default_cursor_field") for stream_name, airbyte_stream in self._current.items()}, + view="tree", + ) + + def assert_is_backward_compatible(self): + self.check_if_stream_was_removed(self.streams_json_schemas_diff) + self.check_if_value_of_type_field_changed(self.streams_json_schemas_diff) + self.check_if_type_of_type_field_changed(self.streams_json_schemas_diff) + self.check_if_cursor_field_was_changed(self.streams_cursor_fields_diff) + + def check_if_stream_was_removed(self, diff: DeepDiff): + """Check if a stream was removed from the catalog.""" + removed_streams = [] + for removal in diff.get("dictionary_item_removed", []): + if removal.path() != "root" and removal.up.path() == "root": + removed_streams.append(removal.path(output_format="list")[0]) + if removed_streams: + self._raise_error(f"The following streams were removed: {','.join(removed_streams)}", diff) + + def check_if_cursor_field_was_changed(self, diff: DeepDiff): + """Check if a default cursor field value was changed.""" + if diff: + self._raise_error("The value of 'default_cursor_field' was changed", diff) diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py index 125db1490927..05a72e407d59 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py @@ -3,30 +3,33 @@ # from dataclasses import dataclass +from typing import MutableMapping, Union import pytest -from airbyte_cdk.models import ConnectorSpecification +from airbyte_cdk.models import AirbyteStream, ConnectorSpecification +from source_acceptance_test.tests.test_core import TestDiscovery as _TestDiscovery from source_acceptance_test.tests.test_core import TestSpec as _TestSpec -from source_acceptance_test.utils.backward_compatibility import NonBackwardCompatibleSpecError +from source_acceptance_test.utils.backward_compatibility import NonBackwardCompatibleError, validate_previous_configs from .conftest import does_not_raise @dataclass -class SpecTransition: +class Transition: """An helper class to improve readability of the test cases""" - previous_connector_specification: ConnectorSpecification - current_connector_specification: ConnectorSpecification + previous: Union[ConnectorSpecification, MutableMapping[str, AirbyteStream]] + current: Union[ConnectorSpecification, MutableMapping[str, AirbyteStream]] should_fail: bool name: str + is_valid_json_schema: bool = True def as_pytest_param(self): - return pytest.param(self.previous_connector_specification, self.current_connector_specification, self.should_fail, id=self.name) + return pytest.param(self.previous, self.current, self.should_fail, id=self.name) FAILING_SPEC_TRANSITIONS = [ - SpecTransition( + Transition( ConnectorSpecification(connectionSpecification={}), ConnectorSpecification( connectionSpecification={ @@ -36,7 +39,7 @@ def as_pytest_param(self): should_fail=True, name="Top level: declaring the required field should fail.", ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -58,7 +61,7 @@ def as_pytest_param(self): should_fail=True, name="Nested level: adding the required field should fail.", ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "required": ["a"], @@ -72,7 +75,7 @@ def as_pytest_param(self): name="Top level: adding a new required property should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -103,7 +106,7 @@ def as_pytest_param(self): name="Nested level: adding a new required property should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -123,7 +126,7 @@ def as_pytest_param(self): name="Nullable: Making a field not nullable should fail (not in a list).", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -143,7 +146,7 @@ def as_pytest_param(self): name="Nested level: Narrowing a field type should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -163,7 +166,7 @@ def as_pytest_param(self): name="Nullable field: Making a field not nullable should fail", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -183,7 +186,7 @@ def as_pytest_param(self): name="Changing a field type should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -203,7 +206,7 @@ def as_pytest_param(self): name="Changing a field type from a string to a list with a different type value should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -223,7 +226,7 @@ def as_pytest_param(self): name="Changing a field type should fail from a list to string with different value should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -243,7 +246,7 @@ def as_pytest_param(self): name="Changing a field type in list should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -263,7 +266,7 @@ def as_pytest_param(self): name="Making a field nullable and changing type should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -283,7 +286,7 @@ def as_pytest_param(self): name="Making a field nullable and changing type should fail (change list order).", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -303,7 +306,7 @@ def as_pytest_param(self): name="Nullable field: Changing a field type should fail", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -333,7 +336,7 @@ def as_pytest_param(self): name="Changing a field type in oneOf should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -363,7 +366,7 @@ def as_pytest_param(self): name="Narrowing a field type in oneOf should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -383,7 +386,7 @@ def as_pytest_param(self): name="Top level: Narrowing a field enum should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -403,7 +406,7 @@ def as_pytest_param(self): name="Nested level: Narrowing a field enum should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -423,7 +426,7 @@ def as_pytest_param(self): name="Top level: Declaring a field enum should fail.", should_fail=True, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -443,10 +446,52 @@ def as_pytest_param(self): name="Nested level: Declaring a field enum should fail.", should_fail=True, ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": {}}, + }, + } + ), + name="Changing a 'type' field from a string to something else than a list should fail.", + should_fail=True, + is_valid_json_schema=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": ["string"]}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": {}}, + }, + } + ), + name="Changing a 'type' field from a list to something else than a string should fail.", + should_fail=True, + is_valid_json_schema=False, + ), ] VALID_SPEC_TRANSITIONS = [ - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -468,7 +513,55 @@ def as_pytest_param(self): name="Not changing a spec should not fail", should_fail=False, ), - SpecTransition( + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "required": ["my_required_string"], + "additionalProperties": False, + "properties": { + "my_required_string": {"type": "string"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "required": ["my_required_string"], + "additionalProperties": True, + "properties": { + "my_required_string": {"type": "string"}, + }, + } + ), + name="Top level: Changing the value of additionalProperties should not fail", + should_fail=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": {"type": "object", "properties": {"my_property": {"type": ["integer"]}}}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_nested_object": { + "type": "object", + "additionalProperties": True, + "properties": {"my_property": {"type": ["integer"]}}, + }, + }, + } + ), + name="Nested level: Changing the value of additionalProperties should not fail", + should_fail=False, + ), + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -491,7 +584,7 @@ def as_pytest_param(self): name="Adding an optional field should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -518,7 +611,7 @@ def as_pytest_param(self): name="Adding an optional object with required properties should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -538,7 +631,7 @@ def as_pytest_param(self): name="No change should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -558,7 +651,7 @@ def as_pytest_param(self): name="Changing a field type from a list to a string with same value should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -578,7 +671,7 @@ def as_pytest_param(self): name="Changing a field type from a string to a list should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -598,7 +691,7 @@ def as_pytest_param(self): name="Adding a field type in list should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -618,7 +711,7 @@ def as_pytest_param(self): name="Making a field nullable should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -638,7 +731,7 @@ def as_pytest_param(self): name="Making a field nullable should not fail (change list order).", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -658,7 +751,7 @@ def as_pytest_param(self): name="Making a field nullable should not fail (from a list).", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -678,7 +771,7 @@ def as_pytest_param(self): name="Making a field nullable should not fail (from a list, changing order).", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -698,7 +791,7 @@ def as_pytest_param(self): name="Nullable field: Changing order should not fail", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -718,7 +811,7 @@ def as_pytest_param(self): name="Nested level: Expanding a field type should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -748,7 +841,7 @@ def as_pytest_param(self): name="Changing a order in oneOf should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -768,7 +861,7 @@ def as_pytest_param(self): name="Top level: Expanding a field enum should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -788,7 +881,7 @@ def as_pytest_param(self): name="Nested level: Expanding a field enum should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -806,7 +899,7 @@ def as_pytest_param(self): name="Top level: Adding a new optional field with enum should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -826,7 +919,7 @@ def as_pytest_param(self): name="Top level: Removing the field enum should not fail.", should_fail=False, ), - SpecTransition( + Transition( ConnectorSpecification( connectionSpecification={ "type": "object", @@ -853,13 +946,265 @@ def as_pytest_param(self): # Checking that all transitions in VALID_SPEC_TRANSITIONS have should_fail = False to prevent typos assert not all([transition.should_fail for transition in VALID_SPEC_TRANSITIONS]) - ALL_SPEC_TRANSITIONS_PARAMS = [transition.as_pytest_param() for transition in FAILING_SPEC_TRANSITIONS + VALID_SPEC_TRANSITIONS] @pytest.mark.parametrize("previous_connector_spec, actual_connector_spec, should_fail", ALL_SPEC_TRANSITIONS_PARAMS) -def test_backward_compatibility(previous_connector_spec, actual_connector_spec, should_fail): +def test_spec_backward_compatibility(previous_connector_spec, actual_connector_spec, should_fail): t = _TestSpec() - expectation = pytest.raises(NonBackwardCompatibleSpecError) if should_fail else does_not_raise() + expectation = pytest.raises(NonBackwardCompatibleError) if should_fail else does_not_raise() with expectation: t.test_backward_compatibility(False, actual_connector_spec, previous_connector_spec, 10) + + +VALID_JSON_SCHEMA_TRANSITIONS_PARAMS = [ + transition.as_pytest_param() for transition in FAILING_SPEC_TRANSITIONS + VALID_SPEC_TRANSITIONS if transition.is_valid_json_schema +] + + +@pytest.mark.slow +@pytest.mark.parametrize("previous_connector_spec, actual_connector_spec, should_fail", VALID_JSON_SCHEMA_TRANSITIONS_PARAMS) +def test_validate_previous_configs(previous_connector_spec, actual_connector_spec, should_fail): + expectation = pytest.raises(NonBackwardCompatibleError) if should_fail else does_not_raise() + with expectation: + validate_previous_configs(previous_connector_spec, actual_connector_spec, 200) + + +FAILING_CATALOG_TRANSITIONS = [ + Transition( + name="Removing a stream from a catalog should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ), + "other_test_stream": AirbyteStream.parse_obj( + { + "name": "other_test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ), + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + ), + Transition( + name="Changing a field type should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "integer"}}}}}, + } + ) + }, + ), + Transition( + name="Renaming a stream should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + current={ + "new_test_stream": AirbyteStream.parse_obj( + { + "name": "new_test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + ), + Transition( + name="Changing a cursor in a stream should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a"], + } + ), + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["b"], + } + ), + }, + ), + Transition( + name="Changing a cursor in a stream should fail (nested cursors).", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a"], + } + ), + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a", "b"], + } + ), + }, + ), + Transition( + name="Changing a cursor in a stream should fail (nested cursors removal).", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a", "b"], + } + ), + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a"], + } + ), + }, + ), +] + +VALID_CATALOG_TRANSITIONS = [ + Transition( + name="Making a field nullable should not fail.", + should_fail=False, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string", "null"]}}}}}, + } + ) + }, + ), + Transition( + name="Changing 'type' field to list should not fail.", + should_fail=False, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string"]}}}}}, + } + ) + }, + ), + Transition( + name="Removing a field should not fail.", + should_fail=False, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": { + "user": {"type": "object", "properties": {"username": {"type": "string"}, "email": {"type": "string"}}} + } + }, + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + } + ) + }, + ), + Transition( + name="Not changing a cursor in a stream should not fail.", + should_fail=False, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a"], + } + ), + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "default_cursor_field": ["a"], + } + ), + }, + ), +] + +# Checking that all transitions in FAILING_CATALOG_TRANSITIONS have should_fail == True to prevent typos +assert all([transition.should_fail for transition in FAILING_CATALOG_TRANSITIONS]) +# Checking that all transitions in VALID_CATALOG_TRANSITIONS have should_fail = False to prevent typos +assert not all([transition.should_fail for transition in VALID_CATALOG_TRANSITIONS]) + +ALL_CATALOG_TRANSITIONS_PARAMS = [transition.as_pytest_param() for transition in FAILING_CATALOG_TRANSITIONS + VALID_CATALOG_TRANSITIONS] + + +@pytest.mark.parametrize("previous_discovered_catalog, discovered_catalog, should_fail", ALL_CATALOG_TRANSITIONS_PARAMS) +def test_catalog_backward_compatibility(previous_discovered_catalog, discovered_catalog, should_fail): + t = _TestDiscovery() + expectation = pytest.raises(NonBackwardCompatibleError) if should_fail else does_not_raise() + with expectation: + t.test_backward_compatibility(False, discovered_catalog, previous_discovered_catalog) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java index 2da738d5c043..ecea6f96332e 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java @@ -110,15 +110,14 @@ public void setUpInternal() throws Exception { localRoot = Files.createTempDirectory(testDir, "output"); environment = new TestDestinationEnv(localRoot); workerConfigs = new WorkerConfigs(new EnvConfigs()); - - setupEnvironment(environment); - processFactory = new DockerProcessFactory( workerConfigs, workspaceRoot, workspaceRoot.toString(), localRoot.toString(), "host"); + + setupEnvironment(environment); } @AfterEach diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java index a804b2dc243b..7f5adf34bc2b 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java @@ -13,6 +13,7 @@ import io.airbyte.db.Database; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -22,6 +23,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import java.io.IOException; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -42,7 +44,7 @@ public abstract class AbstractSourceDatabaseTypeTest extends AbstractSourceConne private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSourceDatabaseTypeTest.class); - private final List testDataHolders = new ArrayList<>(); + protected final List testDataHolders = new ArrayList<>(); /** * The column name will be used for a PK column in the test tables. Override it if default name is @@ -179,7 +181,7 @@ private void setupDatabaseInternal() throws Exception { * * @return configured catalog */ - private ConfiguredAirbyteCatalog getConfiguredCatalog() { + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { return new ConfiguredAirbyteCatalog().withStreams( testDataHolders .stream() @@ -242,4 +244,30 @@ protected void printMarkdownTestTable() { LOGGER.info(getMarkdownTestTable()); } + protected ConfiguredAirbyteStream createDummyTableWithData(final Database database) throws SQLException { + database.query(ctx -> { + ctx.fetch("CREATE TABLE " + getNameSpace() + ".random_dummy_table(id INTEGER PRIMARY KEY, test_column VARCHAR(63));"); + ctx.fetch("INSERT INTO " + getNameSpace() + ".random_dummy_table VALUES (2, 'Random Data');"); + return null; + }); + + return new ConfiguredAirbyteStream().withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + "random_dummy_table", + getNameSpace(), + Field.of("id", JsonSchemaType.INTEGER), + Field.of("test_column", JsonSchemaType.STRING)) + .withSourceDefinedCursor(true) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))); + + } + + protected List extractStateMessages(final List messages) { + return messages.stream().filter(r -> r.getType() == Type.STATE).map(AirbyteMessage::getState) + .collect(Collectors.toList()); + } + } diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 3de730a7634a..85ff8df3058c 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -52,6 +52,7 @@ | Iterable | [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-iterable) | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-jira) | | LinkedIn Ads | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-ads) | +| LinkedIn Pages | [![source-linkedin-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linkedin-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linkedin-pages) | | Linnworks | [![source-linnworks](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-linnworks%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-linnworks) | | Lever Hiring | [![source-lever-hiring](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lever-hiring%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lever-hiring) | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-looker) | diff --git a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs index 49b54192a86e..e60fbe235ae6 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.72", + "airbyte-cdk~=0.1.73", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index 6f8433d72026..82b353d77c92 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -1,42 +1,44 @@ -schema_loader: - type: JsonSchema - file_path: "./source_{{snakeCase name}}/schemas/\{{ options['name'] }}.json" -selector: - type: RecordSelector - extractor: - type: JelloExtractor - transform: "_" -requester: - type: HttpRequester - name: "\{{ options['name'] }}" - http_method: "GET" - authenticator: - type: BearerAuthenticator - api_token: "\{{ config['api_key'] }}" -retriever: - type: SimpleRetriever - $options: - url_base: TODO "your_api_base_url" - name: "\{{ options['name'] }}" - primary_key: "\{{ options['primary_key'] }}" - record_selector: - $ref: "*ref(selector)" - paginator: - type: NoPagination -customers_stream: - type: DeclarativeStream - $options: - name: "customers" - primary_key: "id" +version: "0.1.0" + +definitions: schema_loader: - $ref: "*ref(schema_loader)" + type: JsonSchema + file_path: "./source_{{snakeCase name}}/schemas/\{{ options['name'] }}.json" + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_pointer: [] + requester: + type: HttpRequester + name: "\{{ options['name'] }}" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "\{{ config['api_key'] }}" retriever: - $ref: "*ref(retriever)" - requester: - $ref: "*ref(requester)" - path: TODO "your_endpoint_path" + type: SimpleRetriever + $options: + url_base: TODO "your_api_base_url" + name: "\{{ options['name'] }}" + primary_key: "\{{ options['primary_key'] }}" + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + streams: - - "*ref(customers_stream)" + - type: DeclarativeStream + $options: + name: "customers" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: TODO "your_endpoint_path" check: type: CheckStream stream_names: ["customers"] diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java index be9d14d28eab..3305d960cc8c 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java @@ -45,10 +45,10 @@ public class AzureBlobStorageConsumer extends FailureTrackingAirbyteMessageConsu private final Map streamNameAndNamespaceToWriters; public AzureBlobStorageConsumer( - final AzureBlobStorageDestinationConfig azureBlobStorageDestinationConfig, - final ConfiguredAirbyteCatalog configuredCatalog, - final AzureBlobStorageWriterFactory writerFactory, - final Consumer outputRecordCollector) { + final AzureBlobStorageDestinationConfig azureBlobStorageDestinationConfig, + final ConfiguredAirbyteCatalog configuredCatalog, + final AzureBlobStorageWriterFactory writerFactory, + final Consumer outputRecordCollector) { this.azureBlobStorageDestinationConfig = azureBlobStorageDestinationConfig; this.configuredCatalog = configuredCatalog; this.writerFactory = writerFactory; @@ -91,8 +91,8 @@ protected void startTracked() throws Exception { } private void createContainers(final SpecializedBlobClientBuilder specializedBlobClientBuilder, - final AppendBlobClient appendBlobClient, - final ConfiguredAirbyteStream configuredStream) { + final AppendBlobClient appendBlobClient, + final ConfiguredAirbyteStream configuredStream) { // create container if absent (aka SQl Schema) final BlobContainerClient containerClient = appendBlobClient.getContainerClient(); if (!containerClient.exists()) { @@ -101,7 +101,7 @@ private void createContainers(final SpecializedBlobClientBuilder specializedBlob if (DestinationSyncMode.OVERWRITE.equals(configuredStream.getDestinationSyncMode())) { LOGGER.info("Sync mode is selected to OVERRIDE mode. New container will be automatically" + " created or all data would be overridden (if any) for stream:" + configuredStream - .getStream().getName()); + .getStream().getName()); var blobItemList = StreamSupport.stream(containerClient.listBlobs().spliterator(), false) .collect(Collectors.toList()); blobItemList.forEach(blob -> { diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java index bd3c0972bcda..5e0a5f2482b7 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.azure_blob_storage; import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; @@ -9,20 +13,20 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @DisplayName("AzureBlobRecordConsumer") @ExtendWith(MockitoExtension.class) public class AzureBlobRecordConsumerTest extends PerStreamStateMessageTest { + @Mock private Consumer outputRecordCollector; private AzureBlobStorageConsumer consumer; @Mock - private AzureBlobStorageDestinationConfig azureBlobStorageDestinationConfig; + private AzureBlobStorageDestinationConfig azureBlobStorageDestinationConfig; @Mock private ConfiguredAirbyteCatalog configuredCatalog; @@ -44,4 +48,5 @@ protected Consumer getMockedConsumer() { protected FailureTrackingAirbyteMessageConsumer getMessageConsumer() { return consumer; } + } diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index d00a1aa7085e..d0a8e9d0a05b 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -15,6 +15,31 @@ WORKDIR /airbyte ENV APPLICATION destination-s3 COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.3.12 +RUN /bin/bash -c 'set -e && \ + ARCH=`uname -m` && \ + if [ "$ARCH" == "x86_64" ] || [ "$ARCH" = "amd64" ]; then \ + echo "$ARCH" && \ + apt-get update; \ + apt-get install lzop liblzo2-2 liblzo2-dev -y; \ + elif [ "$ARCH" == "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ + echo "$ARCH" && \ + apt-get update; \ + apt-get install lzop liblzo2-2 liblzo2-dev wget curl unzip zip build-essential maven git -y; \ + wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz -P /tmp; \ + cd /tmp && tar xvfz lzo-2.10.tar.gz; \ + cd /tmp/lzo-2.10/ && ./configure --enable-shared --prefix /usr/local/lzo-2.10; \ + cd /tmp/lzo-2.10/ && make; \ + cd /tmp/lzo-2.10/ && make install; \ + git clone https://github.com/twitter/hadoop-lzo.git /usr/lib/hadoop/lib/hadoop-lzo/; \ + curl -s "https://get.sdkman.io" | bash; \ + source /root/.sdkman/bin/sdkman-init.sh; \ + sdk install java 8.0.342-librca; \ + sdk use java 8.0.342-librca; \ + cd /usr/lib/hadoop/lib/hadoop-lzo/ && C_INCLUDE_PATH=/usr/local/lzo-2.10/include LIBRARY_PATH=/usr/local/lzo-2.10/lib mvn clean package; \ + find /usr/lib/hadoop/lib/hadoop-lzo/ -name '*libgplcompression*' -exec cp {} /usr/lib/ \; ;\ + else \ + echo "unknown arch" ;\ + fi' + +LABEL io.airbyte.version=0.3.13 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index b99d787bd6bd..6ab077c83676 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -32,7 +32,7 @@ dependencies { } implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.0.1') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - + implementation group: 'com.hadoop.gplcompression', name: 'hadoop-lzo', version: '0.4.20' testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation "org.mockito:mockito-inline:4.1.0" diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/JavaProcessRunner.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/JavaProcessRunner.java new file mode 100644 index 000000000000..d14a18738858 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/JavaProcessRunner.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.s3.util; + +import io.airbyte.commons.io.LineGobbler; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JavaProcessRunner { + + private static final Logger LOGGER = LoggerFactory.getLogger(JavaProcessRunner.class); + + public static void runProcess(final String path, final Runtime run, final String... commands) throws IOException, InterruptedException { + LOGGER.info("Running process: " + Arrays.asList(commands)); + final Process pr = path.equals(System.getProperty("user.dir")) ? run.exec(commands) : run.exec(commands, null, new File(path)); + LineGobbler.gobble(pr.getErrorStream(), LOGGER::error); + LineGobbler.gobble(pr.getInputStream(), LOGGER::info); + if (!pr.waitFor(10, TimeUnit.MINUTES)) { + pr.destroy(); + throw new RuntimeException("Timeout while executing: " + Arrays.toString(commands)); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java index b94c007f816a..8cb1a6f7d91e 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3.parquet; +import static io.airbyte.integrations.destination.s3.util.JavaProcessRunner.runProcess; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -77,6 +78,51 @@ public void testCompressedParquetWriter() throws Exception { runTest(195L, 215L, config, getExpectedString()); } + private static String resolveArchitecture() { + return System.getProperty("os.name").replace(' ', '_') + "-" + System.getProperty("os.arch") + "-" + System.getProperty("sun.arch.data.model"); + } + + @Test + public void testLzoCompressedParquet() throws Exception { + final String currentDir = System.getProperty("user.dir"); + Runtime runtime = Runtime.getRuntime(); + final String architecture = resolveArchitecture(); + if (architecture.equals("Linux-amd64-64") || architecture.equals("Linux-x86_64-64")) { + runProcess(currentDir, runtime, "/bin/sh", "-c", "apt-get update"); + runProcess(currentDir, runtime, "/bin/sh", "-c", "apt-get install lzop liblzo2-2 liblzo2-dev -y"); + runLzoParquetTest(); + } else if (architecture.equals("Linux-aarch64-64") || architecture.equals("Linux-arm64-64")) { + runProcess(currentDir, runtime, "/bin/sh", "-c", "apt-get update"); + runProcess(currentDir, runtime, "/bin/sh", "-c", "apt-get install lzop liblzo2-2 liblzo2-dev " + + "wget curl unzip zip build-essential maven git -y"); + runProcess(currentDir, runtime, "/bin/sh", "-c", "wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz -P /usr/local/tmp"); + runProcess("/usr/local/tmp/", runtime, "/bin/sh", "-c", "tar xvfz lzo-2.10.tar.gz"); + runProcess("/usr/local/tmp/lzo-2.10/", runtime, "/bin/sh", "-c", "./configure --enable-shared --prefix /usr/local/lzo-2.10"); + runProcess("/usr/local/tmp/lzo-2.10/", runtime, "/bin/sh", "-c", "make && make install"); + runProcess(currentDir, runtime, "/bin/sh", "-c", "git clone https://github.com/twitter/hadoop-lzo.git /usr/lib/hadoop/lib/hadoop-lzo/"); + runProcess(currentDir, runtime, "/bin/sh", "-c", "curl -s https://get.sdkman.io | bash"); + runProcess(currentDir, runtime, "/bin/bash", "-c", "source /root/.sdkman/bin/sdkman-init.sh;" + + " sdk install java 8.0.342-librca;" + + " sdk use java 8.0.342-librca;" + + " cd /usr/lib/hadoop/lib/hadoop-lzo/ " + + "&& C_INCLUDE_PATH=/usr/local/lzo-2.10/include " + + "LIBRARY_PATH=/usr/local/lzo-2.10/lib mvn clean package"); + runProcess(currentDir, runtime, "/bin/sh", "-c", + "find /usr/lib/hadoop/lib/hadoop-lzo/ -name '*libgplcompression*' -exec cp {} /usr/lib/ \\;"); + runLzoParquetTest(); + } + } + + private void runLzoParquetTest() throws Exception { + final S3DestinationConfig config = S3DestinationConfig.getS3DestinationConfig(Jsons.jsonNode(Map.of( + "format", Map.of( + "format_type", "parquet", + "compression_codec", "LZO"), + "s3_bucket_name", "test", + "s3_bucket_region", "us-east-2"))); + runTest(195L, 215L, config, getExpectedString()); + } + private static String getExpectedString() { return "{\"_airbyte_ab_id\": \"\", \"_airbyte_emitted_at\": \"\", " + "\"field1\": 10000.0, \"another_field\": true, " diff --git a/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java b/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java index 0686797bce14..4e666bc46628 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.scylla; import com.fasterxml.jackson.databind.JsonNode; @@ -17,6 +21,7 @@ @DisplayName("ScyllaRecordConsumer") @ExtendWith(MockitoExtension.class) public class ScyllaRecordConsumerTest extends PerStreamStateMessageTest { + private static ScyllaContainer scyllaContainer; @Mock @@ -68,4 +73,5 @@ public ScyllaContainer() { } } + } diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index dcb20379a8b8..478c5dd6763b 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -1,4 +1,6 @@ -FROM airbyte/source-file:0.2.16 + +FROM airbyte/source-file:0.2.18 + WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure @@ -9,5 +11,6 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.16 + +LABEL io.airbyte.version=0.2.18 LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py index db36e11d2053..52fcff45f431 100644 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py +++ b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py @@ -36,11 +36,11 @@ class URLFileSecure(ParentURLFile): This connector shouldn't work with local files. """ - def __init__(self, url: str, provider: dict): + def __init__(self, url: str, provider: dict, binary=None, encoding=None): storage_name = provider["storage"].lower() if url.startswith("file://") or storage_name == LOCAL_STORAGE_NAME: raise RuntimeError("the local file storage is not supported by this connector.") - super().__init__(url, provider) + super().__init__(url, provider, binary, encoding) class SourceFileSecure(ParentSourceFile): diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index 9cbf49648e62..34fc01910d84 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -17,5 +17,6 @@ COPY source_file ./source_file ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.16 + +LABEL io.airbyte.version=0.2.18 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test_utf16.csv b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test_utf16.csv new file mode 100644 index 000000000000..e786a4a6567a Binary files /dev/null and b/airbyte-integrations/connectors/source-file/integration_tests/sample_files/test_utf16.csv differ diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 352340b965f0..8ec1aba3f4b7 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -13,6 +13,7 @@ "pandas==1.4.3", "paramiko==2.11.0", "s3fs==2022.7.1", + "boto3==1.21.21", "smart-open[all]==6.0.0", "lxml==4.9.1", "html5lib==1.1", @@ -23,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["boto3==1.21.21", "pytest==7.1.2", "pytest-docker==1.0.0", "pytest-mock~=3.8.2"] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-docker==1.0.0", "pytest-mock~=3.6.1"] setup( name="source_file", diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index 144e8fcf861c..9fb7ffa8c564 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -4,6 +4,7 @@ import json +import tempfile import traceback from os import environ from typing import Iterable @@ -54,10 +55,14 @@ class URLFile: ``` """ - def __init__(self, url: str, provider: dict): + def __init__(self, url: str, provider: dict, binary=None, encoding=None): self._url = url self._provider = provider self._file = None + self.args = { + "mode": "rb" if binary else "r", + "encoding": encoding, + } def __enter__(self): return self._file @@ -74,29 +79,28 @@ def close(self): self._file.close() self._file = None - def open(self, binary=False): + def open(self): self.close() try: - self._file = self._open(binary=binary) + self._file = self._open() except google.api_core.exceptions.NotFound as err: raise FileNotFoundError(self.url) from err return self - def _open(self, binary): - mode = "rb" if binary else "r" + def _open(self): storage = self.storage_scheme url = self.url if storage == "gs://": - return self._open_gcs_url(binary=binary) + return self._open_gcs_url() elif storage == "s3://": - return self._open_aws_url(binary=binary) + return self._open_aws_url() elif storage == "azure://": - return self._open_azblob_url(binary=binary) + return self._open_azblob_url() elif storage == "webhdfs://": host = self._provider["host"] port = self._provider["port"] - return smart_open.open(f"webhdfs://{host}:{port}/{url}", mode=mode) + return smart_open.open(f"webhdfs://{host}:{port}/{url}", **self.args) elif storage in ("ssh://", "scp://", "sftp://"): user = self._provider["user"] host = self._provider["host"] @@ -114,19 +118,15 @@ def _open(self, binary): uri = f"{storage}{user}:{password}@{host}:{port}/{url}" else: uri = f"{storage}{user}@{host}:{port}/{url}" - return smart_open.open(uri, transport_params=transport_params, mode=mode) + return smart_open.open(uri, transport_params=transport_params, **self.args) elif storage in ("https://", "http://"): transport_params = None if "user_agent" in self._provider and self._provider["user_agent"]: airbyte_version = environ.get("AIRBYTE_VERSION", "0.0") transport_params = {"headers": {"Accept-Encoding": "identity", "User-Agent": f"Airbyte/{airbyte_version}"}} logger.info(f"TransportParams: {transport_params}") - return smart_open.open( - self.full_url, - mode=mode, - transport_params=transport_params, - ) - return smart_open.open(self.full_url, mode=mode) + return smart_open.open(self.full_url, transport_params=transport_params, **self.args) + return smart_open.open(self.full_url, **self.args) @property def url(self) -> str: @@ -168,8 +168,7 @@ def storage_scheme(self) -> str: logger.error(f"Unknown Storage provider in: {self.full_url}") return "" - def _open_gcs_url(self, binary) -> object: - mode = "rb" if binary else "r" + def _open_gcs_url(self) -> object: service_account_json = self._provider.get("service_account_json") credentials = None if service_account_json: @@ -185,12 +184,11 @@ def _open_gcs_url(self, binary) -> object: client = GCSClient(credentials=credentials, project=credentials._project_id) else: client = GCSClient.create_anonymous_client() - file_to_close = smart_open.open(self.full_url, transport_params=dict(client=client), mode=mode) + file_to_close = smart_open.open(self.full_url, transport_params={"client": client}, **self.args) return file_to_close - def _open_aws_url(self, binary): - mode = "rb" if binary else "r" + def _open_aws_url(self): aws_access_key_id = self._provider.get("aws_access_key_id") aws_secret_access_key = self._provider.get("aws_secret_access_key") use_aws_account = aws_access_key_id and aws_secret_access_key @@ -198,15 +196,15 @@ def _open_aws_url(self, binary): if use_aws_account: aws_access_key_id = self._provider.get("aws_access_key_id", "") aws_secret_access_key = self._provider.get("aws_secret_access_key", "") - result = smart_open.open(f"{self.storage_scheme}{aws_access_key_id}:{aws_secret_access_key}@{self.url}", mode=mode) + url = f"{self.storage_scheme}{aws_access_key_id}:{aws_secret_access_key}@{self.url}" + result = smart_open.open(url, **self.args) else: config = botocore.client.Config(signature_version=botocore.UNSIGNED) params = {"client": boto3.client("s3", config=config)} - result = smart_open.open(self.full_url, transport_params=params, mode=mode) + result = smart_open.open(self.full_url, transport_params=params, **self.args) return result - def _open_azblob_url(self, binary): - mode = "rb" if binary else "r" + def _open_azblob_url(self): storage_account = self._provider.get("storage_account") storage_acc_url = f"https://{storage_account}.blob.core.windows.net" sas_token = self._provider.get("sas_token", None) @@ -220,14 +218,15 @@ def _open_azblob_url(self, binary): # assuming anonymous public read access given no credential client = BlobServiceClient(account_url=storage_acc_url) - result = smart_open.open(f"{self.storage_scheme}{self.url}", transport_params=dict(client=client), mode=mode) - return result + url = f"{self.storage_scheme}{self.url}" + return smart_open.open(url, transport_params=dict(client=client), **self.args) class Client: """Class that manages reading and parsing data from streams""" reader_class = URLFile + binary_formats = {"excel", "feather", "parquet", "orc", "pickle"} def __init__(self, dataset_name: str, url: str, provider: dict, format: str = None, reader_options: str = None): self._dataset_name = dataset_name @@ -243,6 +242,9 @@ def __init__(self, dataset_name: str, url: str, provider: dict, format: str = No logger.error(error_msg) raise ConfigurationError(error_msg) from err + self.binary_source = self._reader_format in self.binary_formats + self.encoding = self._reader_options.get("encoding") + @property def stream_name(self) -> str: if self._dataset_name: @@ -339,17 +341,12 @@ def dtype_to_json_type(dtype) -> str: @property def reader(self) -> reader_class: - return self.reader_class(url=self._url, provider=self._provider) - - @property - def binary_source(self): - binary_formats = {"excel", "feather", "parquet", "orc", "pickle"} - return self._reader_format in binary_formats + return self.reader_class(url=self._url, provider=self._provider, binary=self.binary_source, encoding=self.encoding) def read(self, fields: Iterable = None) -> Iterable[dict]: """Read data from the stream""" - with self.reader.open(binary=self.binary_source) as fp: - if self._reader_format == "json" or self._reader_format == "jsonl": + with self.reader.open() as fp: + if self._reader_format in ["json", "jsonl"]: yield from self.load_nested_json(fp) elif self._reader_format == "yaml": fields = set(fields) if fields else None @@ -359,10 +356,20 @@ def read(self, fields: Iterable = None) -> Iterable[dict]: yield from df[columns].to_dict(orient="records") else: fields = set(fields) if fields else None + if self.binary_source: + fp = self._cache_stream(fp) for df in self.load_dataframes(fp): columns = fields.intersection(set(df.columns)) if fields else df.columns df = df.where(pd.notnull(df), None) - yield from df[columns].to_dict(orient="records") + yield from df[list(columns)].to_dict(orient="records") + + def _cache_stream(self, fp): + """cache stream to file""" + fp_tmp = tempfile.TemporaryFile(mode="w+b") + fp_tmp.write(fp.read()) + fp_tmp.seek(0) + fp.close() + return fp_tmp def _stream_properties(self, fp): if self._reader_format == "yaml": @@ -379,8 +386,8 @@ def _stream_properties(self, fp): def streams(self) -> Iterable: """Discovers available streams""" # TODO handle discovery of directories of multiple files instead - with self.reader.open(binary=self.binary_source) as fp: - if self._reader_format == "json" or self._reader_format == "jsonl": + with self.reader.open() as fp: + if self._reader_format in ["json", "jsonl"]: json_schema = self.load_nested_json_schema(fp) else: json_schema = { diff --git a/airbyte-integrations/connectors/source-file/source_file/source.py b/airbyte-integrations/connectors/source-file/source_file/source.py index bac45796dfc0..ba8ae07334c8 100644 --- a/airbyte-integrations/connectors/source-file/source_file/source.py +++ b/airbyte-integrations/connectors/source-file/source_file/source.py @@ -83,7 +83,7 @@ def check(self, logger, config: Mapping) -> AirbyteConnectionStatus: client = self._get_client(config) logger.info(f"Checking access to {client.reader.full_url}...") try: - with client.reader.open(binary=client.binary_source): + with client.reader.open(): return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as err: reason = f"Failed to load {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py new file mode 100644 index 000000000000..8f6ed5277c62 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from pathlib import Path + +from source_file.source import SourceFile + +HERE = Path(__file__).parent.absolute() + + +def test_csv_with_utf16_encoding(): + + config_local_csv_utf16 = { + "dataset_name": "AAA", + "format": "csv", + "reader_options": '{"encoding":"utf_16"}', + "url": f"{HERE}/../integration_tests/sample_files/test_utf16.csv", + "provider": {"storage": "local"}, + } + expected_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "header1": {"type": ["string", "null"]}, + "header2": {"type": ["number", "null"]}, + "header3": {"type": ["number", "null"]}, + "header4": {"type": ["boolean", "null"]}, + }, + "type": "object", + } + + catalog = SourceFile().discover(logger=logging.getLogger("airbyte"), config=config_local_csv_utf16) + stream = next(iter(catalog.streams)) + assert stream.json_schema == expected_schema diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_companies.json b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_companies.json index f41b9382d95f..79ed8a11b715 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_companies.json +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_companies.json @@ -6,7 +6,6 @@ "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -35,7 +34,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -50,6 +48,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -59,7 +61,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -94,33 +95,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -144,7 +140,6 @@ "locations": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -164,7 +159,6 @@ "contacts": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_invoices.json b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_invoices.json index 1b1fc0277a14..aee951f8ead0 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_invoices.json +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_invoices.json @@ -6,7 +6,6 @@ "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -34,7 +33,6 @@ "total": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -54,7 +52,6 @@ "balance": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -99,7 +96,6 @@ "recipient": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -114,6 +110,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -123,7 +123,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -158,33 +157,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -207,7 +201,6 @@ "issuer": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -222,6 +215,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -231,7 +228,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -266,33 +262,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -318,7 +309,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -340,7 +330,6 @@ "amount": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -366,7 +355,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -394,7 +382,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -422,7 +409,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -432,7 +418,6 @@ "amount": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -449,18 +434,24 @@ } } }, - "category": { - "type": "string", - "description": "Category of the credit memo" - }, - "reason": { - "type": "string", - "description": "Why the credit was applied to the invoice" - }, "credited_at": { "type": "string", "description": "When the credit was applied. In ISO8601 UTC format with timezone denoted by Z.", "format": "date-time" + }, + "categories": { + "type": "array", + "description": "Category of the credit memo", + "items": { + "type": "string" + } + }, + "reasons": { + "type": "array", + "description": "Why the credit was applied to the invoice", + "items": { + "type": ["null", "string"] + } } } } @@ -475,7 +466,6 @@ "shipments": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_locations.json b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_locations.json index b56c516c84c2..7a175af7a969 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_locations.json +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_locations.json @@ -6,7 +6,6 @@ "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -24,7 +23,6 @@ "address": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -59,17 +57,17 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } }, @@ -80,7 +78,6 @@ "contacts": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -100,7 +97,6 @@ "company": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_products.json b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_products.json index bd224ac2b7f9..f0609472e0bd 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_products.json +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_products.json @@ -6,7 +6,6 @@ "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -47,7 +46,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -71,7 +69,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -99,7 +96,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -126,7 +122,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_shipments.json b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_shipments.json index ef376ad9d120..91e00658f97a 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_shipments.json +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/configured_catalog_shipments.json @@ -8,6 +8,10 @@ "type": "object", "additionalProperties": false, "properties": { + "metadata": { + "type": "object", + "description": "Set of custom key-values specific to the object. The keys are strings and values are arrays of strings. The set of valid keys is always the consignee's list of keys, even if call was made by a different party." + }, "_object": { "type": "string", "description": "String representing the object's type. Always /shipment for this object.", @@ -30,7 +34,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -55,7 +58,17 @@ }, "transportation_mode": { "type": "string", - "description": "Transportation mode of the main carriage of the shipment. This can be either Ocean or Air." + "description": "Transportation mode of the main carriage of the shipment.", + "enum": [ + "ocean", + "air", + "truck", + "rail", + "unknown_transportation", + "ocean_air", + "truck_intl", + "warehouse_storage" + ] }, "freight_type": { "type": "string", @@ -67,11 +80,6 @@ "door_to_port" ] }, - "updated_at": { - "type": "string", - "description": "Date when the shipment object was last updated. In ISO8601 UTC format with timezone denoted by Z.", - "format": "date-time" - }, "archived_at": { "type": ["null", "string"], "description": "Date when the shipment was archived, if applicable. In ISO8601 UTC format with timezone denoted by Z.", @@ -79,12 +87,25 @@ }, "incoterm": { "type": "string", - "description": "The Incoterm of your shipment. This can be EXW, FCA, FAS, FOB, CPT, CFR, CIF, CIP, DAT, DAP, DDP, or DPU." + "description": "The Incoterm of your shipment.", + "enum": [ + "EXW", + "FOB", + "FAS", + "FCA", + "CPT", + "CFR", + "CIF", + "CIP", + "DAT", + "DAP", + "DDP", + "DPU" + ] }, "calculated_weight": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -98,14 +119,13 @@ "unit": { "type": "string", "description": "Specifies the unit of measure for this quantity.", - "enum": ["kg", "lbs"] + "enum": ["kg", "lbs", "t"] } } }, "calculated_volume": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -123,29 +143,17 @@ } } }, - "estimated_departure_date": { - "type": ["null", "string"], - "description": "Estimated departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", - "format": "date-time" - }, - "actual_departure_date": { - "type": ["null", "string"], - "description": "Actual departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", - "format": "date-time" - }, - "target_delivery_date": { - "type": ["null", "string"], - "description": "Target date for when the shipment will be fully delivered. This date is set when the shipment''s booking is confirmed. This value may be different from estimated_delivered_in_full_date, which is updated when there is new information about the progress of a shipment. Date only.", - "format": "date" + "pieces": { + "type": ["null", "integer"], + "description": "Total number of pieces in the shipment." }, - "estimated_arrival_date": { + "it_number": { "type": ["null", "string"], - "description": "Estimated arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", - "format": "date-time" + "description": "Inbond Transit number used for US Customs" }, - "actual_arrival_date": { - "type": ["null", "string"], - "description": "Actual arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", + "created_date": { + "type": "string", + "description": "Date the user confirmed the Flexport quote for this Shipment.", "format": "date-time" }, "status": { @@ -162,39 +170,33 @@ ] }, "priority": { - "type": "string" - }, - "pieces": { - "type": ["null", "integer"], - "description": "Total number of pieces in the shipment." - }, - "it_number": { - "type": ["null", "string"], - "description": "Inbond Transit number used for US Customs" + "type": "string", + "description": "The level of attention Flexport should give to this shipment.", + "enum": ["standard", "high"] }, - "created_date": { + "updated_at": { "type": "string", - "description": "Date the user has confirmed Flexport quote and cargo is getting ready to ship.", + "description": "Date when the shipment object was last updated. In ISO8601 UTC format with timezone denoted by Z.", "format": "date-time" }, - "estimated_picked_up_in_full_date": { + "estimated_departure_date": { "type": ["null", "string"], - "description": "Estimated pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Estimated departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, - "actual_picked_up_in_full_date": { + "actual_departure_date": { "type": ["null", "string"], - "description": "Actual pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Actual departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, - "estimated_delivered_in_full_date": { + "estimated_arrival_date": { "type": ["null", "string"], - "description": "Estimated delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Estimated arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, - "actual_delivered_in_full_date": { + "actual_arrival_date": { "type": ["null", "string"], - "description": "Actual delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Actual arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, "cargo_ready_date": { @@ -210,13 +212,17 @@ "type": "boolean", "description": "Determines if a shipment is using Flexport services for a combination of import customs clearance or final leg delivery." }, - "wants_export_customs_service": { - "type": "boolean", - "description": "Determines if the shipment requires Flexport to provide export customs service." + "wants_commercial_invoice_transcription": { + "type": "boolean" }, - "wants_import_customs_service": { - "type": "boolean", - "description": "Determines if the shipment requires Flexport to provide import customs service." + "wants_flexport_insurance": { + "type": "boolean" + }, + "wants_pickup_service": { + "type": "boolean" + }, + "wants_trade_declaration_service": { + "type": ["null", "boolean"] }, "visibility_only": { "type": "boolean", @@ -226,17 +232,96 @@ "type": "boolean", "description": "Determines if Flexport is responsible for door delivery on a shipment." }, - "wants_flexport_insurance": { - "type": "boolean" + "estimated_picked_up_in_full_date": { + "type": ["null", "string"], + "description": "Estimated pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" }, - "wants_pickup_service": { - "type": "boolean" + "actual_picked_up_in_full_date": { + "type": ["null", "string"], + "description": "Actual pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" }, - "wants_commercial_invoice_transcription": { - "type": "boolean" + "target_delivery_date": { + "type": ["null", "string"], + "description": "Target date for when the shipment will be fully delivered. This date is set when the shipment''s booking is confirmed. This value may be different from estimated_delivered_in_full_date, which is updated when there is new information about the progress of a shipment. Date only.", + "format": "date" }, - "wants_trade_declaration_service": { - "type": ["null", "boolean"] + "estimated_delivered_in_full_date": { + "type": ["null", "string"], + "description": "Estimated delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" + }, + "actual_delivered_in_full_date": { + "type": ["null", "string"], + "description": "Actual delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" + }, + "wants_export_customs_service": { + "type": "boolean", + "description": "Determines if the shipment requires Flexport to provide export customs service." + }, + "wants_import_customs_service": { + "type": "boolean", + "description": "Determines if the shipment requires Flexport to provide import customs service." + }, + "ocean_shipment": { + "description": "Ocean-specific Shipment information. Available only if this is an ocean shipment. null otherwise.", + "oneOf": [ + { + "type": "null" + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_object": { + "type": "string", + "description": "String representing the object's type. Always /ocean/shipment for this object.", + "pattern": "^/ocean/shipment$" + }, + "is_lcl": { + "type": "boolean", + "description": "Flag that indicates whether the object is a LCL shipment." + }, + "house_bill_number": { + "type": "string", + "description": "House bill of lading number." + }, + "master_bill_number": { + "type": ["null", "string"], + "description": "Master bill of lading number." + }, + "carrier_booking_number": { + "type": ["null", "string"], + "description": "Ocean carrier booking reference number." + }, + "confirmed_space_released_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "containers": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "_object": { + "type": "string", + "description": "String representing the object's type. Always /api/refs/collection for this object.", + "pattern": "^/api/refs/collection$" + }, + "link": { + "type": "string", + "description": "API end point that points to a list of resources" + }, + "ref_type": { + "type": "string", + "description": "The _object value of each individual element of the list that link points to." + } + } + } + } + } + ] }, "air_shipment": { "description": "Air-specific Shipment information. Available only if this is an air shipment. null otherwise.", @@ -247,7 +332,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -265,7 +349,6 @@ "chargeable_weight": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -279,14 +362,13 @@ "unit": { "type": "string", "description": "Specifies the unit of measure for this quantity.", - "enum": ["kg", "lbs"] + "enum": ["kg", "lbs", "t"] } } }, "chargeable_volume": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -308,70 +390,9 @@ } ] }, - "ocean_shipment": { - "description": "Ocean-specific Shipment information. Available only if this is an ocean shipment. null otherwise.", - "oneOf": [ - { - "type": "null" - }, - { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": false, - "properties": { - "_object": { - "type": "string", - "description": "String representing the object's type. Always /ocean/shipment for this object.", - "pattern": "^/ocean/shipment$" - }, - "is_lcl": { - "type": "boolean", - "description": "Flag that indicates whether the object is a LCL shipment." - }, - "house_bill_number": { - "type": "string", - "description": "House bill of lading number." - }, - "master_bill_number": { - "type": ["null", "string"], - "description": "Master bill of lading number." - }, - "carrier_booking_number": { - "type": ["null", "string"], - "description": "Ocean carrier booking reference number." - }, - "confirmed_space_released_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "containers": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": false, - "properties": { - "_object": { - "type": "string", - "description": "String representing the object's type. Always /api/refs/collection for this object.", - "pattern": "^/api/refs/collection$" - }, - "link": { - "type": "string", - "description": "API end point that points to a list of resources" - }, - "ref_type": { - "type": "string", - "description": "The _object value of each individual element of the list that link points to." - } - } - } - } - } - ] - }, "dangerous_goods": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -392,7 +413,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -407,6 +427,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -416,7 +440,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -451,33 +474,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -504,7 +522,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -519,6 +536,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -528,7 +549,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -563,33 +583,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -616,7 +631,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -631,6 +645,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -640,7 +658,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -675,33 +692,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -728,7 +740,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -743,6 +754,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -752,7 +767,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -787,33 +801,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -840,7 +849,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -855,6 +863,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -864,7 +876,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -899,33 +910,28 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -952,7 +958,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -970,7 +975,6 @@ "total_weight": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -984,14 +988,13 @@ "unit": { "type": "string", "description": "Specifies the unit of measure for this quantity.", - "enum": ["kg", "lbs"] + "enum": ["kg", "lbs", "t"] } } }, "total_volume": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1016,7 +1019,6 @@ "product": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1057,7 +1059,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1081,7 +1082,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1109,7 +1109,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1136,7 +1135,6 @@ "items": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1168,7 +1166,6 @@ "legs": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1188,7 +1185,6 @@ "customs_entries": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1208,7 +1204,6 @@ "commercial_invoices": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1228,7 +1223,6 @@ "documents": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -1245,10 +1239,6 @@ } } }, - "metadata": { - "type": "object", - "description": "User defined metadata attached to the shipment." - }, "departure_date": { "$comment": "deprecated", "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/address.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/address.json index 2f76374bb28f..52a01d4d1dc3 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/address.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/address.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -36,17 +35,17 @@ "type": ["null", "string"], "description": "ZIP or postal code." }, - "unlocode": { - "type": ["null", "string"], - "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." - }, "timezone": { - "type": "string", + "type": ["null", "string"], "description": "Timezone for this address" }, "ref": { "type": ["null", "string"], "description": "Your reference for the address, as set in your network tab" + }, + "unlocode": { + "type": ["null", "string"], + "description": "If port, then UN/LOCODE (United Nations Code for Trade and Transport Locations)." } } } diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/air/shipment.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/air/shipment.json index 747e102fe390..9ddf5f73905b 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/air/shipment.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/air/shipment.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/collection.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/collection.json index 98296a6bfb9a..026450e08e9f 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/collection.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/collection.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/object.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/object.json index 3a2509deb2e2..dd80e51ba577 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/object.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/api/refs/object.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity.json index 2e31c55dbb14..2f84330047d4 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -16,6 +15,10 @@ "type": "string", "description": "Name of the company entity." }, + "ref": { + "type": "string", + "description": "Your reference for this company entity, as set in the Network tab." + }, "mailing_address": { "description": "Address of the company entity.", "oneOf": [ @@ -27,10 +30,6 @@ } ] }, - "ref": { - "type": "string", - "description": "Your reference for this company entity, as set in the Network tab." - }, "vat_numbers": { "type": "array", "description": "Array of VAT numbers of the company entity.", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity/vat_number.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity/vat_number.json index 804fb5733cd3..f787e43b1357 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity/vat_number.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/company_entity/vat_number.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/credit_memo.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/credit_memo.json index bd8cbd94dea6..9f2a051bed37 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/credit_memo.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/credit_memo.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -12,18 +11,24 @@ "description": "Amount of the credit", "$ref": "money.json" }, - "category": { - "type": "string", - "description": "Category of the credit memo" - }, - "reason": { - "type": "string", - "description": "Why the credit was applied to the invoice" - }, "credited_at": { "type": "string", "description": "When the credit was applied. In ISO8601 UTC format with timezone denoted by Z.", "format": "date-time" + }, + "categories": { + "type": "array", + "description": "Category of the credit memo", + "items": { + "type": "string" + } + }, + "reasons": { + "type": "array", + "description": "Why the credit was applied to the invoice", + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/hs_code.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/hs_code.json index 73349a339048..f96137267e33 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/hs_code.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/hs_code.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice.json index 84748ac13ed7..eafa29f1cf5e 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/quantity.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/quantity.json index 456cee19c2db..d93a87a171cb 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/quantity.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/quantity.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/rate.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/rate.json index 85104bce88be..4065655ec686 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/rate.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice/rate.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice_item.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice_item.json index 386942f47e31..30764295ac7d 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice_item.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/invoice_item.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/money.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/money.json index 993bdad67c75..0461a9a8a2fc 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/money.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/money.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/company.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/company.json index a68833d74649..8b76a486072f 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/company.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/company.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/location.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/location.json index bfa15aa9c96c..08c7345ae687 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/location.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/network/location.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/ocean/shipment.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/ocean/shipment.json index b96dc6d6d8b1..4088cff0662d 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/ocean/shipment.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/ocean/shipment.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product.json index 9e42d19beb2e..47c0effead9f 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/classification.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/classification.json index cfe9b2bfc79e..8da4d4f8b791 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/classification.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/classification.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/property.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/property.json index 904ce583e6b5..576e05b14d3f 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/property.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/property.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/supplier.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/supplier.json index e5f8ce599c9e..327c56fd9d6c 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/supplier.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/product/supplier.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/volume.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/volume.json index ef86a371afe5..64c05a8beab4 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/volume.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/volume.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/weight.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/weight.json index a9e876424bbb..acfe75b2e2f7 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/weight.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/quantity/weight.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", @@ -15,7 +14,7 @@ "unit": { "type": "string", "description": "Specifies the unit of measure for this quantity.", - "enum": ["kg", "lbs"] + "enum": ["kg", "lbs", "t"] } } } diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment.json index a59f2008eed9..2ad39411d161 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment.json @@ -3,6 +3,10 @@ "type": "object", "additionalProperties": false, "properties": { + "metadata": { + "type": "object", + "description": "Set of custom key-values specific to the object. The keys are strings and values are arrays of strings. The set of valid keys is always the consignee's list of keys, even if call was made by a different party." + }, "_object": { "type": "string", "description": "String representing the object's type. Always /shipment for this object.", @@ -29,18 +33,23 @@ }, "transportation_mode": { "type": "string", - "description": "Transportation mode of the main carriage of the shipment. This can be either Ocean or Air." + "description": "Transportation mode of the main carriage of the shipment.", + "enum": [ + "ocean", + "air", + "truck", + "rail", + "unknown_transportation", + "ocean_air", + "truck_intl", + "warehouse_storage" + ] }, "freight_type": { "type": "string", "description": "The type of freight service provided. One of:", "enum": ["port_to_door", "port_to_port", "door_to_door", "door_to_port"] }, - "updated_at": { - "type": "string", - "description": "Date when the shipment object was last updated. In ISO8601 UTC format with timezone denoted by Z.", - "format": "date-time" - }, "archived_at": { "type": ["null", "string"], "description": "Date when the shipment was archived, if applicable. In ISO8601 UTC format with timezone denoted by Z.", @@ -48,7 +57,21 @@ }, "incoterm": { "type": "string", - "description": "The Incoterm of your shipment. This can be EXW, FCA, FAS, FOB, CPT, CFR, CIF, CIP, DAT, DAP, DDP, or DPU." + "description": "The Incoterm of your shipment.", + "enum": [ + "EXW", + "FOB", + "FAS", + "FCA", + "CPT", + "CFR", + "CIF", + "CIP", + "DAT", + "DAP", + "DDP", + "DPU" + ] }, "calculated_weight": { "description": "Total weight (kg or lbs) of the shipment, calculated from individual pieces if package dimensions are known.", @@ -58,29 +81,17 @@ "description": "Total volume (cbm or cft) of the shipment, calculated from individual pieces if package dimensions are known.", "$ref": "quantity/volume.json" }, - "estimated_departure_date": { - "type": ["null", "string"], - "description": "Estimated departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", - "format": "date-time" - }, - "actual_departure_date": { - "type": ["null", "string"], - "description": "Actual departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", - "format": "date-time" - }, - "target_delivery_date": { - "type": ["null", "string"], - "description": "Target date for when the shipment will be fully delivered. This date is set when the shipment''s booking is confirmed. This value may be different from estimated_delivered_in_full_date, which is updated when there is new information about the progress of a shipment. Date only.", - "format": "date" + "pieces": { + "type": ["null", "integer"], + "description": "Total number of pieces in the shipment." }, - "estimated_arrival_date": { + "it_number": { "type": ["null", "string"], - "description": "Estimated arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", - "format": "date-time" + "description": "Inbond Transit number used for US Customs" }, - "actual_arrival_date": { - "type": ["null", "string"], - "description": "Actual arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", + "created_date": { + "type": "string", + "description": "Date the user confirmed the Flexport quote for this Shipment.", "format": "date-time" }, "status": { @@ -97,39 +108,33 @@ ] }, "priority": { - "type": "string" - }, - "pieces": { - "type": ["null", "integer"], - "description": "Total number of pieces in the shipment." - }, - "it_number": { - "type": ["null", "string"], - "description": "Inbond Transit number used for US Customs" + "type": "string", + "description": "The level of attention Flexport should give to this shipment.", + "enum": ["standard", "high"] }, - "created_date": { + "updated_at": { "type": "string", - "description": "Date the user has confirmed Flexport quote and cargo is getting ready to ship.", + "description": "Date when the shipment object was last updated. In ISO8601 UTC format with timezone denoted by Z.", "format": "date-time" }, - "estimated_picked_up_in_full_date": { + "estimated_departure_date": { "type": ["null", "string"], - "description": "Estimated pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Estimated departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, - "actual_picked_up_in_full_date": { + "actual_departure_date": { "type": ["null", "string"], - "description": "Actual pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Actual departure date from the first port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, - "estimated_delivered_in_full_date": { + "estimated_arrival_date": { "type": ["null", "string"], - "description": "Estimated delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Estimated arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, - "actual_delivered_in_full_date": { + "actual_arrival_date": { "type": ["null", "string"], - "description": "Actual delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "description": "Actual arrival date to the last port of the main voyage. In ISO8601 format with timezone denoted by +/-HH:MM.", "format": "date-time" }, "cargo_ready_date": { @@ -145,13 +150,17 @@ "type": "boolean", "description": "Determines if a shipment is using Flexport services for a combination of import customs clearance or final leg delivery." }, - "wants_export_customs_service": { - "type": "boolean", - "description": "Determines if the shipment requires Flexport to provide export customs service." + "wants_commercial_invoice_transcription": { + "type": "boolean" }, - "wants_import_customs_service": { - "type": "boolean", - "description": "Determines if the shipment requires Flexport to provide import customs service." + "wants_flexport_insurance": { + "type": "boolean" + }, + "wants_pickup_service": { + "type": "boolean" + }, + "wants_trade_declaration_service": { + "type": ["null", "boolean"] }, "visibility_only": { "type": "boolean", @@ -161,37 +170,58 @@ "type": "boolean", "description": "Determines if Flexport is responsible for door delivery on a shipment." }, - "wants_flexport_insurance": { - "type": "boolean" + "estimated_picked_up_in_full_date": { + "type": ["null", "string"], + "description": "Estimated pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" }, - "wants_pickup_service": { - "type": "boolean" + "actual_picked_up_in_full_date": { + "type": ["null", "string"], + "description": "Actual pickup date from the origin location. For ocean shipments with multiple containers, this is the date of last picked up container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" }, - "wants_commercial_invoice_transcription": { - "type": "boolean" + "target_delivery_date": { + "type": ["null", "string"], + "description": "Target date for when the shipment will be fully delivered. This date is set when the shipment''s booking is confirmed. This value may be different from estimated_delivered_in_full_date, which is updated when there is new information about the progress of a shipment. Date only.", + "format": "date" }, - "wants_trade_declaration_service": { - "type": ["null", "boolean"] + "estimated_delivered_in_full_date": { + "type": ["null", "string"], + "description": "Estimated delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" }, - "air_shipment": { - "description": "Air-specific Shipment information. Available only if this is an air shipment. null otherwise.", + "actual_delivered_in_full_date": { + "type": ["null", "string"], + "description": "Actual delivery date to the destination location. For ocean shipments with multiple containers, this is the date of last delivered container. In ISO8601 format with timezone denoted by +/-HH:MM.", + "format": "date-time" + }, + "wants_export_customs_service": { + "type": "boolean", + "description": "Determines if the shipment requires Flexport to provide export customs service." + }, + "wants_import_customs_service": { + "type": "boolean", + "description": "Determines if the shipment requires Flexport to provide import customs service." + }, + "ocean_shipment": { + "description": "Ocean-specific Shipment information. Available only if this is an ocean shipment. null otherwise.", "oneOf": [ { "type": "null" }, { - "$ref": "air/shipment.json" + "$ref": "ocean/shipment.json" } ] }, - "ocean_shipment": { - "description": "Ocean-specific Shipment information. Available only if this is an ocean shipment. null otherwise.", + "air_shipment": { + "description": "Air-specific Shipment information. Available only if this is an air shipment. null otherwise.", "oneOf": [ { "type": "null" }, { - "$ref": "ocean/shipment.json" + "$ref": "air/shipment.json" } ] }, @@ -257,10 +287,6 @@ "description": "Expandable API link to the documents for this shipment, represented by Document objects.", "$ref": "api/refs/collection.json" }, - "metadata": { - "type": "object", - "description": "User defined metadata attached to the shipment." - }, "departure_date": { "$comment": "deprecated", "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment/dangerous_goods.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment/dangerous_goods.json index 4cbbcb19412a..231e81137e66 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment/dangerous_goods.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment/dangerous_goods.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment_item.json b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment_item.json index 053b39ae0bbb..8ea0c96632f9 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment_item.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/schemas/shared/shipment_item.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, "properties": { "_object": { "type": "string", diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json b/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json index 23ae4f8adb9e..8589bb856cd9 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json @@ -3,9 +3,9 @@ "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Flexport Spec", + "additionalProperties": true, "type": "object", "required": ["api_key", "start_date"], - "additionalProperties": false, "properties": { "api_key": { "order": 0, diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py b/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py index f865ecec888d..d2315e0858d0 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py @@ -22,7 +22,7 @@ class FlexportStream(HttpStream, ABC): url_base = "https://api.flexport.com/" raise_on_http_errors = False primary_key = "id" - page_size = 500 + page_size = 100 def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, start_date: str = None): super().__init__(authenticator=authenticator) @@ -31,7 +31,7 @@ def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, sta self.start_date = start_date def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - # https://apidocs.flexport.com/reference/pagination + # https://apidocs.flexport.com/v3/tag/Pagination/ # All list endpoints return paginated responses. The response object contains # elements of the current page, and links to the previous and next pages. data = response.json()["data"] @@ -56,7 +56,7 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # https://apidocs.flexport.com/reference/response-layout + # https://apidocs.flexport.com/v3/tag/Response-Semantics json = response.json() http_error = None diff --git a/airbyte-integrations/connectors/source-freshcaller/.dockerignore b/airbyte-integrations/connectors/source-freshcaller/.dockerignore new file mode 100644 index 000000000000..9dcc6f34cdf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_freshcaller +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-freshcaller/Dockerfile b/airbyte-integrations/connectors/source-freshcaller/Dockerfile new file mode 100644 index 000000000000..f9bc7c0fca2a --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_freshcaller ./source_freshcaller + + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-freshcaller diff --git a/airbyte-integrations/connectors/source-freshcaller/README.md b/airbyte-integrations/connectors/source-freshcaller/README.md new file mode 100644 index 000000000000..0ea467b8aa97 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/README.md @@ -0,0 +1,133 @@ +# Freshcaller Source + +This is the repository for the Freshcaller source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/freshcaller). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshcaller) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshcaller/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshcaller test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-freshcaller:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-freshcaller:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshcaller:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshcaller:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshcaller:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +docker build . --no-cache -t airbyte/source-freshcaller:dev \ +&& python -m pytest -p source_acceptance_test.plugin +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml new file mode 100644 index 000000000000..a3d462b325a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml @@ -0,0 +1,22 @@ +connector_image: airbyte/source-freshcaller:dev +tests: + spec: + - spec_path: "source_freshcaller/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" + empty_streams: ["teams"] + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" diff --git a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-freshcaller/build.gradle b/airbyte-integrations/connectors/source-freshcaller/build.gradle new file mode 100644 index 000000000000..e09a198b7e9d --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_freshcaller' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..dbc7bff5fef0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json @@ -0,0 +1,8 @@ +{ + "call_metrics": { + "created_time": "2099-03-16T05:33:40.987000+00:00" + }, + "calls": { + "created_time": "2099-03-16T05:33:40.823000+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json new file mode 100644 index 000000000000..e634015fb51f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..e634015fb51f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json new file mode 100644 index 000000000000..0d038391a8c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,30 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["created_time"], + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["created_time"], + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json new file mode 100644 index 000000000000..3e0559353558 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "incorrect_key", + "start_date": "2021-08-02T00:00:00Z", + "domain": "incorrect_domain" +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py new file mode 100644 index 000000000000..7e548eb000c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pendulum +import pytest +from airbyte_cdk.models import SyncMode +from source_freshcaller.streams import APIIncrementalFreshcallerStream, CallMetrics, Calls + + +@pytest.fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(APIIncrementalFreshcallerStream, "cursor_field", "created_time") + mocker.patch.object(APIIncrementalFreshcallerStream, "__abstractmethods__", set()) + + +@pytest.fixture +def args(): + return {"authenticator": None, "config": {"api_key": "", "domain": "airbyte", "start_date": "2021-01-01T00:00:00.000Z"}} + + +@pytest.fixture +def stream(patch_incremental_base_class, args): + return APIIncrementalFreshcallerStream(**args) + + +@pytest.fixture +def call_metrics_stream(args): + return CallMetrics(**args) + + +@pytest.fixture +def calls_stream(args): + return Calls(**args) + + +@pytest.fixture +def streams_dict(calls_stream, call_metrics_stream): + return {"calls_stream": calls_stream, "call_metrics_stream": call_metrics_stream} + + +@pytest.mark.parametrize("fixture_name, expected", [("calls_stream", "created_time"), ("call_metrics_stream", "created_time")]) +def test_cursor_field(streams_dict, fixture_name, expected): + stream = streams_dict[fixture_name] + assert stream.cursor_field == expected + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_get_updated_state(streams_dict, fixture_name): + stream = streams_dict[fixture_name] + inputs = { + "current_stream_state": {"created_time": "2021-10-10T00:00:00.00Z"}, + "latest_record": {"created_time": "2021-10-20T00:00:00.00Z"}, + } + state = stream.get_updated_state(**inputs) + assert state["created_time"] == pendulum.parse("2021-10-20T00:00:00.00Z") + + inputs = {"current_stream_state": state, "latest_record": {"created_time": "2021-10-30T00:00:00.00Z"}} + state = stream.get_updated_state(**inputs) + assert state["created_time"] == pendulum.parse("2021-10-30T00:00:00.00Z") + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_get_updated_state_2(streams_dict, fixture_name): + stream = streams_dict[fixture_name] + current_stream_state = {"created_time": pendulum.now().add(days=-40)} + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": current_stream_state} + # The number of slices should be total lookback days by window_in_days i.e., 40 / 5 = 8 + assert len(stream.stream_slices(**inputs)) == 8 + + +def test_end_of_stream_state(calls_stream, requests_mock): + stream = calls_stream + requests_mock.get( + "https://airbyte.freshcaller.com/api/v1/calls?per_page=1000", + json={ + "calls": [{"created_time": "2021-10-30T00:00:00.00Z"}, {"created_time": "2021-10-29T00:00:00.00Z"}], + "meta": {"total_pages": 40, "current": 40}, + }, + ) + + state = {"created_time": "2021-10-01T00:00:00.00Z"} + sync_mode = SyncMode.incremental + last_state = None + for idx, app_slice in enumerate(stream.stream_slices(state, **MagicMock())): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=app_slice): + state = stream.get_updated_state(state, record) + last_state = state["created_time"] + assert last_state == pendulum.parse("2021-10-30T00:00:00.00Z") + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_supports_incremental(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + mocker.patch.object(APIIncrementalFreshcallerStream, "cursor_field", "dummy_field") + assert stream.supports_incremental + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_source_defined_cursor(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + assert stream.source_defined_cursor + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_stream_checkpoint_interval(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_request_params(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + inputs = { + "stream_state": {}, + "next_page_token": {"page": "5"}, + "stream_slice": {"by_time[from]": "2022-03-04 18:27:40", "by_time[to]": "2022-03-09 18:27:39"}, + } + expected_request_params = {"per_page": 1000, "page": "5", "by_time[from]": "2022-03-04 18:27:40", "by_time[to]": "2022-03-09 18:27:39"} + if stream.path() == "calls": + expected_request_params.update({"has_ancestry": "true"}) + assert stream.request_params(**inputs) == expected_request_params diff --git a/airbyte-integrations/connectors/source-freshcaller/main.py b/airbyte-integrations/connectors/source-freshcaller/main.py new file mode 100644 index 000000000000..9a9d1d2de2fb --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshcaller import SourceFreshcaller + +if __name__ == "__main__": + source = SourceFreshcaller() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshcaller/requirements.txt b/airbyte-integrations/connectors/source-freshcaller/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json new file mode 100644 index 000000000000..8c560859a49f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json @@ -0,0 +1,5 @@ +{ + "api_key": "YOUR_FRESHCALLER_API_KEY", + "start_date": "2021-08-02T00:00:00Z", + "domain": "FRESHCALLER_ACCOUNT_DOMAIN_NAME" +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json new file mode 100644 index 000000000000..c9226aa5bdc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json @@ -0,0 +1,407 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "status": { + "type": ["integer", "null"] + }, + "preference": { + "type": ["integer", "null"] + }, + "mobile_app_preference": { + "type": ["integer", "null"] + }, + "last_call_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "last_seen_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "confirmed": { + "type": ["boolean", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "time_zone": { + "type": ["string", "null"] + }, + "deleted": { + "type": ["boolean", "null"] + }, + "role": { + "type": ["string", "null"] + }, + "teams": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "teams", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "users": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "omni_channel": { + "type": ["boolean", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "calls", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "direction": { + "type": ["string", "null"] + }, + "parent_call_id": { + "type": ["integer", "null"] + }, + "root_call_id": { + "type": ["integer", "null"] + }, + "phone_number_id": { + "type": ["integer", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "assigned_agent_id": { + "type": ["integer", "null"] + }, + "assigned_agent_name": { + "type": ["string", "null"] + }, + "assigned_team_id": { + "type": ["integer", "null"] + }, + "assigned_team_name": { + "type": ["string", "null"] + }, + "assigned_call_queue_id": { + "type": ["integer", "null"] + }, + "assigned_call_queue_name": { + "type": ["string", "null"] + }, + "assigned_ivr_id": { + "type": ["integer", "null"] + }, + "assigned_ivr_name": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "call_notes": { + "type": ["string", "null"] + }, + "integrated_resources": { + "items": { + "properties": { + "integration_name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "recording": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "transcription_url": { + "type": ["string", "null"] + }, + "duration": { + "type": ["number", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + + "participants": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "caller_id": { + "type": ["integer", "null"] + }, + "caller_number": { + "type": ["string", "null"] + }, + "caller_name": { + "type": ["string", "null"] + }, + "participant_id": { + "type": ["integer", "null"] + }, + "participant_type": { + "type": ["string", "null"] + }, + "connection_type": { + "type": ["number", "null"] + }, + "call_status": { + "type": ["integer", "null"] + }, + "duration": { + "type": ["integer", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "enqueued_time": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "ivr_time": { + "type": ["integer", "null"] + }, + "ivr_time_unit": { + "type": ["string", "null"] + }, + "hold_duration": { + "type": ["number", "null"] + }, + "hold_duration_unit": { + "type": ["string", "null"] + }, + "call_work_time": { + "type": ["number", "null"] + }, + "call_work_time_unit": { + "type": ["string", "null"] + }, + "total_ringing_time": { + "type": ["number", "null"] + }, + "total_ringing_time_unit": { + "type": ["string", "null"] + }, + "talk_time": { + "type": ["number", "null"] + }, + "talk_time_unit": { + "type": ["string", "null"] + }, + "answering_speed": { + "type": ["number", "null"] + }, + "answering_speed_unit": { + "type": ["string", "null"] + }, + "recording_duration": { + "type": ["number", "null"] + }, + "recording_duration_unit": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "csat": { + "properties": { + "transfer_made": { + "type": ["boolean", "null"] + }, + "outcome": { + "type": ["string", "null"] + }, + "time": { + "type": ["number", "null"] + }, + "time_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "tags": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "default": { + "type": ["boolean", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json new file mode 100644 index 000000000000..743c16e299e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json @@ -0,0 +1,177 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "direction": { + "type": ["string", "null"] + }, + "parent_call_id": { + "type": ["integer", "null"] + }, + "root_call_id": { + "type": ["integer", "null"] + }, + "phone_number_id": { + "type": ["integer", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "assigned_agent_id": { + "type": ["integer", "null"] + }, + "assigned_agent_name": { + "type": ["string", "null"] + }, + "assigned_team_id": { + "type": ["integer", "null"] + }, + "assigned_team_name": { + "type": ["string", "null"] + }, + "assigned_call_queue_id": { + "type": ["integer", "null"] + }, + "assigned_call_queue_name": { + "type": ["string", "null"] + }, + "assigned_ivr_id": { + "type": ["integer", "null"] + }, + "assigned_ivr_name": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "call_notes": { + "type": ["string", "null"] + }, + "integrated_resources": { + "items": { + "properties": { + "integration_name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "recording": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "transcription_url": { + "type": ["string", "null"] + }, + "duration": { + "type": ["number", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + + "participants": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "caller_id": { + "type": ["integer", "null"] + }, + "caller_number": { + "type": ["string", "null"] + }, + "caller_name": { + "type": ["string", "null"] + }, + "participant_id": { + "type": ["integer", "null"] + }, + "participant_type": { + "type": ["string", "null"] + }, + "connection_type": { + "type": ["number", "null"] + }, + "call_status": { + "type": ["integer", "null"] + }, + "duration": { + "type": ["integer", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "enqueued_time": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..e634015fb51f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json new file mode 100644 index 000000000000..6ba515640165 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json @@ -0,0 +1,30 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["updated_time"], + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["updated_time"], + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json new file mode 100644 index 000000000000..2ac580a06e50 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json @@ -0,0 +1,124 @@ +{ + "streams": [ + { + "stream": { + "name": "call_metrics", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "ivr_time": { + "type": ["integer", "null"] + }, + "ivr_time_unit": { + "type": ["string", "null"] + }, + "hold_duration": { + "type": ["number", "null"] + }, + "hold_duration_unit": { + "type": ["string", "null"] + }, + "call_work_time": { + "type": ["number", "null"] + }, + "call_work_time_unit": { + "type": ["string", "null"] + }, + "total_ringing_time": { + "type": ["number", "null"] + }, + "total_ringing_time_unit": { + "type": ["string", "null"] + }, + "talk_time": { + "type": ["number", "null"] + }, + "talk_time_unit": { + "type": ["string", "null"] + }, + "answering_speed": { + "type": ["number", "null"] + }, + "answering_speed_unit": { + "type": ["string", "null"] + }, + "recording_duration": { + "type": ["number", "null"] + }, + "recording_duration_unit": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "csat": { + "properties": { + "transfer_made": { + "type": ["boolean", "null"] + }, + "outcome": { + "type": ["string", "null"] + }, + "time": { + "type": ["number", "null"] + }, + "time_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "tags": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "default": { + "type": ["boolean", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/setup.py b/airbyte-integrations/connectors/source-freshcaller/setup.py new file mode 100644 index 000000000000..862872947a06 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/setup.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum==1.2.0", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "requests-mock~=1.9.3", + "source-acceptance-test", +] + +setup( + name="source_freshcaller", + description="Source implementation for Freshcaller.", + author="Jay Bujala (Snapcommerce)", + author_email="jay.bujala@snapcommerce.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py new file mode 100644 index 000000000000..c4276956372e --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py @@ -0,0 +1,3 @@ +from .source import SourceFreshcaller + +__all__ = ["SourceFreshcaller"] diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json new file mode 100644 index 000000000000..51d07052b8a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json @@ -0,0 +1,109 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "ivr_time": { + "type": ["integer", "null"] + }, + "ivr_time_unit": { + "type": ["string", "null"] + }, + "hold_duration": { + "type": ["number", "null"] + }, + "hold_duration_unit": { + "type": ["string", "null"] + }, + "call_work_time": { + "type": ["number", "null"] + }, + "call_work_time_unit": { + "type": ["string", "null"] + }, + "total_ringing_time": { + "type": ["number", "null"] + }, + "total_ringing_time_unit": { + "type": ["string", "null"] + }, + "talk_time": { + "type": ["number", "null"] + }, + "talk_time_unit": { + "type": ["string", "null"] + }, + "answering_speed": { + "type": ["number", "null"] + }, + "answering_speed_unit": { + "type": ["string", "null"] + }, + "recording_duration": { + "type": ["number", "null"] + }, + "recording_duration_unit": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "csat": { + "properties": { + "transfer_made": { + "type": ["boolean", "null"] + }, + "outcome": { + "type": ["string", "null"] + }, + "time": { + "type": ["number", "null"] + }, + "time_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "tags": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "default": { + "type": ["boolean", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json new file mode 100644 index 000000000000..11d2e5ab64c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json @@ -0,0 +1,162 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "direction": { + "type": ["string", "null"] + }, + "parent_call_id": { + "type": ["integer", "null"] + }, + "root_call_id": { + "type": ["integer", "null"] + }, + "phone_number_id": { + "type": ["integer", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "assigned_agent_id": { + "type": ["integer", "null"] + }, + "assigned_agent_name": { + "type": ["string", "null"] + }, + "assigned_team_id": { + "type": ["integer", "null"] + }, + "assigned_team_name": { + "type": ["string", "null"] + }, + "assigned_call_queue_id": { + "type": ["integer", "null"] + }, + "assigned_call_queue_name": { + "type": ["string", "null"] + }, + "assigned_ivr_id": { + "type": ["integer", "null"] + }, + "assigned_ivr_name": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "call_notes": { + "type": ["string", "null"] + }, + "integrated_resources": { + "items": { + "properties": { + "integration_name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "recording": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "transcription_url": { + "type": ["string", "null"] + }, + "duration": { + "type": ["number", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + + "participants": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "caller_id": { + "type": ["integer", "null"] + }, + "caller_number": { + "type": ["string", "null"] + }, + "caller_name": { + "type": ["string", "null"] + }, + "participant_id": { + "type": ["integer", "null"] + }, + "participant_type": { + "type": ["string", "null"] + }, + "connection_type": { + "type": ["number", "null"] + }, + "call_status": { + "type": ["integer", "null"] + }, + "duration": { + "type": ["integer", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "enqueued_time": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json new file mode 100644 index 000000000000..ac0a75fc902d --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "users": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "omni_channel": { + "type": ["boolean", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json new file mode 100644 index 000000000000..2e257174594a --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "status": { + "type": ["integer", "null"] + }, + "preference": { + "type": ["integer", "null"] + }, + "mobile_app_preference": { + "type": ["integer", "null"] + }, + "last_call_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "last_seen_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "confirmed": { + "type": ["boolean", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "time_zone": { + "type": ["string", "null"] + }, + "deleted": { + "type": ["boolean", "null"] + }, + "role": { + "type": ["string", "null"] + }, + "teams": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py new file mode 100644 index 000000000000..6e34418d3122 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, List, Mapping, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from source_freshcaller.streams import CallMetrics, Calls, Teams, Users + +logger = logging.getLogger("airbyte") + + +class FreshcallerTokenAuthenticator(TokenAuthenticator): + def get_auth_header(self) -> Mapping[str, Any]: + return {"X-Api-Auth": self._token} + + +class SourceFreshcaller(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + api_url = f"https://{config['domain']}.freshcaller.com/api/v1" + auth = FreshcallerTokenAuthenticator(token=config["api_key"]).get_auth_header() + url = "{api_url}/users".format(api_url=api_url) + auth.update({"Accept": "application/json"}) + auth.update({"Content-Type": "application/json"}) + + try: + session = requests.get(url, headers=auth) + session.raise_for_status() + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + authenticator = FreshcallerTokenAuthenticator(token=config["api_key"]) + args = {"authenticator": authenticator, "config": config} + return [ + Users(**args), + Teams(**args), + Calls(**args), + CallMetrics(**args), + ] diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json new file mode 100644 index 000000000000..e3202175d5ff --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json @@ -0,0 +1,41 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshcaller", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Freshcaller Spec", + "type": "object", + "required": ["domain", "api_key", "start_date"], + "additionalProperties": true, + "properties": { + "domain": { + "type": "string", + "title": "Domain for Freshcaller account", + "description": "Used to construct Base URL for the Freshcaller APIs", + "examples": ["snaptravel"] + }, + "api_key": { + "type": "string", + "title": "API Key", + "description": "Freshcaller API Key. See the docs for more information on how to obtain this key.", + "airbyte_secret": true + }, + "requests_per_minute": { + "title": "Requests per minute", + "type": "integer", + "description": "The number of requests per minute that this source allowed to use. There is a rate limit of 50 requests per minute per app per account." + }, + "start_date": { + "title": "Start Date", + "description": "UTC date and time. Any data created after this date will be replicated.", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2022-01-01T12:00:00Z"] + }, + "sync_lag_minutes": { + "title": "Lag in minutes for each sync", + "type": "integer", + "description": "Lag in minutes for each sync, i.e., at time T, data for the time range [prev_sync_time, T-30] will be fetched" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py new file mode 100644 index 000000000000..064e8f5de8cc --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py @@ -0,0 +1,202 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Union + +import pendulum +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +class FreshcallerStream(HttpStream, ABC): + """Abstract class curated for Freshcaller""" + + primary_key = "id" + data_field = "" + start = 1 + page_limit = 1000 + api_version = 2 + curr_page_param = "page" + + def __init__(self, config: Dict, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def url_base(self) -> str: + return f"https://{self.config['domain']}.freshcaller.com/api/v1/" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + + params = {"per_page": self.page_limit, self.curr_page_param: self.start} + + # Handle pagination by inserting the next page's token in the request parameters + if next_page_token: + self.logger.debug(f"The next page is: {next_page_token}") + params.update(next_page_token) + return params + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + return {"Accept": "application/json", "User-Agent": "PostmanRuntime/7.28.0", "Content-Type": "application/json"} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + meta_data = decoded_response.get("meta") + if meta_data: + total_pages = meta_data["total_pages"] + current_page = meta_data["current"] + if current_page < total_pages: + current_page += 1 + return {self.curr_page_param: current_page} + return None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + yield from response_json.get(self.data_field, []) + + @property + def max_retries(self) -> Union[int, None]: + return 10 + + +class APIIncrementalFreshcallerStream(FreshcallerStream): + """ + Base abstract class for a "true" incremental stream, i.e., for an endpoint that supports + filtering by date or time + """ + + start_param = "by_time[from]" + end_param = "by_time[to]" + + def __init__(self, config: Dict, **kwargs): + super().__init__(config, **kwargs) + self.config = config + self.start_date = config["start_date"] + self.window_in_days = config.get("window_in_days", 5) + self.sync_lag_minutes = config.get("sync_lag_minutes", 30) + + @property + @abstractmethod + def cursor_field(self) -> str: + """ + Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class + and define a cursor field. + """ + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + """ + Override default get_updated_state CDK method to return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + last_synced_at = current_stream_state.get(self.cursor_field, self.start_date) + last_synced_at = pendulum.parse(last_synced_at) if isinstance(last_synced_at, str) else last_synced_at + return {self.cursor_field: max(pendulum.parse(latest_record.get(self.cursor_field)).in_tz("UTC"), last_synced_at)} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params[self.start_param] = stream_slice[self.start_param] + params[self.end_param] = stream_slice[self.end_param] + self.logger.info(f"Endpoint[{self.path()}] - Request params: {params}") + return params + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. + Returns list of dict, example: [{ + "by_time[from]": "2022-03-07 15:00:01", + "by_time[to]": "2022-03-07 18:00:00" + }, + { + "by_time[from]": "2022-03-07 18:00:01", + "by_time[to]": "2022-03-07 21:00:00" + }, + ...] + """ + start_date = pendulum.parse(self.start_date).in_timezone("UTC") + end_date = pendulum.utcnow().subtract(minutes=self.sync_lag_minutes) # have a safe lag + + # Determine stream_state, if no stream_state we use start_date + if stream_state: + start_date = stream_state.get(self.cursor_field) + start_date = pendulum.parse(start_date) if isinstance(start_date, str) else start_date + start_date = start_date.in_tz("UTC") + # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future + start_date: pendulum.Pendulum = min(start_date, end_date) + date_slices = [] + + while start_date <= end_date: + end_date_slice = start_date.add(days=self.window_in_days) + # add 1 second for start next slice to not duplicate data from previous slice end date. + stream_slice = { + self.start_param: start_date.add(seconds=1).to_datetime_string(), + self.end_param: min(end_date_slice, end_date).to_datetime_string(), + } + date_slices.append(stream_slice) + start_date = end_date_slice + + return date_slices + + +class Users(FreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#users + """ + + data_field = "users" + + def path(self, **kwargs) -> str: + return "users" + + +class Teams(FreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#teams + """ + + data_field = "teams" + + def path(self, **kwargs) -> str: + return "teams" + + +class Calls(APIIncrementalFreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#calls + """ + + data_field = "calls" + cursor_field = "created_time" + + def path(self, **kwargs) -> str: + return "calls" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = {**super().request_params(stream_state, stream_slice, next_page_token), "has_ancestry": "true"} + return params + + +class CallMetrics(APIIncrementalFreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#call-metrics + """ + + data_field = "call_metrics" + cursor_field = "created_time" + + def path(self, **kwargs) -> str: + return "call_metrics" diff --git a/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py new file mode 100644 index 000000000000..fd4055ebfaa9 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pendulum +from source_freshcaller.source import FreshcallerTokenAuthenticator, SourceFreshcaller + +now_dt = pendulum.now() + + +def test_authenticator(requests_mock): + URL = "https://example.com/" + TOKEN = "test_token" + config = { + "domain": "https://example.com", + "api_key": "test_token", + } + requests_mock.post(URL, json={"token": TOKEN}) + a = FreshcallerTokenAuthenticator(config["api_key"]) + auth_headers = a.get_auth_header() + assert auth_headers["X-Api-Auth"] == TOKEN + + +def test_count_streams(mocker): + source = SourceFreshcaller() + config_mock = mocker.MagicMock() + streams = source.streams(config_mock) + assert len(streams) == 4 diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py index 70b9e6d3fa46..233deec4f97d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ b/airbyte-integrations/connectors/source-freshdesk/setup.py @@ -15,7 +15,7 @@ TEST_REQUIREMENTS = [ "pytest==6.1.2", "pytest-mock~=3.6", - "requests_mock==1.8.0", + "requests_mock~=1.9.3", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index a1269661f05c..26b18158a930 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.44 +LABEL io.airbyte.version=0.2.45 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 6e3cb62c73b0..001751587bed 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -802,7 +802,8 @@ def _get_records(self, pull_request, repository_name): record["pull_request_url"] = pull_request["url"] if record["commit"]: record["commit_id"] = record.pop("commit")["oid"] - record["user"]["type"] = record["user"].pop("__typename") + if record["user"]: + record["user"]["type"] = record["user"].pop("__typename") # for backward compatibility with REST API response record["_links"] = { "html": {"href": record["html_url"]}, @@ -1019,7 +1020,8 @@ def _get_reactions_from_comment(self, comment, repository): for reaction in comment["reactions"]["nodes"]: reaction["repository"] = self._get_name(repository) reaction["comment_id"] = comment["id"] - reaction["user"]["type"] = "User" + if reaction["user"]: + reaction["user"]["type"] = "User" yield reaction def _get_reactions_from_review(self, review, repository): diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile index 8a7949105995..62c34bc6f85c 100644 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-greenhouse/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.7 +LABEL io.airbyte.version=0.2.8 LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml index 3c111edb3767..6c30ad302d24 100644 --- a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml @@ -7,7 +7,7 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" - - config_path: "secrets/config_users_only.json" + - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/config_invalid.json" status: "failed" @@ -17,6 +17,11 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: yes + exact_order: yes + extra_records: no - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_users_only.json" full_refresh: diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt new file mode 100644 index 000000000000..8e2a632edbd1 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt @@ -0,0 +1,533 @@ +{"stream":"applications","data":{"status":"active","source":{"public_name":"HRMARKET","id":4000067003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":{"name":"John Lafleur","id":4218086003}},"prospect":true,"location":null,"last_activity_at":"2020-11-24T23:24:37.049Z","jobs":[],"job_post_id":null,"id":19214950003,"current_stage":null,"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130511003,"applied_at":"2020-11-24T23:24:37.023Z","answers":[]},"emitted_at":1660156521774} +{"stream":"applications","data":{"status":"active","source":{"public_name":"Jobs page on your website","id":4000177003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":{"name":"John Lafleur","id":4218086003}},"prospect":true,"location":null,"last_activity_at":"2020-11-24T23:25:13.804Z","jobs":[],"job_post_id":null,"id":19214993003,"current_stage":null,"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130554003,"applied_at":"2020-11-24T23:25:13.781Z","answers":[]},"emitted_at":1660156521777} +{"stream":"applications","data":{"status":"active","source":{"public_name":"Internal Applicant","id":4000142003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2020-11-24T23:28:19.779Z","jobs":[{"name":"Test job","id":4177046003}],"job_post_id":null,"id":19215172003,"current_stage":{"name":"Preliminary Phone Screen","id":5245804003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130732003,"applied_at":"2020-11-24T23:28:19.712Z","answers":[]},"emitted_at":1660156521777} +{"stream":"applications","data":{"status":"active","source":{"public_name":"Referral","id":4000161003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2020-12-05T02:50:25.811Z","jobs":[{"name":"Test Job 2","id":4177048003}],"job_post_id":null,"id":19215333003,"current_stage":{"name":"Offer","id":5245823003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130848003,"applied_at":"2020-11-24T23:30:14.394Z","answers":[]},"emitted_at":1660156521778} +{"stream":"applications","data":{"status":"rejected","source":{"public_name":"Test agency","id":4013544003},"rejection_reason":{"type":{"name":"We rejected them","id":4000000003},"name":"Other (add notes below)","id":4000004003},"rejection_details":{},"rejected_at":"2021-09-29T16:38:03.637Z","prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-09-29T16:38:03.660Z","jobs":[{"name":"Test job","id":4177046003}],"job_post_id":null,"id":44933447003,"current_stage":{"name":"Phone Interview","id":5245805003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":40513954003,"applied_at":"2021-09-29T16:37:27.589Z","answers":[]},"emitted_at":1660156521778} +{"stream":"applications","data":{"status":"active","source":{"public_name":"Test agency","id":4013544003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-10-10T16:22:13.708Z","jobs":[{"name":"Test job","id":4177046003}],"job_post_id":null,"id":44937562003,"current_stage":{"name":"Preliminary Phone Screen","id":5245804003},"credited_to":{"name":"Greenhouse Admin","last_name":"Admin","id":4218085003,"first_name":"Greenhouse","employee_id":null},"candidate_id":40517966003,"applied_at":"2021-09-29T17:20:36.063Z","answers":[]},"emitted_at":1660156521778} +{"stream":"applications","data":{"status":"active","source":{"public_name":"Test agency","id":4013544003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-11-03T19:56:07.402Z","jobs":[{"name":"Test job 3","id":4466310003}],"job_post_id":4797691003,"id":47459993003,"current_stage":{"name":"Application Review","id":7332462003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":42921157003,"applied_at":"2021-11-03T19:51:14.644Z","answers":[{"question":"Website","answer":null},{"question":"LinkedIn Profile","answer":null}]},"emitted_at":1660156521779} +{"stream":"applications","data":{"status":"active","source":{"public_name":"Bubblesort","id":4000032003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-11-22T08:41:55.713Z","jobs":[{"name":"Copy of Test Job 2","id":4446240003}],"job_post_id":null,"id":48693310003,"current_stage":{"name":"Application Review","id":7179760003},"credited_to":{"name":"emily.brooks+airbyte_integration@greenhouse.io","last_name":null,"id":4218087003,"first_name":null,"employee_id":null},"candidate_id":44081361003,"applied_at":"2021-11-22T08:41:55.640Z","answers":[]},"emitted_at":1660156521779} +{ "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-11-24T23:24:37.050Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Test","last_activity": "2020-11-24T23:24:37.049Z","is_private": false,"id": 17130511003,"first_name": "Test","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:24:37.018Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19214950003 ],"addresses": [ ] },"emitted_at": 1660156191149 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-11-24T23:25:13.806Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Test2","last_activity": "2020-11-24T23:25:13.804Z","is_private": false,"id": 17130554003,"first_name": "Test2","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:25:13.777Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19214993003 ],"addresses": [ ] },"emitted_at": 1660156191151 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-11-24T23:28:19.781Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Lastname","last_activity": "2020-11-24T23:28:19.779Z","is_private": false,"id": 17130732003,"first_name": "Name","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:28:19.710Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19215172003 ],"addresses": [ ] },"emitted_at": 1660156191152 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-12-05T02:50:25.823Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "D","last_activity": "2020-12-05T02:50:25.811Z","is_private": false,"id": 17130848003,"first_name": "Jack","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:30:14.386Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19215333003 ],"addresses": [ ] },"emitted_at": 1660156191152 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-09-29T16:38:03.672Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "User","last_activity": "2021-09-29T16:38:03.660Z","is_private": false,"id": 40513954003,"first_name": "Test","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2021-09-29T16:37:27.585Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 44933447003 ],"addresses": [ ] },"emitted_at": 1660156191152 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-10-10T16:22:13.718Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Scheduled Interview","last_activity": "2021-10-10T16:22:13.708Z","is_private": false,"id": 40517966003,"first_name": "Test","employments": [ ],"email_addresses": [ { "value": "vadym.hevlich@zazmic.com","type": "personal" } ],"educations": [ ],"created_at": "2021-09-29T17:20:36.038Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 44937562003 ],"addresses": [ ] },"emitted_at": 1660156191153 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-11-03T19:56:07.423Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Candidate","last_activity": "2021-11-03T19:56:07.402Z","is_private": false,"id": 42921157003,"first_name": "Test","employments": [ ],"email_addresses": [ { "value": "vadym.hevlich@zazmic.com","type": "work" } ],"educations": [ ],"created_at": "2021-11-03T19:51:14.639Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 47459993003 ],"addresses": [ ] },"emitted_at": 1660156191153 } + { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-11-22T08:41:55.716Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Cherniaev","last_activity": "2021-11-22T08:41:55.713Z","is_private": false,"id": 44081361003,"first_name": "Yurii","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2021-11-22T08:41:55.634Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 48693310003 ],"addresses": [ ] },"emitted_at": 1660156191153 } +{"stream":"close_reasons","data":{"id":4010635003,"name":"Not Filling"},"emitted_at":1660156523420} +{"stream":"close_reasons","data":{"id":4010634003,"name":"On Hold"},"emitted_at":1660156523422} +{"stream":"close_reasons","data":{"id":4010633003,"name":"Hire - New Headcount"},"emitted_at":1660156523422} +{"stream":"close_reasons","data":{"id":4010632003,"name":"Hire - Backfill"},"emitted_at":1660156523422} +{"stream":"degrees","data":{"id":10848287003,"name":"High School","priority":0,"external_id":null},"emitted_at":1660156523707} +{"stream":"degrees","data":{"id":10848288003,"name":"Associate's Degree","priority":1,"external_id":null},"emitted_at":1660156523708} +{"stream":"degrees","data":{"id":10848289003,"name":"Bachelor's Degree","priority":2,"external_id":null},"emitted_at":1660156523709} +{"stream":"degrees","data":{"id":10848290003,"name":"Master's Degree","priority":3,"external_id":null},"emitted_at":1660156523709} +{"stream":"degrees","data":{"id":10848291003,"name":"Master of Business Administration (M.B.A.)","priority":4,"external_id":null},"emitted_at":1660156523709} +{"stream":"degrees","data":{"id":10848292003,"name":"Juris Doctor (J.D.)","priority":5,"external_id":null},"emitted_at":1660156523709} +{"stream":"degrees","data":{"id":10848293003,"name":"Doctor of Medicine (M.D.)","priority":6,"external_id":null},"emitted_at":1660156523709} +{"stream":"degrees","data":{"id":10848294003,"name":"Doctor of Philosophy (Ph.D.)","priority":7,"external_id":null},"emitted_at":1660156523710} +{"stream":"degrees","data":{"id":10848295003,"name":"Engineer's Degree","priority":8,"external_id":null},"emitted_at":1660156523710} +{"stream":"degrees","data":{"id":10848296003,"name":"Other","priority":9,"external_id":null},"emitted_at":1660156523710} +{"stream":"departments","data":{"id":4028123003,"name":"test dep 2","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null},"emitted_at":1660156524114} +{"stream":"departments","data":{"id":4028122003,"name":"Test dep1","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null},"emitted_at":1660156524116} +{"stream":"job_posts","data":{"id":4252332003,"active":true,"live":false,"first_published_at":null,"title":"Test job","location":{"id":4219721003,"name":"test","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4177046003,"content":"

Test description

","internal_content":null,"updated_at":"2021-04-02T17:38:54.835Z","created_at":"2020-11-24T23:29:24.315Z","demographic_question_set_id":null,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_5125927003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_5125928003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524473} +{"stream":"job_posts","data":{"id":4751597003,"active":true,"live":false,"first_published_at":null,"title":"Test Job 2","location":{"id":4700649003,"name":"US","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4177048003,"content":"

Job post content

","internal_content":null,"updated_at":"2021-10-07T18:46:59.032Z","created_at":"2021-10-07T18:46:58.846Z","demographic_question_set_id":null,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_7911674003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_7911675003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524477} +{"stream":"job_posts","data":{"id":4752433003,"active":true,"live":false,"first_published_at":null,"title":"Test Job 2","location":{"id":4701484003,"name":"US","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4446240003,"content":"

Job post content

","internal_content":null,"updated_at":"2021-10-08T08:19:42.720Z","created_at":"2021-10-08T08:19:42.720Z","demographic_question_set_id":null,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_7918434003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_7918435003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524478} +{"stream":"job_posts","data":{"id":4797691003,"active":true,"live":false,"first_published_at":null,"title":"Test job 3","location":{"id":4746722003,"name":"US","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4466310003,"content":"

job description

","internal_content":null,"updated_at":"2021-11-03T19:48:29.808Z","created_at":"2021-11-03T19:48:29.629Z","demographic_question_set_id":4000198003,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_8215275003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_8215276003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524479} +{"stream":"jobs","data":{"id":4177046003,"name":"Test job","requisition_id":"3","notes":null,"confidential":false,"is_template":true,"copied_from_id":null,"status":"open","created_at":"2020-11-24T23:27:11.699Z","opened_at":"2020-11-24T23:27:11.878Z","closed_at":null,"updated_at":"2021-04-21T17:48:24.779Z","departments":[{"id":4028122003,"name":"Test dep1","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4320015003,"opening_id":"3-1","status":"open","opened_at":"2020-11-24T23:27:11.723Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":null},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":null}}},"emitted_at":1660156524869} +{"stream":"jobs","data":{"id":4177048003,"name":"Test Job 2","requisition_id":"4","notes":null,"confidential":false,"is_template":false,"copied_from_id":null,"status":"open","created_at":"2020-11-24T23:27:45.634Z","opened_at":"2020-11-24T23:27:45.878Z","closed_at":null,"updated_at":"2021-10-07T18:46:58.982Z","departments":[{"id":4028123003,"name":"test dep 2","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4320018003,"opening_id":"4-1","status":"open","opened_at":"2020-11-24T23:27:45.665Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":null},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":null}}},"emitted_at":1660156524875} +{"stream":"jobs","data":{"id":4446240003,"name":"Copy of Test Job 2","requisition_id":"5","notes":null,"confidential":false,"is_template":false,"copied_from_id":4177048003,"status":"open","created_at":"2021-10-08T08:19:42.383Z","opened_at":"2021-10-08T08:19:42.818Z","closed_at":null,"updated_at":"2021-10-08T08:19:42.821Z","departments":[{"id":4028123003,"name":"test dep 2","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4928188003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:24.949Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4928187003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:08.365Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4928186003,"opening_id":"5-1","status":"open","opened_at":"2021-10-10T16:38:57.407Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4926182003,"opening_id":"5-1","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4926183003,"opening_id":"5-2","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":"Full-time"},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":"Full-time"}}},"emitted_at":1660156524875} +{"stream":"jobs","data":{"id":4466310003,"name":"Test job 3","requisition_id":"6","notes":null,"confidential":false,"is_template":false,"copied_from_id":null,"status":"open","created_at":"2021-11-03T19:46:51.107Z","opened_at":"2021-11-03T19:46:51.347Z","closed_at":null,"updated_at":"2021-11-03T19:48:29.760Z","departments":[{"id":4028122003,"name":"Test dep1","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4970166003,"opening_id":"6-1","status":"open","opened_at":"2021-11-30T01:00:00.000Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":"Full-time"},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":"Full-time"}}},"emitted_at":1660156524876} +{"stream":"offers","data":{"id":4154100003,"version":1,"application_id":19215333003,"created_at":"2020-11-24T23:32:25.760Z","updated_at":"2020-11-24T23:32:25.772Z","sent_at":null,"resolved_at":null,"starts_at":"2020-12-04","status":"unresolved","job_id":4177048003,"candidate_id":17130848003,"opening":{"id":4320018003,"opening_id":"4-1","status":"open","opened_at":"2020-11-24T23:27:45.665Z","closed_at":null,"application_id":null,"close_reason":null},"custom_fields":{"employment_type":"Contract"},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":"Contract"}}},"emitted_at":1660156525177} +{"stream":"scorecards","data":{"id":5253031003,"updated_at":"2020-11-24T23:33:10.440Z","created_at":"2020-11-24T23:33:10.440Z","interview":"Application Review","interview_step":{"id":5628634003,"name":"Application Review"},"candidate_id":17130848003,"application_id":19215333003,"interviewed_at":"2020-11-25T01:00:00.000Z","submitted_by":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"interviewer":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"submitted_at":"2020-11-24T23:33:10.440Z","overall_recommendation":"no_decision","attributes":[{"name":"Willing to do required travel","type":"Details","note":null,"rating":"no_decision"},{"name":"Three to five years of experience","type":"Qualifications","note":null,"rating":"no_decision"},{"name":"Personable","type":"Personality Traits","note":null,"rating":"no_decision"},{"name":"Passionate","type":"Personality Traits","note":null,"rating":"no_decision"},{"name":"Organizational Skills","type":"Skills","note":null,"rating":"no_decision"},{"name":"Manage competing priorities","type":"Skills","note":null,"rating":"no_decision"},{"name":"Fits our salary range","type":"Details","note":null,"rating":"no_decision"},{"name":"Empathetic","type":"Personality Traits","note":null,"rating":"no_decision"},{"name":"Currently based locally","type":"Details","note":null,"rating":"no_decision"},{"name":"Communication","type":"Skills","note":null,"rating":"no_decision"}],"ratings":{"definitely_not":[],"no":[],"mixed":[],"yes":[],"strong_yes":[]},"questions":[{"id":null,"question":"Key Take-Aways","answer":""},{"id":null,"question":"Private Notes","answer":""}]},"emitted_at":1660156525508} +{"stream":"scorecards","data":{"id":9664505003,"updated_at":"2021-09-29T17:23:11.468Z","created_at":"2021-09-29T17:23:11.468Z","interview":"Preliminary Screening Call","interview_step":{"id":5628615003,"name":"Preliminary Screening Call"},"candidate_id":40517966003,"application_id":44937562003,"interviewed_at":"2021-09-29T01:00:00.000Z","submitted_by":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"interviewer":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"submitted_at":"2021-09-29T17:23:11.468Z","overall_recommendation":"no_decision","attributes":[{"name":"Willing to do required travel","type":"Details","note":null,"rating":"yes"},{"name":"Three to five years of experience","type":"Qualifications","note":null,"rating":"mixed"},{"name":"Personable","type":"Personality Traits","note":null,"rating":"yes"},{"name":"Passionate","type":"Personality Traits","note":null,"rating":"mixed"},{"name":"Organizational Skills","type":"Skills","note":null,"rating":"yes"},{"name":"Manage competing priorities","type":"Skills","note":null,"rating":"yes"},{"name":"Fits our salary range","type":"Details","note":null,"rating":"yes"},{"name":"Empathetic","type":"Personality Traits","note":null,"rating":"strong_yes"},{"name":"Currently based locally","type":"Details","note":null,"rating":"mixed"},{"name":"Communication","type":"Skills","note":null,"rating":"no"}],"ratings":{"definitely_not":[],"no":["Communication"],"mixed":["Three to five years of experience","Passionate","Currently based locally"],"yes":["Willing to do required travel","Personable","Organizational Skills","Manage competing priorities","Fits our salary range"],"strong_yes":["Empathetic"]},"questions":[{"id":null,"question":"Key Take-Aways","answer":"test"},{"id":null,"question":"Private Notes","answer":""}]},"emitted_at":1660156525511} +{"stream":"users","data":{"id":4218085003,"name":"Greenhouse Admin","first_name":"Greenhouse","last_name":"Admin","primary_email_address":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","updated_at":"2020-11-18T14:09:08.401Z","created_at":"2020-11-18T14:09:08.401Z","disabled":false,"site_admin":true,"emails":["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525823} +{"stream":"users","data":{"id":4218086003,"name":"John Lafleur","first_name":"John","last_name":"Lafleur","primary_email_address":"integration-test@airbyte.io","updated_at":"2022-04-06T12:41:57.185Z","created_at":"2020-11-18T14:09:08.481Z","disabled":false,"site_admin":true,"emails":["integration-test@airbyte.io"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} +{"stream":"users","data":{"id":4218087003,"name":"emily.brooks+airbyte_integration@greenhouse.io","first_name":null,"last_name":null,"primary_email_address":"emily.brooks+airbyte_integration@greenhouse.io","updated_at":"2020-11-18T14:09:08.991Z","created_at":"2020-11-18T14:09:08.809Z","disabled":false,"site_admin":true,"emails":["emily.brooks+airbyte_integration@greenhouse.io"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} +{"stream":"users","data":{"id":4460715003,"name":"Vadym Ratniuk","first_name":"Vadym","last_name":"Ratniuk","primary_email_address":"vadym.ratniuk@globallogic.com","updated_at":"2021-09-18T10:09:16.846Z","created_at":"2021-09-14T14:03:01.050Z","disabled":false,"site_admin":false,"emails":["vadym.ratniuk@globallogic.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} +{"stream":"users","data":{"id":4481107003,"name":"Vadym Hevlich","first_name":"Vadym","last_name":"Hevlich","primary_email_address":"vadym.hevlich@zazmic.com","updated_at":"2021-10-10T17:49:28.058Z","created_at":"2021-10-10T17:48:41.978Z","disabled":false,"site_admin":true,"emails":["vadym.hevlich@zazmic.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525827} +{"stream":"custom_fields","data":{"id":4680898003,"name":"School Name","active":true,"field_type":"candidate","priority":0,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"school_name","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845822003,"name":"Abraham Baldwin Agricultural College","priority":0,"external_id":null},{"id":10845823003,"name":"Academy of Art University","priority":1,"external_id":null},{"id":10845824003,"name":"Acadia University","priority":2,"external_id":null},{"id":10845825003,"name":"Adams State University","priority":3,"external_id":null},{"id":10845826003,"name":"Adelphi University","priority":4,"external_id":null},{"id":10845827003,"name":"Adrian College","priority":5,"external_id":null},{"id":10845828003,"name":"Adventist University of Health Sciences","priority":6,"external_id":null},{"id":10845829003,"name":"Agnes Scott College","priority":7,"external_id":null},{"id":10845830003,"name":"AIB College of Business","priority":8,"external_id":null},{"id":10845831003,"name":"Alaska Pacific University","priority":9,"external_id":null},{"id":10845832003,"name":"Albany College of Pharmacy and Health Sciences","priority":10,"external_id":null},{"id":10845833003,"name":"Albany State University","priority":11,"external_id":null},{"id":10845834003,"name":"Albertus Magnus College","priority":12,"external_id":null},{"id":10845835003,"name":"Albion College","priority":13,"external_id":null},{"id":10845836003,"name":"Albright College","priority":14,"external_id":null},{"id":10845837003,"name":"Alderson Broaddus University","priority":15,"external_id":null},{"id":10845838003,"name":"Alfred University","priority":16,"external_id":null},{"id":10845839003,"name":"Alice Lloyd College","priority":17,"external_id":null},{"id":10845840003,"name":"Allegheny College","priority":18,"external_id":null},{"id":10845841003,"name":"Allen College","priority":19,"external_id":null},{"id":10845842003,"name":"Allen University","priority":20,"external_id":null},{"id":10845843003,"name":"Alliant International University","priority":21,"external_id":null},{"id":10845844003,"name":"Alma College","priority":22,"external_id":null},{"id":10845845003,"name":"Alvernia University","priority":23,"external_id":null},{"id":10845846003,"name":"Alverno College","priority":24,"external_id":null},{"id":10845847003,"name":"Amberton University","priority":25,"external_id":null},{"id":10845848003,"name":"American Academy of Art","priority":26,"external_id":null},{"id":10845849003,"name":"American Indian College of the Assemblies of God","priority":27,"external_id":null},{"id":10845850003,"name":"American InterContinental University","priority":28,"external_id":null},{"id":10845851003,"name":"American International College","priority":29,"external_id":null},{"id":10845852003,"name":"American Jewish University","priority":30,"external_id":null},{"id":10845853003,"name":"American Public University System","priority":31,"external_id":null},{"id":10845854003,"name":"American University","priority":32,"external_id":null},{"id":10845855003,"name":"American University in Bulgaria","priority":33,"external_id":null},{"id":10845856003,"name":"American University in Cairo","priority":34,"external_id":null},{"id":10845857003,"name":"American University of Beirut","priority":35,"external_id":null},{"id":10845858003,"name":"American University of Paris","priority":36,"external_id":null},{"id":10845859003,"name":"American University of Puerto Rico","priority":37,"external_id":null},{"id":10845860003,"name":"Amherst College","priority":38,"external_id":null},{"id":10845861003,"name":"Amridge University","priority":39,"external_id":null},{"id":10845862003,"name":"Anderson University","priority":40,"external_id":null},{"id":10845863003,"name":"Andrews University","priority":41,"external_id":null},{"id":10845864003,"name":"Angelo State University","priority":42,"external_id":null},{"id":10845865003,"name":"Anna Maria College","priority":43,"external_id":null},{"id":10845866003,"name":"Antioch University","priority":44,"external_id":null},{"id":10845867003,"name":"Appalachian Bible College","priority":45,"external_id":null},{"id":10845868003,"name":"Aquinas College","priority":46,"external_id":null},{"id":10845869003,"name":"Arcadia University","priority":47,"external_id":null},{"id":10845870003,"name":"Argosy University","priority":48,"external_id":null},{"id":10845871003,"name":"Arizona Christian University","priority":49,"external_id":null},{"id":10845872003,"name":"Arizona State University - West","priority":50,"external_id":null},{"id":10845873003,"name":"Arkansas Baptist College","priority":51,"external_id":null},{"id":10845874003,"name":"Arkansas Tech University","priority":52,"external_id":null},{"id":10845875003,"name":"Armstrong Atlantic State University","priority":53,"external_id":null},{"id":10845876003,"name":"Art Academy of Cincinnati","priority":54,"external_id":null},{"id":10845877003,"name":"Art Center College of Design","priority":55,"external_id":null},{"id":10845878003,"name":"Art Institute of Atlanta","priority":56,"external_id":null},{"id":10845879003,"name":"Art Institute of Colorado","priority":57,"external_id":null},{"id":10845880003,"name":"Art Institute of Houston","priority":58,"external_id":null},{"id":10845881003,"name":"Art Institute of Pittsburgh","priority":59,"external_id":null},{"id":10845882003,"name":"Art Institute of Portland","priority":60,"external_id":null},{"id":10845883003,"name":"Art Institute of Seattle","priority":61,"external_id":null},{"id":10845884003,"name":"Asbury University","priority":62,"external_id":null},{"id":10845885003,"name":"Ashford University","priority":63,"external_id":null},{"id":10845886003,"name":"Ashland University","priority":64,"external_id":null},{"id":10845887003,"name":"Assumption College","priority":65,"external_id":null},{"id":10845888003,"name":"Athens State University","priority":66,"external_id":null},{"id":10845889003,"name":"Auburn University - Montgomery","priority":67,"external_id":null},{"id":10845890003,"name":"Augsburg College","priority":68,"external_id":null},{"id":10845891003,"name":"Augustana College","priority":69,"external_id":null},{"id":10845892003,"name":"Aurora University","priority":70,"external_id":null},{"id":10845893003,"name":"Austin College","priority":71,"external_id":null},{"id":10845894003,"name":"Alcorn State University","priority":72,"external_id":null},{"id":10845895003,"name":"Ave Maria University","priority":73,"external_id":null},{"id":10845896003,"name":"Averett University","priority":74,"external_id":null},{"id":10845897003,"name":"Avila University","priority":75,"external_id":null},{"id":10845898003,"name":"Azusa Pacific University","priority":76,"external_id":null},{"id":10845899003,"name":"Babson College","priority":77,"external_id":null},{"id":10845900003,"name":"Bacone College","priority":78,"external_id":null},{"id":10845901003,"name":"Baker College of Flint","priority":79,"external_id":null},{"id":10845902003,"name":"Baker University","priority":80,"external_id":null},{"id":10845903003,"name":"Baldwin Wallace University","priority":81,"external_id":null},{"id":10845904003,"name":"Christian Brothers University","priority":82,"external_id":null},{"id":10845905003,"name":"Abilene Christian University","priority":83,"external_id":null},{"id":10845906003,"name":"Arizona State University","priority":84,"external_id":null},{"id":10845907003,"name":"Auburn University","priority":85,"external_id":null},{"id":10845908003,"name":"Alabama A&M University","priority":86,"external_id":null},{"id":10845909003,"name":"Alabama State University","priority":87,"external_id":null},{"id":10845910003,"name":"Arkansas State University","priority":88,"external_id":null},{"id":10845911003,"name":"Baptist Bible College","priority":89,"external_id":null},{"id":10845912003,"name":"Baptist Bible College and Seminary","priority":90,"external_id":null},{"id":10845913003,"name":"Baptist College of Florida","priority":91,"external_id":null},{"id":10845914003,"name":"Baptist Memorial College of Health Sciences","priority":92,"external_id":null},{"id":10845915003,"name":"Baptist Missionary Association Theological Seminary","priority":93,"external_id":null},{"id":10845916003,"name":"Bard College","priority":94,"external_id":null},{"id":10845917003,"name":"Bard College at Simon's Rock","priority":95,"external_id":null},{"id":10845918003,"name":"Barnard College","priority":96,"external_id":null},{"id":10845919003,"name":"Barry University","priority":97,"external_id":null},{"id":10845920003,"name":"Barton College","priority":98,"external_id":null},{"id":10845921003,"name":"Bastyr University","priority":99,"external_id":null},{"id":10845922003,"name":"Bates College","priority":100,"external_id":null},{"id":10845923003,"name":"Bauder College","priority":101,"external_id":null},{"id":10845924003,"name":"Bay Path College","priority":102,"external_id":null},{"id":10845925003,"name":"Bay State College","priority":103,"external_id":null},{"id":10845926003,"name":"Bayamon Central University","priority":104,"external_id":null},{"id":10845927003,"name":"Beacon College","priority":105,"external_id":null},{"id":10845928003,"name":"Becker College","priority":106,"external_id":null},{"id":10845929003,"name":"Belhaven University","priority":107,"external_id":null},{"id":10845930003,"name":"Bellarmine University","priority":108,"external_id":null},{"id":10845931003,"name":"Bellevue College","priority":109,"external_id":null},{"id":10845932003,"name":"Bellevue University","priority":110,"external_id":null},{"id":10845933003,"name":"Bellin College","priority":111,"external_id":null},{"id":10845934003,"name":"Belmont Abbey College","priority":112,"external_id":null},{"id":10845935003,"name":"Belmont University","priority":113,"external_id":null},{"id":10845936003,"name":"Beloit College","priority":114,"external_id":null},{"id":10845937003,"name":"Bemidji State University","priority":115,"external_id":null},{"id":10845938003,"name":"Benedict College","priority":116,"external_id":null},{"id":10845939003,"name":"Benedictine College","priority":117,"external_id":null},{"id":10845940003,"name":"Benedictine University","priority":118,"external_id":null},{"id":10845941003,"name":"Benjamin Franklin Institute of Technology","priority":119,"external_id":null},{"id":10845942003,"name":"Bennett College","priority":120,"external_id":null},{"id":10845943003,"name":"Bennington College","priority":121,"external_id":null},{"id":10845944003,"name":"Bentley University","priority":122,"external_id":null},{"id":10845945003,"name":"Berea College","priority":123,"external_id":null},{"id":10845946003,"name":"Berkeley College","priority":124,"external_id":null},{"id":10845947003,"name":"Berklee College of Music","priority":125,"external_id":null},{"id":10845948003,"name":"Berry College","priority":126,"external_id":null},{"id":10845949003,"name":"Bethany College","priority":127,"external_id":null},{"id":10845950003,"name":"Bethany Lutheran College","priority":128,"external_id":null},{"id":10845951003,"name":"Bethel College","priority":129,"external_id":null},{"id":10845952003,"name":"Bethel University","priority":130,"external_id":null},{"id":10845953003,"name":"BI Norwegian Business School","priority":131,"external_id":null},{"id":10845954003,"name":"Binghamton University - SUNY","priority":132,"external_id":null},{"id":10845955003,"name":"Biola University","priority":133,"external_id":null},{"id":10845956003,"name":"Birmingham-Southern College","priority":134,"external_id":null},{"id":10845957003,"name":"Bismarck State College","priority":135,"external_id":null},{"id":10845958003,"name":"Black Hills State University","priority":136,"external_id":null},{"id":10845959003,"name":"Blackburn College","priority":137,"external_id":null},{"id":10845960003,"name":"Blessing-Rieman College of Nursing","priority":138,"external_id":null},{"id":10845961003,"name":"Bloomfield College","priority":139,"external_id":null},{"id":10845962003,"name":"Bloomsburg University of Pennsylvania","priority":140,"external_id":null},{"id":10845963003,"name":"Blue Mountain College","priority":141,"external_id":null},{"id":10845964003,"name":"Bluefield College","priority":142,"external_id":null},{"id":10845965003,"name":"Bluefield State College","priority":143,"external_id":null},{"id":10845966003,"name":"Bluffton University","priority":144,"external_id":null},{"id":10845967003,"name":"Boricua College","priority":145,"external_id":null},{"id":10845968003,"name":"Boston Architectural College","priority":146,"external_id":null},{"id":10845969003,"name":"Boston Conservatory","priority":147,"external_id":null},{"id":10845970003,"name":"Boston University","priority":148,"external_id":null},{"id":10845971003,"name":"Bowdoin College","priority":149,"external_id":null},{"id":10845972003,"name":"Bowie State University","priority":150,"external_id":null},{"id":10845973003,"name":"Bradley University","priority":151,"external_id":null},{"id":10845974003,"name":"Brandeis University","priority":152,"external_id":null},{"id":10845975003,"name":"Brandman University","priority":153,"external_id":null},{"id":10845976003,"name":"Brazosport College","priority":154,"external_id":null},{"id":10845977003,"name":"Brenau University","priority":155,"external_id":null},{"id":10845978003,"name":"Brescia University","priority":156,"external_id":null},{"id":10845979003,"name":"Brevard College","priority":157,"external_id":null},{"id":10845980003,"name":"Brewton-Parker College","priority":158,"external_id":null},{"id":10845981003,"name":"Briar Cliff University","priority":159,"external_id":null},{"id":10845982003,"name":"Briarcliffe College","priority":160,"external_id":null},{"id":10845983003,"name":"Bridgewater College","priority":161,"external_id":null},{"id":10845984003,"name":"Bridgewater State University","priority":162,"external_id":null},{"id":10845985003,"name":"Brigham Young University - Hawaii","priority":163,"external_id":null},{"id":10845986003,"name":"Brigham Young University - Idaho","priority":164,"external_id":null},{"id":10845987003,"name":"Brock University","priority":165,"external_id":null},{"id":10845988003,"name":"Bryan College","priority":166,"external_id":null},{"id":10845989003,"name":"Bryn Athyn College of the New Church","priority":167,"external_id":null},{"id":10845990003,"name":"Bryn Mawr College","priority":168,"external_id":null},{"id":10845991003,"name":"Boston College","priority":169,"external_id":null},{"id":10845992003,"name":"Buena Vista University","priority":170,"external_id":null},{"id":10845993003,"name":"Burlington College","priority":171,"external_id":null},{"id":10845994003,"name":"Bowling Green State University","priority":172,"external_id":null},{"id":10845995003,"name":"Brown University","priority":173,"external_id":null},{"id":10845996003,"name":"Appalachian State University","priority":174,"external_id":null},{"id":10845997003,"name":"Brigham Young University - Provo","priority":175,"external_id":null},{"id":10845998003,"name":"Boise State University","priority":176,"external_id":null},{"id":10845999003,"name":"Bethune-Cookman University","priority":177,"external_id":null},{"id":10846000003,"name":"Bryant University","priority":178,"external_id":null},{"id":10846001003,"name":"Cabarrus College of Health Sciences","priority":179,"external_id":null},{"id":10846002003,"name":"Cabrini College","priority":180,"external_id":null},{"id":10846003003,"name":"Cairn University","priority":181,"external_id":null},{"id":10846004003,"name":"Caldwell College","priority":182,"external_id":null},{"id":10846005003,"name":"California Baptist University","priority":183,"external_id":null},{"id":10846006003,"name":"California College of the Arts","priority":184,"external_id":null},{"id":10846007003,"name":"California Institute of Integral Studies","priority":185,"external_id":null},{"id":10846008003,"name":"California Institute of Technology","priority":186,"external_id":null},{"id":10846009003,"name":"California Institute of the Arts","priority":187,"external_id":null},{"id":10846010003,"name":"California Lutheran University","priority":188,"external_id":null},{"id":10846011003,"name":"California Maritime Academy","priority":189,"external_id":null},{"id":10846012003,"name":"California State Polytechnic University - Pomona","priority":190,"external_id":null},{"id":10846013003,"name":"California State University - Bakersfield","priority":191,"external_id":null},{"id":10846014003,"name":"California State University - Channel Islands","priority":192,"external_id":null},{"id":10846015003,"name":"California State University - Chico","priority":193,"external_id":null},{"id":10846016003,"name":"California State University - Dominguez Hills","priority":194,"external_id":null},{"id":10846017003,"name":"California State University - East Bay","priority":195,"external_id":null},{"id":10846018003,"name":"California State University - Fullerton","priority":196,"external_id":null},{"id":10846019003,"name":"California State University - Los Angeles","priority":197,"external_id":null},{"id":10846020003,"name":"California State University - Monterey Bay","priority":198,"external_id":null},{"id":10846021003,"name":"California State University - Northridge","priority":199,"external_id":null},{"id":10846022003,"name":"California State University - San Bernardino","priority":200,"external_id":null},{"id":10846023003,"name":"California State University - San Marcos","priority":201,"external_id":null},{"id":10846024003,"name":"California State University - Stanislaus","priority":202,"external_id":null},{"id":10846025003,"name":"California University of Pennsylvania","priority":203,"external_id":null},{"id":10846026003,"name":"Calumet College of St. Joseph","priority":204,"external_id":null},{"id":10846027003,"name":"Calvary Bible College and Theological Seminary","priority":205,"external_id":null},{"id":10846028003,"name":"Calvin College","priority":206,"external_id":null},{"id":10846029003,"name":"Cambridge College","priority":207,"external_id":null},{"id":10846030003,"name":"Cameron University","priority":208,"external_id":null},{"id":10846031003,"name":"Campbellsville University","priority":209,"external_id":null},{"id":10846032003,"name":"Canisius College","priority":210,"external_id":null},{"id":10846033003,"name":"Capella University","priority":211,"external_id":null},{"id":10846034003,"name":"Capital University","priority":212,"external_id":null},{"id":10846035003,"name":"Capitol College","priority":213,"external_id":null},{"id":10846036003,"name":"Cardinal Stritch University","priority":214,"external_id":null},{"id":10846037003,"name":"Caribbean University","priority":215,"external_id":null},{"id":10846038003,"name":"Carleton College","priority":216,"external_id":null},{"id":10846039003,"name":"Carleton University","priority":217,"external_id":null},{"id":10846040003,"name":"Carlos Albizu University","priority":218,"external_id":null},{"id":10846041003,"name":"Carlow University","priority":219,"external_id":null},{"id":10846042003,"name":"Carnegie Mellon University","priority":220,"external_id":null},{"id":10846043003,"name":"Carroll College","priority":221,"external_id":null},{"id":10846044003,"name":"Carroll University","priority":222,"external_id":null},{"id":10846045003,"name":"Carson-Newman University","priority":223,"external_id":null},{"id":10846046003,"name":"Carthage College","priority":224,"external_id":null},{"id":10846047003,"name":"Case Western Reserve University","priority":225,"external_id":null},{"id":10846048003,"name":"Castleton State College","priority":226,"external_id":null},{"id":10846049003,"name":"Catawba College","priority":227,"external_id":null},{"id":10846050003,"name":"Cazenovia College","priority":228,"external_id":null},{"id":10846051003,"name":"Cedar Crest College","priority":229,"external_id":null},{"id":10846052003,"name":"Cedarville University","priority":230,"external_id":null},{"id":10846053003,"name":"Centenary College","priority":231,"external_id":null},{"id":10846054003,"name":"Centenary College of Louisiana","priority":232,"external_id":null},{"id":10846055003,"name":"Central Baptist College","priority":233,"external_id":null},{"id":10846056003,"name":"Central Bible College","priority":234,"external_id":null},{"id":10846057003,"name":"Central Christian College","priority":235,"external_id":null},{"id":10846058003,"name":"Central College","priority":236,"external_id":null},{"id":10846059003,"name":"Central Methodist University","priority":237,"external_id":null},{"id":10846060003,"name":"Central Penn College","priority":238,"external_id":null},{"id":10846061003,"name":"Central State University","priority":239,"external_id":null},{"id":10846062003,"name":"Central Washington University","priority":240,"external_id":null},{"id":10846063003,"name":"Centre College","priority":241,"external_id":null},{"id":10846064003,"name":"Chadron State College","priority":242,"external_id":null},{"id":10846065003,"name":"Chamberlain College of Nursing","priority":243,"external_id":null},{"id":10846066003,"name":"Chaminade University of Honolulu","priority":244,"external_id":null},{"id":10846067003,"name":"Champlain College","priority":245,"external_id":null},{"id":10846068003,"name":"Chancellor University","priority":246,"external_id":null},{"id":10846069003,"name":"Chapman University","priority":247,"external_id":null},{"id":10846070003,"name":"Charles R. Drew University of Medicine and Science","priority":248,"external_id":null},{"id":10846071003,"name":"Charter Oak State College","priority":249,"external_id":null},{"id":10846072003,"name":"Chatham University","priority":250,"external_id":null},{"id":10846073003,"name":"Chestnut Hill College","priority":251,"external_id":null},{"id":10846074003,"name":"Cheyney University of Pennsylvania","priority":252,"external_id":null},{"id":10846075003,"name":"Chicago State University","priority":253,"external_id":null},{"id":10846076003,"name":"Chipola College","priority":254,"external_id":null},{"id":10846077003,"name":"Chowan University","priority":255,"external_id":null},{"id":10846078003,"name":"Christendom College","priority":256,"external_id":null},{"id":10846079003,"name":"Baylor University","priority":257,"external_id":null},{"id":10846080003,"name":"Central Connecticut State University","priority":258,"external_id":null},{"id":10846081003,"name":"Central Michigan University","priority":259,"external_id":null},{"id":10846082003,"name":"Charleston Southern University","priority":260,"external_id":null},{"id":10846083003,"name":"California State University - Sacramento","priority":261,"external_id":null},{"id":10846084003,"name":"California State University - Fresno","priority":262,"external_id":null},{"id":10846085003,"name":"Campbell University","priority":263,"external_id":null},{"id":10846086003,"name":"Christopher Newport University","priority":264,"external_id":null},{"id":10846087003,"name":"Cincinnati Christian University","priority":265,"external_id":null},{"id":10846088003,"name":"Cincinnati College of Mortuary Science","priority":266,"external_id":null},{"id":10846089003,"name":"City University of Seattle","priority":267,"external_id":null},{"id":10846090003,"name":"Claflin University","priority":268,"external_id":null},{"id":10846091003,"name":"Claremont McKenna College","priority":269,"external_id":null},{"id":10846092003,"name":"Clarion University of Pennsylvania","priority":270,"external_id":null},{"id":10846093003,"name":"Clark Atlanta University","priority":271,"external_id":null},{"id":10846094003,"name":"Clark University","priority":272,"external_id":null},{"id":10846095003,"name":"Clarke University","priority":273,"external_id":null},{"id":10846096003,"name":"Clarkson College","priority":274,"external_id":null},{"id":10846097003,"name":"Clarkson University","priority":275,"external_id":null},{"id":10846098003,"name":"Clayton State University","priority":276,"external_id":null},{"id":10846099003,"name":"Clear Creek Baptist Bible College","priority":277,"external_id":null},{"id":10846100003,"name":"Clearwater Christian College","priority":278,"external_id":null},{"id":10846101003,"name":"Cleary University","priority":279,"external_id":null},{"id":10846102003,"name":"College of William and Mary","priority":280,"external_id":null},{"id":10846103003,"name":"Cleveland Chiropractic College","priority":281,"external_id":null},{"id":10846104003,"name":"Cleveland Institute of Art","priority":282,"external_id":null},{"id":10846105003,"name":"Cleveland Institute of Music","priority":283,"external_id":null},{"id":10846106003,"name":"Cleveland State University","priority":284,"external_id":null},{"id":10846107003,"name":"Coe College","priority":285,"external_id":null},{"id":10846108003,"name":"Cogswell Polytechnical College","priority":286,"external_id":null},{"id":10846109003,"name":"Coker College","priority":287,"external_id":null},{"id":10846110003,"name":"Colby College","priority":288,"external_id":null},{"id":10846111003,"name":"Colby-Sawyer College","priority":289,"external_id":null},{"id":10846112003,"name":"College at Brockport - SUNY","priority":290,"external_id":null},{"id":10846113003,"name":"College for Creative Studies","priority":291,"external_id":null},{"id":10846114003,"name":"College of Charleston","priority":292,"external_id":null},{"id":10846115003,"name":"College of Idaho","priority":293,"external_id":null},{"id":10846116003,"name":"College of Mount St. Joseph","priority":294,"external_id":null},{"id":10846117003,"name":"College of Mount St. Vincent","priority":295,"external_id":null},{"id":10846118003,"name":"College of New Jersey","priority":296,"external_id":null},{"id":10846119003,"name":"College of New Rochelle","priority":297,"external_id":null},{"id":10846120003,"name":"College of Our Lady of the Elms","priority":298,"external_id":null},{"id":10846121003,"name":"College of Saints John Fisher & Thomas More","priority":299,"external_id":null},{"id":10846122003,"name":"College of Southern Nevada","priority":300,"external_id":null},{"id":10846123003,"name":"College of St. Benedict","priority":301,"external_id":null},{"id":10846124003,"name":"College of St. Elizabeth","priority":302,"external_id":null},{"id":10846125003,"name":"College of St. Joseph","priority":303,"external_id":null},{"id":10846126003,"name":"College of St. Mary","priority":304,"external_id":null},{"id":10846127003,"name":"College of St. Rose","priority":305,"external_id":null},{"id":10846128003,"name":"College of St. Scholastica","priority":306,"external_id":null},{"id":10846129003,"name":"College of the Atlantic","priority":307,"external_id":null},{"id":10846130003,"name":"College of the Holy Cross","priority":308,"external_id":null},{"id":10846131003,"name":"College of the Ozarks","priority":309,"external_id":null},{"id":10846132003,"name":"College of Wooster","priority":310,"external_id":null},{"id":10846133003,"name":"Colorado Christian University","priority":311,"external_id":null},{"id":10846134003,"name":"Colorado College","priority":312,"external_id":null},{"id":10846135003,"name":"Colorado Mesa University","priority":313,"external_id":null},{"id":10846136003,"name":"Colorado School of Mines","priority":314,"external_id":null},{"id":10846137003,"name":"Colorado State University - Pueblo","priority":315,"external_id":null},{"id":10846138003,"name":"Colorado Technical University","priority":316,"external_id":null},{"id":10846139003,"name":"Columbia College","priority":317,"external_id":null},{"id":10846140003,"name":"Columbia College Chicago","priority":318,"external_id":null},{"id":10846141003,"name":"Columbia College of Nursing","priority":319,"external_id":null},{"id":10846142003,"name":"Columbia International University","priority":320,"external_id":null},{"id":10846143003,"name":"Columbus College of Art and Design","priority":321,"external_id":null},{"id":10846144003,"name":"Columbus State University","priority":322,"external_id":null},{"id":10846145003,"name":"Conception Seminary College","priority":323,"external_id":null},{"id":10846146003,"name":"Concord University","priority":324,"external_id":null},{"id":10846147003,"name":"Concordia College","priority":325,"external_id":null},{"id":10846148003,"name":"Concordia College - Moorhead","priority":326,"external_id":null},{"id":10846149003,"name":"Concordia University","priority":327,"external_id":null},{"id":10846150003,"name":"Concordia University Chicago","priority":328,"external_id":null},{"id":10846151003,"name":"Concordia University Texas","priority":329,"external_id":null},{"id":10846152003,"name":"Concordia University Wisconsin","priority":330,"external_id":null},{"id":10846153003,"name":"Concordia University - St. Paul","priority":331,"external_id":null},{"id":10846154003,"name":"Connecticut College","priority":332,"external_id":null},{"id":10846155003,"name":"Converse College","priority":333,"external_id":null},{"id":10846156003,"name":"Cooper Union","priority":334,"external_id":null},{"id":10846157003,"name":"Coppin State University","priority":335,"external_id":null},{"id":10846158003,"name":"Corban University","priority":336,"external_id":null},{"id":10846159003,"name":"Corcoran College of Art and Design","priority":337,"external_id":null},{"id":10846160003,"name":"Cornell College","priority":338,"external_id":null},{"id":10846161003,"name":"Cornerstone University","priority":339,"external_id":null},{"id":10846162003,"name":"Cornish College of the Arts","priority":340,"external_id":null},{"id":10846163003,"name":"Covenant College","priority":341,"external_id":null},{"id":10846164003,"name":"Cox College","priority":342,"external_id":null},{"id":10846165003,"name":"Creighton University","priority":343,"external_id":null},{"id":10846166003,"name":"Criswell College","priority":344,"external_id":null},{"id":10846167003,"name":"Crown College","priority":345,"external_id":null},{"id":10846168003,"name":"Culinary Institute of America","priority":346,"external_id":null},{"id":10846169003,"name":"Culver-Stockton College","priority":347,"external_id":null},{"id":10846170003,"name":"Cumberland University","priority":348,"external_id":null},{"id":10846171003,"name":"Columbia University","priority":349,"external_id":null},{"id":10846172003,"name":"Cornell University","priority":350,"external_id":null},{"id":10846173003,"name":"Colorado State University","priority":351,"external_id":null},{"id":10846174003,"name":"University of Virginia","priority":352,"external_id":null},{"id":10846175003,"name":"Colgate University","priority":353,"external_id":null},{"id":10846176003,"name":"CUNY - Baruch College","priority":354,"external_id":null},{"id":10846177003,"name":"CUNY - Brooklyn College","priority":355,"external_id":null},{"id":10846178003,"name":"CUNY - City College","priority":356,"external_id":null},{"id":10846179003,"name":"CUNY - College of Staten Island","priority":357,"external_id":null},{"id":10846180003,"name":"CUNY - Hunter College","priority":358,"external_id":null},{"id":10846181003,"name":"CUNY - John Jay College of Criminal Justice","priority":359,"external_id":null},{"id":10846182003,"name":"CUNY - Lehman College","priority":360,"external_id":null},{"id":10846183003,"name":"CUNY - Medgar Evers College","priority":361,"external_id":null},{"id":10846184003,"name":"CUNY - New York City College of Technology","priority":362,"external_id":null},{"id":10846185003,"name":"CUNY - Queens College","priority":363,"external_id":null},{"id":10846186003,"name":"CUNY - York College","priority":364,"external_id":null},{"id":10846187003,"name":"Curry College","priority":365,"external_id":null},{"id":10846188003,"name":"Curtis Institute of Music","priority":366,"external_id":null},{"id":10846189003,"name":"D'Youville College","priority":367,"external_id":null},{"id":10846190003,"name":"Daemen College","priority":368,"external_id":null},{"id":10846191003,"name":"Dakota State University","priority":369,"external_id":null},{"id":10846192003,"name":"Dakota Wesleyan University","priority":370,"external_id":null},{"id":10846193003,"name":"Dalhousie University","priority":371,"external_id":null},{"id":10846194003,"name":"Dallas Baptist University","priority":372,"external_id":null},{"id":10846195003,"name":"Dallas Christian College","priority":373,"external_id":null},{"id":10846196003,"name":"Dalton State College","priority":374,"external_id":null},{"id":10846197003,"name":"Daniel Webster College","priority":375,"external_id":null},{"id":10846198003,"name":"Davenport University","priority":376,"external_id":null},{"id":10846199003,"name":"Davis and Elkins College","priority":377,"external_id":null},{"id":10846200003,"name":"Davis College","priority":378,"external_id":null},{"id":10846201003,"name":"Daytona State College","priority":379,"external_id":null},{"id":10846202003,"name":"Dean College","priority":380,"external_id":null},{"id":10846203003,"name":"Defiance College","priority":381,"external_id":null},{"id":10846204003,"name":"Delaware Valley College","priority":382,"external_id":null},{"id":10846205003,"name":"Delta State University","priority":383,"external_id":null},{"id":10846206003,"name":"Denison University","priority":384,"external_id":null},{"id":10846207003,"name":"DePaul University","priority":385,"external_id":null},{"id":10846208003,"name":"DePauw University","priority":386,"external_id":null},{"id":10846209003,"name":"DEREE - The American College of Greece","priority":387,"external_id":null},{"id":10846210003,"name":"DeSales University","priority":388,"external_id":null},{"id":10846211003,"name":"DeVry University","priority":389,"external_id":null},{"id":10846212003,"name":"Dickinson College","priority":390,"external_id":null},{"id":10846213003,"name":"Dickinson State University","priority":391,"external_id":null},{"id":10846214003,"name":"Dillard University","priority":392,"external_id":null},{"id":10846215003,"name":"Divine Word College","priority":393,"external_id":null},{"id":10846216003,"name":"Dixie State College of Utah","priority":394,"external_id":null},{"id":10846217003,"name":"Doane College","priority":395,"external_id":null},{"id":10846218003,"name":"Dominican College","priority":396,"external_id":null},{"id":10846219003,"name":"Dominican University","priority":397,"external_id":null},{"id":10846220003,"name":"Dominican University of California","priority":398,"external_id":null},{"id":10846221003,"name":"Donnelly College","priority":399,"external_id":null},{"id":10846222003,"name":"Dordt College","priority":400,"external_id":null},{"id":10846223003,"name":"Dowling College","priority":401,"external_id":null},{"id":10846224003,"name":"Drew University","priority":402,"external_id":null},{"id":10846225003,"name":"Drexel University","priority":403,"external_id":null},{"id":10846226003,"name":"Drury University","priority":404,"external_id":null},{"id":10846227003,"name":"Dunwoody College of Technology","priority":405,"external_id":null},{"id":10846228003,"name":"Earlham College","priority":406,"external_id":null},{"id":10846229003,"name":"Drake University","priority":407,"external_id":null},{"id":10846230003,"name":"East Central University","priority":408,"external_id":null},{"id":10846231003,"name":"East Stroudsburg University of Pennsylvania","priority":409,"external_id":null},{"id":10846232003,"name":"East Tennessee State University","priority":410,"external_id":null},{"id":10846233003,"name":"East Texas Baptist University","priority":411,"external_id":null},{"id":10846234003,"name":"East-West University","priority":412,"external_id":null},{"id":10846235003,"name":"Eastern Connecticut State University","priority":413,"external_id":null},{"id":10846236003,"name":"Eastern Mennonite University","priority":414,"external_id":null},{"id":10846237003,"name":"Eastern Nazarene College","priority":415,"external_id":null},{"id":10846238003,"name":"Eastern New Mexico University","priority":416,"external_id":null},{"id":10846239003,"name":"Eastern Oregon University","priority":417,"external_id":null},{"id":10846240003,"name":"Eastern University","priority":418,"external_id":null},{"id":10846241003,"name":"Eckerd College","priority":419,"external_id":null},{"id":10846242003,"name":"ECPI University","priority":420,"external_id":null},{"id":10846243003,"name":"Edgewood College","priority":421,"external_id":null},{"id":10846244003,"name":"Edinboro University of Pennsylvania","priority":422,"external_id":null},{"id":10846245003,"name":"Edison State College","priority":423,"external_id":null},{"id":10846246003,"name":"Edward Waters College","priority":424,"external_id":null},{"id":10846247003,"name":"Elizabeth City State University","priority":425,"external_id":null},{"id":10846248003,"name":"Elizabethtown College","priority":426,"external_id":null},{"id":10846249003,"name":"Elmhurst College","priority":427,"external_id":null},{"id":10846250003,"name":"Elmira College","priority":428,"external_id":null},{"id":10846251003,"name":"Embry-Riddle Aeronautical University","priority":429,"external_id":null},{"id":10846252003,"name":"Embry-Riddle Aeronautical University - Prescott","priority":430,"external_id":null},{"id":10846253003,"name":"Emerson College","priority":431,"external_id":null},{"id":10846254003,"name":"Duquesne University","priority":432,"external_id":null},{"id":10846255003,"name":"Eastern Washington University","priority":433,"external_id":null},{"id":10846256003,"name":"Eastern Illinois University","priority":434,"external_id":null},{"id":10846257003,"name":"Eastern Kentucky University","priority":435,"external_id":null},{"id":10846258003,"name":"Eastern Michigan University","priority":436,"external_id":null},{"id":10846259003,"name":"Elon University","priority":437,"external_id":null},{"id":10846260003,"name":"Delaware State University","priority":438,"external_id":null},{"id":10846261003,"name":"Duke University","priority":439,"external_id":null},{"id":10846262003,"name":"California Polytechnic State University - San Luis Obispo","priority":440,"external_id":null},{"id":10846263003,"name":"Emmanuel College","priority":441,"external_id":null},{"id":10846264003,"name":"Emmaus Bible College","priority":442,"external_id":null},{"id":10846265003,"name":"Emory and Henry College","priority":443,"external_id":null},{"id":10846266003,"name":"Emory University","priority":444,"external_id":null},{"id":10846267003,"name":"Emporia State University","priority":445,"external_id":null},{"id":10846268003,"name":"Endicott College","priority":446,"external_id":null},{"id":10846269003,"name":"Erskine College","priority":447,"external_id":null},{"id":10846270003,"name":"Escuela de Artes Plasticas de Puerto Rico","priority":448,"external_id":null},{"id":10846271003,"name":"Eureka College","priority":449,"external_id":null},{"id":10846272003,"name":"Evangel University","priority":450,"external_id":null},{"id":10846273003,"name":"Everest College - Phoenix","priority":451,"external_id":null},{"id":10846274003,"name":"Everglades University","priority":452,"external_id":null},{"id":10846275003,"name":"Evergreen State College","priority":453,"external_id":null},{"id":10846276003,"name":"Excelsior College","priority":454,"external_id":null},{"id":10846277003,"name":"Fairfield University","priority":455,"external_id":null},{"id":10846278003,"name":"Fairleigh Dickinson University","priority":456,"external_id":null},{"id":10846279003,"name":"Fairmont State University","priority":457,"external_id":null},{"id":10846280003,"name":"Faith Baptist Bible College and Theological Seminary","priority":458,"external_id":null},{"id":10846281003,"name":"Farmingdale State College - SUNY","priority":459,"external_id":null},{"id":10846282003,"name":"Fashion Institute of Technology","priority":460,"external_id":null},{"id":10846283003,"name":"Faulkner University","priority":461,"external_id":null},{"id":10846284003,"name":"Fayetteville State University","priority":462,"external_id":null},{"id":10846285003,"name":"Felician College","priority":463,"external_id":null},{"id":10846286003,"name":"Ferris State University","priority":464,"external_id":null},{"id":10846287003,"name":"Ferrum College","priority":465,"external_id":null},{"id":10846288003,"name":"Finlandia University","priority":466,"external_id":null},{"id":10846289003,"name":"Fisher College","priority":467,"external_id":null},{"id":10846290003,"name":"Fisk University","priority":468,"external_id":null},{"id":10846291003,"name":"Fitchburg State University","priority":469,"external_id":null},{"id":10846292003,"name":"Five Towns College","priority":470,"external_id":null},{"id":10846293003,"name":"Flagler College","priority":471,"external_id":null},{"id":10846294003,"name":"Florida Christian College","priority":472,"external_id":null},{"id":10846295003,"name":"Florida College","priority":473,"external_id":null},{"id":10846296003,"name":"Florida Gulf Coast University","priority":474,"external_id":null},{"id":10846297003,"name":"Florida Institute of Technology","priority":475,"external_id":null},{"id":10846298003,"name":"Florida Memorial University","priority":476,"external_id":null},{"id":10846299003,"name":"Florida Southern College","priority":477,"external_id":null},{"id":10846300003,"name":"Florida State College - Jacksonville","priority":478,"external_id":null},{"id":10846301003,"name":"Fontbonne University","priority":479,"external_id":null},{"id":10846302003,"name":"Fort Hays State University","priority":480,"external_id":null},{"id":10846303003,"name":"Fort Lewis College","priority":481,"external_id":null},{"id":10846304003,"name":"Fort Valley State University","priority":482,"external_id":null},{"id":10846305003,"name":"Framingham State University","priority":483,"external_id":null},{"id":10846306003,"name":"Francis Marion University","priority":484,"external_id":null},{"id":10846307003,"name":"Franciscan University of Steubenville","priority":485,"external_id":null},{"id":10846308003,"name":"Frank Lloyd Wright School of Architecture","priority":486,"external_id":null},{"id":10846309003,"name":"Franklin and Marshall College","priority":487,"external_id":null},{"id":10846310003,"name":"Franklin College","priority":488,"external_id":null},{"id":10846311003,"name":"Franklin College Switzerland","priority":489,"external_id":null},{"id":10846312003,"name":"Franklin Pierce University","priority":490,"external_id":null},{"id":10846313003,"name":"Franklin University","priority":491,"external_id":null},{"id":10846314003,"name":"Franklin W. Olin College of Engineering","priority":492,"external_id":null},{"id":10846315003,"name":"Freed-Hardeman University","priority":493,"external_id":null},{"id":10846316003,"name":"Fresno Pacific University","priority":494,"external_id":null},{"id":10846317003,"name":"Friends University","priority":495,"external_id":null},{"id":10846318003,"name":"Frostburg State University","priority":496,"external_id":null},{"id":10846319003,"name":"Gallaudet University","priority":497,"external_id":null},{"id":10846320003,"name":"Gannon University","priority":498,"external_id":null},{"id":10846321003,"name":"Geneva College","priority":499,"external_id":null},{"id":10846322003,"name":"George Fox University","priority":500,"external_id":null},{"id":10846323003,"name":"George Mason University","priority":501,"external_id":null},{"id":10846324003,"name":"George Washington University","priority":502,"external_id":null},{"id":10846325003,"name":"Georgetown College","priority":503,"external_id":null},{"id":10846326003,"name":"Georgia College & State University","priority":504,"external_id":null},{"id":10846327003,"name":"Georgia Gwinnett College","priority":505,"external_id":null},{"id":10846328003,"name":"Georgia Regents University","priority":506,"external_id":null},{"id":10846329003,"name":"Georgia Southwestern State University","priority":507,"external_id":null},{"id":10846330003,"name":"Georgian Court University","priority":508,"external_id":null},{"id":10846331003,"name":"Gettysburg College","priority":509,"external_id":null},{"id":10846332003,"name":"Glenville State College","priority":510,"external_id":null},{"id":10846333003,"name":"God's Bible School and College","priority":511,"external_id":null},{"id":10846334003,"name":"Goddard College","priority":512,"external_id":null},{"id":10846335003,"name":"Golden Gate University","priority":513,"external_id":null},{"id":10846336003,"name":"Goldey-Beacom College","priority":514,"external_id":null},{"id":10846337003,"name":"Goldfarb School of Nursing at Barnes-Jewish College","priority":515,"external_id":null},{"id":10846338003,"name":"Gonzaga University","priority":516,"external_id":null},{"id":10846339003,"name":"Gordon College","priority":517,"external_id":null},{"id":10846340003,"name":"Fordham University","priority":518,"external_id":null},{"id":10846341003,"name":"Georgia Institute of Technology","priority":519,"external_id":null},{"id":10846342003,"name":"Gardner-Webb University","priority":520,"external_id":null},{"id":10846343003,"name":"Georgia Southern University","priority":521,"external_id":null},{"id":10846344003,"name":"Georgia State University","priority":522,"external_id":null},{"id":10846345003,"name":"Florida State University","priority":523,"external_id":null},{"id":10846346003,"name":"Dartmouth College","priority":524,"external_id":null},{"id":10846347003,"name":"Florida International University","priority":525,"external_id":null},{"id":10846348003,"name":"Georgetown University","priority":526,"external_id":null},{"id":10846349003,"name":"Furman University","priority":527,"external_id":null},{"id":10846350003,"name":"Gordon State College","priority":528,"external_id":null},{"id":10846351003,"name":"Goshen College","priority":529,"external_id":null},{"id":10846352003,"name":"Goucher College","priority":530,"external_id":null},{"id":10846353003,"name":"Governors State University","priority":531,"external_id":null},{"id":10846354003,"name":"Grace Bible College","priority":532,"external_id":null},{"id":10846355003,"name":"Grace College and Seminary","priority":533,"external_id":null},{"id":10846356003,"name":"Grace University","priority":534,"external_id":null},{"id":10846357003,"name":"Graceland University","priority":535,"external_id":null},{"id":10846358003,"name":"Grand Canyon University","priority":536,"external_id":null},{"id":10846359003,"name":"Grand Valley State University","priority":537,"external_id":null},{"id":10846360003,"name":"Grand View University","priority":538,"external_id":null},{"id":10846361003,"name":"Granite State College","priority":539,"external_id":null},{"id":10846362003,"name":"Gratz College","priority":540,"external_id":null},{"id":10846363003,"name":"Great Basin College","priority":541,"external_id":null},{"id":10846364003,"name":"Great Lakes Christian College","priority":542,"external_id":null},{"id":10846365003,"name":"Green Mountain College","priority":543,"external_id":null},{"id":10846366003,"name":"Greensboro College","priority":544,"external_id":null},{"id":10846367003,"name":"Greenville College","priority":545,"external_id":null},{"id":10846368003,"name":"Grinnell College","priority":546,"external_id":null},{"id":10846369003,"name":"Grove City College","priority":547,"external_id":null},{"id":10846370003,"name":"Guilford College","priority":548,"external_id":null},{"id":10846371003,"name":"Gustavus Adolphus College","priority":549,"external_id":null},{"id":10846372003,"name":"Gwynedd-Mercy College","priority":550,"external_id":null},{"id":10846373003,"name":"Hamilton College","priority":551,"external_id":null},{"id":10846374003,"name":"Hamline University","priority":552,"external_id":null},{"id":10846375003,"name":"Hampden-Sydney College","priority":553,"external_id":null},{"id":10846376003,"name":"Hampshire College","priority":554,"external_id":null},{"id":10846377003,"name":"Hannibal-LaGrange University","priority":555,"external_id":null},{"id":10846378003,"name":"Hanover College","priority":556,"external_id":null},{"id":10846379003,"name":"Hardin-Simmons University","priority":557,"external_id":null},{"id":10846380003,"name":"Harding University","priority":558,"external_id":null},{"id":10846381003,"name":"Harrington College of Design","priority":559,"external_id":null},{"id":10846382003,"name":"Harris-Stowe State University","priority":560,"external_id":null},{"id":10846383003,"name":"Harrisburg University of Science and Technology","priority":561,"external_id":null},{"id":10846384003,"name":"Hartwick College","priority":562,"external_id":null},{"id":10846385003,"name":"Harvey Mudd College","priority":563,"external_id":null},{"id":10846386003,"name":"Haskell Indian Nations University","priority":564,"external_id":null},{"id":10846387003,"name":"Hastings College","priority":565,"external_id":null},{"id":10846388003,"name":"Haverford College","priority":566,"external_id":null},{"id":10846389003,"name":"Hawaii Pacific University","priority":567,"external_id":null},{"id":10846390003,"name":"Hebrew Theological College","priority":568,"external_id":null},{"id":10846391003,"name":"Heidelberg University","priority":569,"external_id":null},{"id":10846392003,"name":"Hellenic College","priority":570,"external_id":null},{"id":10846393003,"name":"Henderson State University","priority":571,"external_id":null},{"id":10846394003,"name":"Hendrix College","priority":572,"external_id":null},{"id":10846395003,"name":"Heritage University","priority":573,"external_id":null},{"id":10846396003,"name":"Herzing University","priority":574,"external_id":null},{"id":10846397003,"name":"Hesser College","priority":575,"external_id":null},{"id":10846398003,"name":"High Point University","priority":576,"external_id":null},{"id":10846399003,"name":"Hilbert College","priority":577,"external_id":null},{"id":10846400003,"name":"Hillsdale College","priority":578,"external_id":null},{"id":10846401003,"name":"Hiram College","priority":579,"external_id":null},{"id":10846402003,"name":"Hobart and William Smith Colleges","priority":580,"external_id":null},{"id":10846403003,"name":"Hodges University","priority":581,"external_id":null},{"id":10846404003,"name":"Hofstra University","priority":582,"external_id":null},{"id":10846405003,"name":"Hollins University","priority":583,"external_id":null},{"id":10846406003,"name":"Holy Apostles College and Seminary","priority":584,"external_id":null},{"id":10846407003,"name":"Indiana State University","priority":585,"external_id":null},{"id":10846408003,"name":"Holy Family University","priority":586,"external_id":null},{"id":10846409003,"name":"Holy Names University","priority":587,"external_id":null},{"id":10846410003,"name":"Hood College","priority":588,"external_id":null},{"id":10846411003,"name":"Hope College","priority":589,"external_id":null},{"id":10846412003,"name":"Hope International University","priority":590,"external_id":null},{"id":10846413003,"name":"Houghton College","priority":591,"external_id":null},{"id":10846414003,"name":"Howard Payne University","priority":592,"external_id":null},{"id":10846415003,"name":"Hult International Business School","priority":593,"external_id":null},{"id":10846416003,"name":"Humboldt State University","priority":594,"external_id":null},{"id":10846417003,"name":"Humphreys College","priority":595,"external_id":null},{"id":10846418003,"name":"Huntingdon College","priority":596,"external_id":null},{"id":10846419003,"name":"Huntington University","priority":597,"external_id":null},{"id":10846420003,"name":"Husson University","priority":598,"external_id":null},{"id":10846421003,"name":"Huston-Tillotson University","priority":599,"external_id":null},{"id":10846422003,"name":"Illinois College","priority":600,"external_id":null},{"id":10846423003,"name":"Illinois Institute of Art at Chicago","priority":601,"external_id":null},{"id":10846424003,"name":"Illinois Institute of Technology","priority":602,"external_id":null},{"id":10846425003,"name":"Illinois Wesleyan University","priority":603,"external_id":null},{"id":10846426003,"name":"Immaculata University","priority":604,"external_id":null},{"id":10846427003,"name":"Indian River State College","priority":605,"external_id":null},{"id":10846428003,"name":"Indiana Institute of Technology","priority":606,"external_id":null},{"id":10846429003,"name":"Indiana University East","priority":607,"external_id":null},{"id":10846430003,"name":"Indiana University Northwest","priority":608,"external_id":null},{"id":10846431003,"name":"Indiana University of Pennsylvania","priority":609,"external_id":null},{"id":10846432003,"name":"Indiana University Southeast","priority":610,"external_id":null},{"id":10846433003,"name":"Illinois State University","priority":611,"external_id":null},{"id":10846434003,"name":"Indiana University - Bloomington","priority":612,"external_id":null},{"id":10846435003,"name":"Davidson College","priority":613,"external_id":null},{"id":10846436003,"name":"Idaho State University","priority":614,"external_id":null},{"id":10846437003,"name":"Harvard University","priority":615,"external_id":null},{"id":10846438003,"name":"Howard University","priority":616,"external_id":null},{"id":10846439003,"name":"Houston Baptist University","priority":617,"external_id":null},{"id":10846440003,"name":"Indiana University - Kokomo","priority":618,"external_id":null},{"id":10846441003,"name":"Indiana University - South Bend","priority":619,"external_id":null},{"id":10846442003,"name":"Indiana University-Purdue University - Fort Wayne","priority":620,"external_id":null},{"id":10846443003,"name":"Indiana University-Purdue University - Indianapolis","priority":621,"external_id":null},{"id":10846444003,"name":"Indiana Wesleyan University","priority":622,"external_id":null},{"id":10846445003,"name":"Institute of American Indian and Alaska Native Culture and Arts Development","priority":623,"external_id":null},{"id":10846446003,"name":"Inter American University of Puerto Rico - Aguadilla","priority":624,"external_id":null},{"id":10846447003,"name":"Inter American University of Puerto Rico - Arecibo","priority":625,"external_id":null},{"id":10846448003,"name":"Inter American University of Puerto Rico - Barranquitas","priority":626,"external_id":null},{"id":10846449003,"name":"Inter American University of Puerto Rico - Bayamon","priority":627,"external_id":null},{"id":10846450003,"name":"Inter American University of Puerto Rico - Fajardo","priority":628,"external_id":null},{"id":10846451003,"name":"Inter American University of Puerto Rico - Guayama","priority":629,"external_id":null},{"id":10846452003,"name":"Inter American University of Puerto Rico - Metropolitan Campus","priority":630,"external_id":null},{"id":10846453003,"name":"Inter American University of Puerto Rico - Ponce","priority":631,"external_id":null},{"id":10846454003,"name":"Inter American University of Puerto Rico - San German","priority":632,"external_id":null},{"id":10846455003,"name":"International College of the Cayman Islands","priority":633,"external_id":null},{"id":10846456003,"name":"Iona College","priority":634,"external_id":null},{"id":10846457003,"name":"Iowa Wesleyan College","priority":635,"external_id":null},{"id":10846458003,"name":"Ithaca College","priority":636,"external_id":null},{"id":10846459003,"name":"Jarvis Christian College","priority":637,"external_id":null},{"id":10846460003,"name":"Jewish Theological Seminary of America","priority":638,"external_id":null},{"id":10846461003,"name":"John Brown University","priority":639,"external_id":null},{"id":10846462003,"name":"John Carroll University","priority":640,"external_id":null},{"id":10846463003,"name":"John F. Kennedy University","priority":641,"external_id":null},{"id":10846464003,"name":"Johns Hopkins University","priority":642,"external_id":null},{"id":10846465003,"name":"Johnson & Wales University","priority":643,"external_id":null},{"id":10846466003,"name":"Johnson C. Smith University","priority":644,"external_id":null},{"id":10846467003,"name":"Johnson State College","priority":645,"external_id":null},{"id":10846468003,"name":"Johnson University","priority":646,"external_id":null},{"id":10846469003,"name":"Jones International University","priority":647,"external_id":null},{"id":10846470003,"name":"Judson College","priority":648,"external_id":null},{"id":10846471003,"name":"Judson University","priority":649,"external_id":null},{"id":10846472003,"name":"Juilliard School","priority":650,"external_id":null},{"id":10846473003,"name":"Juniata College","priority":651,"external_id":null},{"id":10846474003,"name":"Kalamazoo College","priority":652,"external_id":null},{"id":10846475003,"name":"Kansas City Art Institute","priority":653,"external_id":null},{"id":10846476003,"name":"Kansas Wesleyan University","priority":654,"external_id":null},{"id":10846477003,"name":"Kaplan University","priority":655,"external_id":null},{"id":10846478003,"name":"Kean University","priority":656,"external_id":null},{"id":10846479003,"name":"Keene State College","priority":657,"external_id":null},{"id":10846480003,"name":"Keiser University","priority":658,"external_id":null},{"id":10846481003,"name":"Kendall College","priority":659,"external_id":null},{"id":10846482003,"name":"Kennesaw State University","priority":660,"external_id":null},{"id":10846483003,"name":"Kentucky Christian University","priority":661,"external_id":null},{"id":10846484003,"name":"Kentucky State University","priority":662,"external_id":null},{"id":10846485003,"name":"Kentucky Wesleyan College","priority":663,"external_id":null},{"id":10846486003,"name":"Kenyon College","priority":664,"external_id":null},{"id":10846487003,"name":"Kettering College","priority":665,"external_id":null},{"id":10846488003,"name":"Kettering University","priority":666,"external_id":null},{"id":10846489003,"name":"Keuka College","priority":667,"external_id":null},{"id":10846490003,"name":"Keystone College","priority":668,"external_id":null},{"id":10846491003,"name":"King University","priority":669,"external_id":null},{"id":10846492003,"name":"King's College","priority":670,"external_id":null},{"id":10846493003,"name":"Knox College","priority":671,"external_id":null},{"id":10846494003,"name":"Kutztown University of Pennsylvania","priority":672,"external_id":null},{"id":10846495003,"name":"Kuyper College","priority":673,"external_id":null},{"id":10846496003,"name":"La Roche College","priority":674,"external_id":null},{"id":10846497003,"name":"La Salle University","priority":675,"external_id":null},{"id":10846498003,"name":"La Sierra University","priority":676,"external_id":null},{"id":10846499003,"name":"LaGrange College","priority":677,"external_id":null},{"id":10846500003,"name":"Laguna College of Art and Design","priority":678,"external_id":null},{"id":10846501003,"name":"Lake Erie College","priority":679,"external_id":null},{"id":10846502003,"name":"Lake Forest College","priority":680,"external_id":null},{"id":10846503003,"name":"Lake Superior State University","priority":681,"external_id":null},{"id":10846504003,"name":"Lakeland College","priority":682,"external_id":null},{"id":10846505003,"name":"Lakeview College of Nursing","priority":683,"external_id":null},{"id":10846506003,"name":"Lancaster Bible College","priority":684,"external_id":null},{"id":10846507003,"name":"Lander University","priority":685,"external_id":null},{"id":10846508003,"name":"Lane College","priority":686,"external_id":null},{"id":10846509003,"name":"Langston University","priority":687,"external_id":null},{"id":10846510003,"name":"Lasell College","priority":688,"external_id":null},{"id":10846511003,"name":"Lawrence Technological University","priority":689,"external_id":null},{"id":10846512003,"name":"Lawrence University","priority":690,"external_id":null},{"id":10846513003,"name":"Le Moyne College","priority":691,"external_id":null},{"id":10846514003,"name":"Lebanon Valley College","priority":692,"external_id":null},{"id":10846515003,"name":"Lee University","priority":693,"external_id":null},{"id":10846516003,"name":"Lees-McRae College","priority":694,"external_id":null},{"id":10846517003,"name":"Kansas State University","priority":695,"external_id":null},{"id":10846518003,"name":"James Madison University","priority":696,"external_id":null},{"id":10846519003,"name":"Lafayette College","priority":697,"external_id":null},{"id":10846520003,"name":"Jacksonville University","priority":698,"external_id":null},{"id":10846521003,"name":"Kent State University","priority":699,"external_id":null},{"id":10846522003,"name":"Lamar University","priority":700,"external_id":null},{"id":10846523003,"name":"Jackson State University","priority":701,"external_id":null},{"id":10846524003,"name":"Lehigh University","priority":702,"external_id":null},{"id":10846525003,"name":"Jacksonville State University","priority":703,"external_id":null},{"id":10846526003,"name":"LeMoyne-Owen College","priority":704,"external_id":null},{"id":10846527003,"name":"Lenoir-Rhyne University","priority":705,"external_id":null},{"id":10846528003,"name":"Lesley University","priority":706,"external_id":null},{"id":10846529003,"name":"LeTourneau University","priority":707,"external_id":null},{"id":10846530003,"name":"Lewis & Clark College","priority":708,"external_id":null},{"id":10846531003,"name":"Lewis University","priority":709,"external_id":null},{"id":10846532003,"name":"Lewis-Clark State College","priority":710,"external_id":null},{"id":10846533003,"name":"Lexington College","priority":711,"external_id":null},{"id":10846534003,"name":"Life Pacific College","priority":712,"external_id":null},{"id":10846535003,"name":"Life University","priority":713,"external_id":null},{"id":10846536003,"name":"LIM College","priority":714,"external_id":null},{"id":10846537003,"name":"Limestone College","priority":715,"external_id":null},{"id":10846538003,"name":"Lincoln Christian University","priority":716,"external_id":null},{"id":10846539003,"name":"Lincoln College","priority":717,"external_id":null},{"id":10846540003,"name":"Lincoln Memorial University","priority":718,"external_id":null},{"id":10846541003,"name":"Lincoln University","priority":719,"external_id":null},{"id":10846542003,"name":"Lindenwood University","priority":720,"external_id":null},{"id":10846543003,"name":"Lindsey Wilson College","priority":721,"external_id":null},{"id":10846544003,"name":"Linfield College","priority":722,"external_id":null},{"id":10846545003,"name":"Lipscomb University","priority":723,"external_id":null},{"id":10846546003,"name":"LIU Post","priority":724,"external_id":null},{"id":10846547003,"name":"Livingstone College","priority":725,"external_id":null},{"id":10846548003,"name":"Lock Haven University of Pennsylvania","priority":726,"external_id":null},{"id":10846549003,"name":"Loma Linda University","priority":727,"external_id":null},{"id":10846550003,"name":"Longwood University","priority":728,"external_id":null},{"id":10846551003,"name":"Loras College","priority":729,"external_id":null},{"id":10846552003,"name":"Louisiana College","priority":730,"external_id":null},{"id":10846553003,"name":"Louisiana State University Health Sciences Center","priority":731,"external_id":null},{"id":10846554003,"name":"Louisiana State University - Alexandria","priority":732,"external_id":null},{"id":10846555003,"name":"Louisiana State University - Shreveport","priority":733,"external_id":null},{"id":10846556003,"name":"Lourdes University","priority":734,"external_id":null},{"id":10846557003,"name":"Loyola Marymount University","priority":735,"external_id":null},{"id":10846558003,"name":"Loyola University Chicago","priority":736,"external_id":null},{"id":10846559003,"name":"Loyola University Maryland","priority":737,"external_id":null},{"id":10846560003,"name":"Loyola University New Orleans","priority":738,"external_id":null},{"id":10846561003,"name":"Lubbock Christian University","priority":739,"external_id":null},{"id":10846562003,"name":"Luther College","priority":740,"external_id":null},{"id":10846563003,"name":"Lycoming College","priority":741,"external_id":null},{"id":10846564003,"name":"Lyme Academy College of Fine Arts","priority":742,"external_id":null},{"id":10846565003,"name":"Lynchburg College","priority":743,"external_id":null},{"id":10846566003,"name":"Lyndon State College","priority":744,"external_id":null},{"id":10846567003,"name":"Lynn University","priority":745,"external_id":null},{"id":10846568003,"name":"Lyon College","priority":746,"external_id":null},{"id":10846569003,"name":"Macalester College","priority":747,"external_id":null},{"id":10846570003,"name":"MacMurray College","priority":748,"external_id":null},{"id":10846571003,"name":"Madonna University","priority":749,"external_id":null},{"id":10846572003,"name":"Maharishi University of Management","priority":750,"external_id":null},{"id":10846573003,"name":"Maine College of Art","priority":751,"external_id":null},{"id":10846574003,"name":"Maine Maritime Academy","priority":752,"external_id":null},{"id":10846575003,"name":"Malone University","priority":753,"external_id":null},{"id":10846576003,"name":"Manchester University","priority":754,"external_id":null},{"id":10846577003,"name":"Manhattan Christian College","priority":755,"external_id":null},{"id":10846578003,"name":"Manhattan College","priority":756,"external_id":null},{"id":10846579003,"name":"Manhattan School of Music","priority":757,"external_id":null},{"id":10846580003,"name":"Manhattanville College","priority":758,"external_id":null},{"id":10846581003,"name":"Mansfield University of Pennsylvania","priority":759,"external_id":null},{"id":10846582003,"name":"Maranatha Baptist Bible College","priority":760,"external_id":null},{"id":10846583003,"name":"Marian University","priority":761,"external_id":null},{"id":10846584003,"name":"Marietta College","priority":762,"external_id":null},{"id":10846585003,"name":"Marlboro College","priority":763,"external_id":null},{"id":10846586003,"name":"Marquette University","priority":764,"external_id":null},{"id":10846587003,"name":"Mars Hill University","priority":765,"external_id":null},{"id":10846588003,"name":"Martin Luther College","priority":766,"external_id":null},{"id":10846589003,"name":"Martin Methodist College","priority":767,"external_id":null},{"id":10846590003,"name":"Martin University","priority":768,"external_id":null},{"id":10846591003,"name":"Mary Baldwin College","priority":769,"external_id":null},{"id":10846592003,"name":"Marygrove College","priority":770,"external_id":null},{"id":10846593003,"name":"Maryland Institute College of Art","priority":771,"external_id":null},{"id":10846594003,"name":"Marylhurst University","priority":772,"external_id":null},{"id":10846595003,"name":"Marymount Manhattan College","priority":773,"external_id":null},{"id":10846596003,"name":"Marymount University","priority":774,"external_id":null},{"id":10846597003,"name":"Maryville College","priority":775,"external_id":null},{"id":10846598003,"name":"Maryville University of St. Louis","priority":776,"external_id":null},{"id":10846599003,"name":"Marywood University","priority":777,"external_id":null},{"id":10846600003,"name":"Massachusetts College of Art and Design","priority":778,"external_id":null},{"id":10846601003,"name":"Massachusetts College of Liberal Arts","priority":779,"external_id":null},{"id":10846602003,"name":"Massachusetts College of Pharmacy and Health Sciences","priority":780,"external_id":null},{"id":10846603003,"name":"Massachusetts Institute of Technology","priority":781,"external_id":null},{"id":10846604003,"name":"Massachusetts Maritime Academy","priority":782,"external_id":null},{"id":10846605003,"name":"Master's College and Seminary","priority":783,"external_id":null},{"id":10846606003,"name":"Mayville State University","priority":784,"external_id":null},{"id":10846607003,"name":"McDaniel College","priority":785,"external_id":null},{"id":10846608003,"name":"McGill University","priority":786,"external_id":null},{"id":10846609003,"name":"McKendree University","priority":787,"external_id":null},{"id":10846610003,"name":"McMurry University","priority":788,"external_id":null},{"id":10846611003,"name":"McPherson College","priority":789,"external_id":null},{"id":10846612003,"name":"Medaille College","priority":790,"external_id":null},{"id":10846613003,"name":"Marist College","priority":791,"external_id":null},{"id":10846614003,"name":"McNeese State University","priority":792,"external_id":null},{"id":10846615003,"name":"Louisiana Tech University","priority":793,"external_id":null},{"id":10846616003,"name":"Marshall University","priority":794,"external_id":null},{"id":10846617003,"name":"Medical University of South Carolina","priority":795,"external_id":null},{"id":10846618003,"name":"Memorial University of Newfoundland","priority":796,"external_id":null},{"id":10846619003,"name":"Memphis College of Art","priority":797,"external_id":null},{"id":10846620003,"name":"Menlo College","priority":798,"external_id":null},{"id":10846621003,"name":"Mercy College","priority":799,"external_id":null},{"id":10846622003,"name":"Mercy College of Health Sciences","priority":800,"external_id":null},{"id":10846623003,"name":"Mercy College of Ohio","priority":801,"external_id":null},{"id":10846624003,"name":"Mercyhurst University","priority":802,"external_id":null},{"id":10846625003,"name":"Meredith College","priority":803,"external_id":null},{"id":10846626003,"name":"Merrimack College","priority":804,"external_id":null},{"id":10846627003,"name":"Messiah College","priority":805,"external_id":null},{"id":10846628003,"name":"Methodist University","priority":806,"external_id":null},{"id":10846629003,"name":"Metropolitan College of New York","priority":807,"external_id":null},{"id":10846630003,"name":"Metropolitan State University","priority":808,"external_id":null},{"id":10846631003,"name":"Metropolitan State University of Denver","priority":809,"external_id":null},{"id":10846632003,"name":"Miami Dade College","priority":810,"external_id":null},{"id":10846633003,"name":"Miami International University of Art & Design","priority":811,"external_id":null},{"id":10846634003,"name":"Michigan Technological University","priority":812,"external_id":null},{"id":10846635003,"name":"Mid-America Christian University","priority":813,"external_id":null},{"id":10846636003,"name":"Mid-Atlantic Christian University","priority":814,"external_id":null},{"id":10846637003,"name":"Mid-Continent University","priority":815,"external_id":null},{"id":10846638003,"name":"MidAmerica Nazarene University","priority":816,"external_id":null},{"id":10846639003,"name":"Middle Georgia State College","priority":817,"external_id":null},{"id":10846640003,"name":"Middlebury College","priority":818,"external_id":null},{"id":10846641003,"name":"Midland College","priority":819,"external_id":null},{"id":10846642003,"name":"Midland University","priority":820,"external_id":null},{"id":10846643003,"name":"Midstate College","priority":821,"external_id":null},{"id":10846644003,"name":"Midway College","priority":822,"external_id":null},{"id":10846645003,"name":"Midwestern State University","priority":823,"external_id":null},{"id":10846646003,"name":"Miles College","priority":824,"external_id":null},{"id":10846647003,"name":"Millersville University of Pennsylvania","priority":825,"external_id":null},{"id":10846648003,"name":"Milligan College","priority":826,"external_id":null},{"id":10846649003,"name":"Millikin University","priority":827,"external_id":null},{"id":10846650003,"name":"Mills College","priority":828,"external_id":null},{"id":10846651003,"name":"Millsaps College","priority":829,"external_id":null},{"id":10846652003,"name":"Milwaukee Institute of Art and Design","priority":830,"external_id":null},{"id":10846653003,"name":"Milwaukee School of Engineering","priority":831,"external_id":null},{"id":10846654003,"name":"Minneapolis College of Art and Design","priority":832,"external_id":null},{"id":10846655003,"name":"Minnesota State University - Mankato","priority":833,"external_id":null},{"id":10846656003,"name":"Minnesota State University - Moorhead","priority":834,"external_id":null},{"id":10846657003,"name":"Minot State University","priority":835,"external_id":null},{"id":10846658003,"name":"Misericordia University","priority":836,"external_id":null},{"id":10846659003,"name":"Mississippi College","priority":837,"external_id":null},{"id":10846660003,"name":"Mississippi University for Women","priority":838,"external_id":null},{"id":10846661003,"name":"Missouri Baptist University","priority":839,"external_id":null},{"id":10846662003,"name":"Missouri Southern State University","priority":840,"external_id":null},{"id":10846663003,"name":"Missouri University of Science & Technology","priority":841,"external_id":null},{"id":10846664003,"name":"Missouri Valley College","priority":842,"external_id":null},{"id":10846665003,"name":"Missouri Western State University","priority":843,"external_id":null},{"id":10846666003,"name":"Mitchell College","priority":844,"external_id":null},{"id":10846667003,"name":"Molloy College","priority":845,"external_id":null},{"id":10846668003,"name":"Monmouth College","priority":846,"external_id":null},{"id":10846669003,"name":"Monroe College","priority":847,"external_id":null},{"id":10846670003,"name":"Montana State University - Billings","priority":848,"external_id":null},{"id":10846671003,"name":"Montana State University - Northern","priority":849,"external_id":null},{"id":10846672003,"name":"Montana Tech of the University of Montana","priority":850,"external_id":null},{"id":10846673003,"name":"Montclair State University","priority":851,"external_id":null},{"id":10846674003,"name":"Monterrey Institute of Technology and Higher Education - Monterrey","priority":852,"external_id":null},{"id":10846675003,"name":"Montreat College","priority":853,"external_id":null},{"id":10846676003,"name":"Montserrat College of Art","priority":854,"external_id":null},{"id":10846677003,"name":"Moody Bible Institute","priority":855,"external_id":null},{"id":10846678003,"name":"Moore College of Art & Design","priority":856,"external_id":null},{"id":10846679003,"name":"Moravian College","priority":857,"external_id":null},{"id":10846680003,"name":"Morehouse College","priority":858,"external_id":null},{"id":10846681003,"name":"Morningside College","priority":859,"external_id":null},{"id":10846682003,"name":"Morris College","priority":860,"external_id":null},{"id":10846683003,"name":"Morrisville State College","priority":861,"external_id":null},{"id":10846684003,"name":"Mount Aloysius College","priority":862,"external_id":null},{"id":10846685003,"name":"Mount Angel Seminary","priority":863,"external_id":null},{"id":10846686003,"name":"Mount Carmel College of Nursing","priority":864,"external_id":null},{"id":10846687003,"name":"Mount Holyoke College","priority":865,"external_id":null},{"id":10846688003,"name":"Mount Ida College","priority":866,"external_id":null},{"id":10846689003,"name":"Mount Marty College","priority":867,"external_id":null},{"id":10846690003,"name":"Mount Mary University","priority":868,"external_id":null},{"id":10846691003,"name":"Mount Mercy University","priority":869,"external_id":null},{"id":10846692003,"name":"Mount Olive College","priority":870,"external_id":null},{"id":10846693003,"name":"Mississippi State University","priority":871,"external_id":null},{"id":10846694003,"name":"Montana State University","priority":872,"external_id":null},{"id":10846695003,"name":"Mississippi Valley State University","priority":873,"external_id":null},{"id":10846696003,"name":"Monmouth University","priority":874,"external_id":null},{"id":10846697003,"name":"Morehead State University","priority":875,"external_id":null},{"id":10846698003,"name":"Miami University - Oxford","priority":876,"external_id":null},{"id":10846699003,"name":"Morgan State University","priority":877,"external_id":null},{"id":10846700003,"name":"Missouri State University","priority":878,"external_id":null},{"id":10846701003,"name":"Michigan State University","priority":879,"external_id":null},{"id":10846702003,"name":"Mount St. Mary College","priority":880,"external_id":null},{"id":10846703003,"name":"Mount St. Mary's College","priority":881,"external_id":null},{"id":10846704003,"name":"Mount St. Mary's University","priority":882,"external_id":null},{"id":10846705003,"name":"Mount Vernon Nazarene University","priority":883,"external_id":null},{"id":10846706003,"name":"Muhlenberg College","priority":884,"external_id":null},{"id":10846707003,"name":"Multnomah University","priority":885,"external_id":null},{"id":10846708003,"name":"Muskingum University","priority":886,"external_id":null},{"id":10846709003,"name":"Naropa University","priority":887,"external_id":null},{"id":10846710003,"name":"National American University","priority":888,"external_id":null},{"id":10846711003,"name":"National Graduate School of Quality Management","priority":889,"external_id":null},{"id":10846712003,"name":"National Hispanic University","priority":890,"external_id":null},{"id":10846713003,"name":"National Labor College","priority":891,"external_id":null},{"id":10846714003,"name":"National University","priority":892,"external_id":null},{"id":10846715003,"name":"National-Louis University","priority":893,"external_id":null},{"id":10846716003,"name":"Nazarene Bible College","priority":894,"external_id":null},{"id":10846717003,"name":"Nazareth College","priority":895,"external_id":null},{"id":10846718003,"name":"Nebraska Methodist College","priority":896,"external_id":null},{"id":10846719003,"name":"Nebraska Wesleyan University","priority":897,"external_id":null},{"id":10846720003,"name":"Neumann University","priority":898,"external_id":null},{"id":10846721003,"name":"Nevada State College","priority":899,"external_id":null},{"id":10846722003,"name":"New College of Florida","priority":900,"external_id":null},{"id":10846723003,"name":"New England College","priority":901,"external_id":null},{"id":10846724003,"name":"New England Conservatory of Music","priority":902,"external_id":null},{"id":10846725003,"name":"New England Institute of Art","priority":903,"external_id":null},{"id":10846726003,"name":"New England Institute of Technology","priority":904,"external_id":null},{"id":10846727003,"name":"New Jersey City University","priority":905,"external_id":null},{"id":10846728003,"name":"New Jersey Institute of Technology","priority":906,"external_id":null},{"id":10846729003,"name":"New Mexico Highlands University","priority":907,"external_id":null},{"id":10846730003,"name":"New Mexico Institute of Mining and Technology","priority":908,"external_id":null},{"id":10846731003,"name":"New Orleans Baptist Theological Seminary","priority":909,"external_id":null},{"id":10846732003,"name":"New School","priority":910,"external_id":null},{"id":10846733003,"name":"New York Institute of Technology","priority":911,"external_id":null},{"id":10846734003,"name":"New York University","priority":912,"external_id":null},{"id":10846735003,"name":"Newberry College","priority":913,"external_id":null},{"id":10846736003,"name":"Newbury College","priority":914,"external_id":null},{"id":10846737003,"name":"Newman University","priority":915,"external_id":null},{"id":10846738003,"name":"Niagara University","priority":916,"external_id":null},{"id":10846739003,"name":"Nichols College","priority":917,"external_id":null},{"id":10846740003,"name":"North Carolina Wesleyan College","priority":918,"external_id":null},{"id":10846741003,"name":"North Central College","priority":919,"external_id":null},{"id":10846742003,"name":"North Central University","priority":920,"external_id":null},{"id":10846743003,"name":"North Greenville University","priority":921,"external_id":null},{"id":10846744003,"name":"North Park University","priority":922,"external_id":null},{"id":10846745003,"name":"Northcentral University","priority":923,"external_id":null},{"id":10846746003,"name":"Northeastern Illinois University","priority":924,"external_id":null},{"id":10846747003,"name":"Northeastern State University","priority":925,"external_id":null},{"id":10846748003,"name":"Northeastern University","priority":926,"external_id":null},{"id":10846749003,"name":"Northern Kentucky University","priority":927,"external_id":null},{"id":10846750003,"name":"Northern Michigan University","priority":928,"external_id":null},{"id":10846751003,"name":"Northern New Mexico College","priority":929,"external_id":null},{"id":10846752003,"name":"Northern State University","priority":930,"external_id":null},{"id":10846753003,"name":"Northland College","priority":931,"external_id":null},{"id":10846754003,"name":"Northwest Christian University","priority":932,"external_id":null},{"id":10846755003,"name":"Northwest Florida State College","priority":933,"external_id":null},{"id":10846756003,"name":"Northwest Missouri State University","priority":934,"external_id":null},{"id":10846757003,"name":"Northwest Nazarene University","priority":935,"external_id":null},{"id":10846758003,"name":"Northwest University","priority":936,"external_id":null},{"id":10846759003,"name":"Northwestern College","priority":937,"external_id":null},{"id":10846760003,"name":"Northwestern Health Sciences University","priority":938,"external_id":null},{"id":10846761003,"name":"Northwestern Oklahoma State University","priority":939,"external_id":null},{"id":10846762003,"name":"Northwood University","priority":940,"external_id":null},{"id":10846763003,"name":"Norwich University","priority":941,"external_id":null},{"id":10846764003,"name":"Notre Dame College of Ohio","priority":942,"external_id":null},{"id":10846765003,"name":"Notre Dame de Namur University","priority":943,"external_id":null},{"id":10846766003,"name":"Notre Dame of Maryland University","priority":944,"external_id":null},{"id":10846767003,"name":"Nova Scotia College of Art and Design","priority":945,"external_id":null},{"id":10846768003,"name":"Nova Southeastern University","priority":946,"external_id":null},{"id":10846769003,"name":"Nyack College","priority":947,"external_id":null},{"id":10846770003,"name":"Oakland City University","priority":948,"external_id":null},{"id":10846771003,"name":"Oakland University","priority":949,"external_id":null},{"id":10846772003,"name":"Oakwood University","priority":950,"external_id":null},{"id":10846773003,"name":"Oberlin College","priority":951,"external_id":null},{"id":10846774003,"name":"Occidental College","priority":952,"external_id":null},{"id":10846775003,"name":"Oglala Lakota College","priority":953,"external_id":null},{"id":10846776003,"name":"North Carolina A&T State University","priority":954,"external_id":null},{"id":10846777003,"name":"Northern Illinois University","priority":955,"external_id":null},{"id":10846778003,"name":"North Dakota State University","priority":956,"external_id":null},{"id":10846779003,"name":"Nicholls State University","priority":957,"external_id":null},{"id":10846780003,"name":"North Carolina Central University","priority":958,"external_id":null},{"id":10846781003,"name":"Norfolk State University","priority":959,"external_id":null},{"id":10846782003,"name":"Northwestern State University of Louisiana","priority":960,"external_id":null},{"id":10846783003,"name":"Northern Arizona University","priority":961,"external_id":null},{"id":10846784003,"name":"North Carolina State University - Raleigh","priority":962,"external_id":null},{"id":10846785003,"name":"Northwestern University","priority":963,"external_id":null},{"id":10846786003,"name":"Oglethorpe University","priority":964,"external_id":null},{"id":10846787003,"name":"Ohio Christian University","priority":965,"external_id":null},{"id":10846788003,"name":"Ohio Dominican University","priority":966,"external_id":null},{"id":10846789003,"name":"Ohio Northern University","priority":967,"external_id":null},{"id":10846790003,"name":"Ohio Valley University","priority":968,"external_id":null},{"id":10846791003,"name":"Ohio Wesleyan University","priority":969,"external_id":null},{"id":10846792003,"name":"Oklahoma Baptist University","priority":970,"external_id":null},{"id":10846793003,"name":"Oklahoma Christian University","priority":971,"external_id":null},{"id":10846794003,"name":"Oklahoma City University","priority":972,"external_id":null},{"id":10846795003,"name":"Oklahoma Panhandle State University","priority":973,"external_id":null},{"id":10846796003,"name":"Oklahoma State University Institute of Technology - Okmulgee","priority":974,"external_id":null},{"id":10846797003,"name":"Oklahoma State University - Oklahoma City","priority":975,"external_id":null},{"id":10846798003,"name":"Oklahoma Wesleyan University","priority":976,"external_id":null},{"id":10846799003,"name":"Olivet College","priority":977,"external_id":null},{"id":10846800003,"name":"Olivet Nazarene University","priority":978,"external_id":null},{"id":10846801003,"name":"Olympic College","priority":979,"external_id":null},{"id":10846802003,"name":"Oral Roberts University","priority":980,"external_id":null},{"id":10846803003,"name":"Oregon College of Art and Craft","priority":981,"external_id":null},{"id":10846804003,"name":"Oregon Health and Science University","priority":982,"external_id":null},{"id":10846805003,"name":"Oregon Institute of Technology","priority":983,"external_id":null},{"id":10846806003,"name":"Otis College of Art and Design","priority":984,"external_id":null},{"id":10846807003,"name":"Ottawa University","priority":985,"external_id":null},{"id":10846808003,"name":"Otterbein University","priority":986,"external_id":null},{"id":10846809003,"name":"Ouachita Baptist University","priority":987,"external_id":null},{"id":10846810003,"name":"Our Lady of Holy Cross College","priority":988,"external_id":null},{"id":10846811003,"name":"Our Lady of the Lake College","priority":989,"external_id":null},{"id":10846812003,"name":"Our Lady of the Lake University","priority":990,"external_id":null},{"id":10846813003,"name":"Pace University","priority":991,"external_id":null},{"id":10846814003,"name":"Pacific Lutheran University","priority":992,"external_id":null},{"id":10846815003,"name":"Pacific Northwest College of Art","priority":993,"external_id":null},{"id":10846816003,"name":"Pacific Oaks College","priority":994,"external_id":null},{"id":10846817003,"name":"Pacific Union College","priority":995,"external_id":null},{"id":10846818003,"name":"Pacific University","priority":996,"external_id":null},{"id":10846819003,"name":"Paine College","priority":997,"external_id":null},{"id":10846820003,"name":"Palm Beach Atlantic University","priority":998,"external_id":null},{"id":10846821003,"name":"Palmer College of Chiropractic","priority":999,"external_id":null},{"id":10846822003,"name":"Park University","priority":1000,"external_id":null},{"id":10846823003,"name":"Parker University","priority":1001,"external_id":null},{"id":10846824003,"name":"Patten University","priority":1002,"external_id":null},{"id":10846825003,"name":"Paul Smith's College","priority":1003,"external_id":null},{"id":10846826003,"name":"Peirce College","priority":1004,"external_id":null},{"id":10846827003,"name":"Peninsula College","priority":1005,"external_id":null},{"id":10846828003,"name":"Pennsylvania College of Art and Design","priority":1006,"external_id":null},{"id":10846829003,"name":"Pennsylvania College of Technology","priority":1007,"external_id":null},{"id":10846830003,"name":"Pennsylvania State University - Erie, The Behrend College","priority":1008,"external_id":null},{"id":10846831003,"name":"Pennsylvania State University - Harrisburg","priority":1009,"external_id":null},{"id":10846832003,"name":"Pepperdine University","priority":1010,"external_id":null},{"id":10846833003,"name":"Peru State College","priority":1011,"external_id":null},{"id":10846834003,"name":"Pfeiffer University","priority":1012,"external_id":null},{"id":10846835003,"name":"Philadelphia University","priority":1013,"external_id":null},{"id":10846836003,"name":"Philander Smith College","priority":1014,"external_id":null},{"id":10846837003,"name":"Piedmont College","priority":1015,"external_id":null},{"id":10846838003,"name":"Pine Manor College","priority":1016,"external_id":null},{"id":10846839003,"name":"Pittsburg State University","priority":1017,"external_id":null},{"id":10846840003,"name":"Pitzer College","priority":1018,"external_id":null},{"id":10846841003,"name":"Plaza College","priority":1019,"external_id":null},{"id":10846842003,"name":"Plymouth State University","priority":1020,"external_id":null},{"id":10846843003,"name":"Point Loma Nazarene University","priority":1021,"external_id":null},{"id":10846844003,"name":"Point Park University","priority":1022,"external_id":null},{"id":10846845003,"name":"Point University","priority":1023,"external_id":null},{"id":10846846003,"name":"Polytechnic Institute of New York University","priority":1024,"external_id":null},{"id":10846847003,"name":"Pomona College","priority":1025,"external_id":null},{"id":10846848003,"name":"Pontifical Catholic University of Puerto Rico","priority":1026,"external_id":null},{"id":10846849003,"name":"Pontifical College Josephinum","priority":1027,"external_id":null},{"id":10846850003,"name":"Post University","priority":1028,"external_id":null},{"id":10846851003,"name":"Potomac College","priority":1029,"external_id":null},{"id":10846852003,"name":"Pratt Institute","priority":1030,"external_id":null},{"id":10846853003,"name":"Prescott College","priority":1031,"external_id":null},{"id":10846854003,"name":"Presentation College","priority":1032,"external_id":null},{"id":10846855003,"name":"Principia College","priority":1033,"external_id":null},{"id":10846856003,"name":"Providence College","priority":1034,"external_id":null},{"id":10846857003,"name":"Puerto Rico Conservatory of Music","priority":1035,"external_id":null},{"id":10846858003,"name":"Purchase College - SUNY","priority":1036,"external_id":null},{"id":10846859003,"name":"Purdue University - Calumet","priority":1037,"external_id":null},{"id":10846860003,"name":"Purdue University - North Central","priority":1038,"external_id":null},{"id":10846861003,"name":"Queens University of Charlotte","priority":1039,"external_id":null},{"id":10846862003,"name":"Oklahoma State University","priority":1040,"external_id":null},{"id":10846863003,"name":"Oregon State University","priority":1041,"external_id":null},{"id":10846864003,"name":"Portland State University","priority":1042,"external_id":null},{"id":10846865003,"name":"Old Dominion University","priority":1043,"external_id":null},{"id":10846866003,"name":"Prairie View A&M University","priority":1044,"external_id":null},{"id":10846867003,"name":"Presbyterian College","priority":1045,"external_id":null},{"id":10846868003,"name":"Purdue University - West Lafayette","priority":1046,"external_id":null},{"id":10846869003,"name":"Ohio University","priority":1047,"external_id":null},{"id":10846870003,"name":"Princeton University","priority":1048,"external_id":null},{"id":10846871003,"name":"Quincy University","priority":1049,"external_id":null},{"id":10846872003,"name":"Quinnipiac University","priority":1050,"external_id":null},{"id":10846873003,"name":"Radford University","priority":1051,"external_id":null},{"id":10846874003,"name":"Ramapo College of New Jersey","priority":1052,"external_id":null},{"id":10846875003,"name":"Randolph College","priority":1053,"external_id":null},{"id":10846876003,"name":"Randolph-Macon College","priority":1054,"external_id":null},{"id":10846877003,"name":"Ranken Technical College","priority":1055,"external_id":null},{"id":10846878003,"name":"Reed College","priority":1056,"external_id":null},{"id":10846879003,"name":"Regent University","priority":1057,"external_id":null},{"id":10846880003,"name":"Regent's American College London","priority":1058,"external_id":null},{"id":10846881003,"name":"Regis College","priority":1059,"external_id":null},{"id":10846882003,"name":"Regis University","priority":1060,"external_id":null},{"id":10846883003,"name":"Reinhardt University","priority":1061,"external_id":null},{"id":10846884003,"name":"Rensselaer Polytechnic Institute","priority":1062,"external_id":null},{"id":10846885003,"name":"Research College of Nursing","priority":1063,"external_id":null},{"id":10846886003,"name":"Resurrection University","priority":1064,"external_id":null},{"id":10846887003,"name":"Rhode Island College","priority":1065,"external_id":null},{"id":10846888003,"name":"Rhode Island School of Design","priority":1066,"external_id":null},{"id":10846889003,"name":"Rhodes College","priority":1067,"external_id":null},{"id":10846890003,"name":"Richard Stockton College of New Jersey","priority":1068,"external_id":null},{"id":10846891003,"name":"Richmond - The American International University in London","priority":1069,"external_id":null},{"id":10846892003,"name":"Rider University","priority":1070,"external_id":null},{"id":10846893003,"name":"Ringling College of Art and Design","priority":1071,"external_id":null},{"id":10846894003,"name":"Ripon College","priority":1072,"external_id":null},{"id":10846895003,"name":"Rivier University","priority":1073,"external_id":null},{"id":10846896003,"name":"Roanoke College","priority":1074,"external_id":null},{"id":10846897003,"name":"Robert B. Miller College","priority":1075,"external_id":null},{"id":10846898003,"name":"Roberts Wesleyan College","priority":1076,"external_id":null},{"id":10846899003,"name":"Rochester College","priority":1077,"external_id":null},{"id":10846900003,"name":"Rochester Institute of Technology","priority":1078,"external_id":null},{"id":10846901003,"name":"Rockford University","priority":1079,"external_id":null},{"id":10846902003,"name":"Rockhurst University","priority":1080,"external_id":null},{"id":10846903003,"name":"Rocky Mountain College","priority":1081,"external_id":null},{"id":10846904003,"name":"Rocky Mountain College of Art and Design","priority":1082,"external_id":null},{"id":10846905003,"name":"Roger Williams University","priority":1083,"external_id":null},{"id":10846906003,"name":"Rogers State University","priority":1084,"external_id":null},{"id":10846907003,"name":"Rollins College","priority":1085,"external_id":null},{"id":10846908003,"name":"Roosevelt University","priority":1086,"external_id":null},{"id":10846909003,"name":"Rosalind Franklin University of Medicine and Science","priority":1087,"external_id":null},{"id":10846910003,"name":"Rose-Hulman Institute of Technology","priority":1088,"external_id":null},{"id":10846911003,"name":"Rosemont College","priority":1089,"external_id":null},{"id":10846912003,"name":"Rowan University","priority":1090,"external_id":null},{"id":10846913003,"name":"Rush University","priority":1091,"external_id":null},{"id":10846914003,"name":"Rust College","priority":1092,"external_id":null},{"id":10846915003,"name":"Rutgers, the State University of New Jersey - Camden","priority":1093,"external_id":null},{"id":10846916003,"name":"Rutgers, the State University of New Jersey - Newark","priority":1094,"external_id":null},{"id":10846917003,"name":"Ryerson University","priority":1095,"external_id":null},{"id":10846918003,"name":"Sacred Heart Major Seminary","priority":1096,"external_id":null},{"id":10846919003,"name":"Saginaw Valley State University","priority":1097,"external_id":null},{"id":10846920003,"name":"Salem College","priority":1098,"external_id":null},{"id":10846921003,"name":"Salem International University","priority":1099,"external_id":null},{"id":10846922003,"name":"Salem State University","priority":1100,"external_id":null},{"id":10846923003,"name":"Salisbury University","priority":1101,"external_id":null},{"id":10846924003,"name":"Salish Kootenai College","priority":1102,"external_id":null},{"id":10846925003,"name":"Salve Regina University","priority":1103,"external_id":null},{"id":10846926003,"name":"Samuel Merritt University","priority":1104,"external_id":null},{"id":10846927003,"name":"San Diego Christian College","priority":1105,"external_id":null},{"id":10846928003,"name":"San Francisco Art Institute","priority":1106,"external_id":null},{"id":10846929003,"name":"San Francisco Conservatory of Music","priority":1107,"external_id":null},{"id":10846930003,"name":"San Francisco State University","priority":1108,"external_id":null},{"id":10846931003,"name":"Sanford College of Nursing","priority":1109,"external_id":null},{"id":10846932003,"name":"Santa Clara University","priority":1110,"external_id":null},{"id":10846933003,"name":"Santa Fe University of Art and Design","priority":1111,"external_id":null},{"id":10846934003,"name":"Sarah Lawrence College","priority":1112,"external_id":null},{"id":10846935003,"name":"Savannah College of Art and Design","priority":1113,"external_id":null},{"id":10846936003,"name":"School of the Art Institute of Chicago","priority":1114,"external_id":null},{"id":10846937003,"name":"School of Visual Arts","priority":1115,"external_id":null},{"id":10846938003,"name":"Schreiner University","priority":1116,"external_id":null},{"id":10846939003,"name":"Scripps College","priority":1117,"external_id":null},{"id":10846940003,"name":"Seattle Pacific University","priority":1118,"external_id":null},{"id":10846941003,"name":"Seattle University","priority":1119,"external_id":null},{"id":10846942003,"name":"Seton Hall University","priority":1120,"external_id":null},{"id":10846943003,"name":"Seton Hill University","priority":1121,"external_id":null},{"id":10846944003,"name":"Sewanee - University of the South","priority":1122,"external_id":null},{"id":10846945003,"name":"Shaw University","priority":1123,"external_id":null},{"id":10846946003,"name":"Shawnee State University","priority":1124,"external_id":null},{"id":10846947003,"name":"Shenandoah University","priority":1125,"external_id":null},{"id":10846948003,"name":"Shepherd University","priority":1126,"external_id":null},{"id":10846949003,"name":"Shimer College","priority":1127,"external_id":null},{"id":10846950003,"name":"Sacred Heart University","priority":1128,"external_id":null},{"id":10846951003,"name":"Robert Morris University","priority":1129,"external_id":null},{"id":10846952003,"name":"Sam Houston State University","priority":1130,"external_id":null},{"id":10846953003,"name":"Samford University","priority":1131,"external_id":null},{"id":10846954003,"name":"Savannah State University","priority":1132,"external_id":null},{"id":10846955003,"name":"San Jose State University","priority":1133,"external_id":null},{"id":10846956003,"name":"Rutgers, the State University of New Jersey - New Brunswick","priority":1134,"external_id":null},{"id":10846957003,"name":"San Diego State University","priority":1135,"external_id":null},{"id":10846958003,"name":"Shippensburg University of Pennsylvania","priority":1136,"external_id":null},{"id":10846959003,"name":"Shorter University","priority":1137,"external_id":null},{"id":10846960003,"name":"Siena College","priority":1138,"external_id":null},{"id":10846961003,"name":"Siena Heights University","priority":1139,"external_id":null},{"id":10846962003,"name":"Sierra Nevada College","priority":1140,"external_id":null},{"id":10846963003,"name":"Silver Lake College","priority":1141,"external_id":null},{"id":10846964003,"name":"Simmons College","priority":1142,"external_id":null},{"id":10846965003,"name":"Simon Fraser University","priority":1143,"external_id":null},{"id":10846966003,"name":"Simpson College","priority":1144,"external_id":null},{"id":10846967003,"name":"Simpson University","priority":1145,"external_id":null},{"id":10846968003,"name":"Sinte Gleska University","priority":1146,"external_id":null},{"id":10846969003,"name":"Sitting Bull College","priority":1147,"external_id":null},{"id":10846970003,"name":"Skidmore College","priority":1148,"external_id":null},{"id":10846971003,"name":"Slippery Rock University of Pennsylvania","priority":1149,"external_id":null},{"id":10846972003,"name":"Smith College","priority":1150,"external_id":null},{"id":10846973003,"name":"Sojourner-Douglass College","priority":1151,"external_id":null},{"id":10846974003,"name":"Soka University of America","priority":1152,"external_id":null},{"id":10846975003,"name":"Sonoma State University","priority":1153,"external_id":null},{"id":10846976003,"name":"South College","priority":1154,"external_id":null},{"id":10846977003,"name":"South Dakota School of Mines and Technology","priority":1155,"external_id":null},{"id":10846978003,"name":"South Seattle Community College","priority":1156,"external_id":null},{"id":10846979003,"name":"South Texas College","priority":1157,"external_id":null},{"id":10846980003,"name":"South University","priority":1158,"external_id":null},{"id":10846981003,"name":"Southeastern Oklahoma State University","priority":1159,"external_id":null},{"id":10846982003,"name":"Southeastern University","priority":1160,"external_id":null},{"id":10846983003,"name":"Southern Adventist University","priority":1161,"external_id":null},{"id":10846984003,"name":"Southern Arkansas University","priority":1162,"external_id":null},{"id":10846985003,"name":"Southern Baptist Theological Seminary","priority":1163,"external_id":null},{"id":10846986003,"name":"Southern California Institute of Architecture","priority":1164,"external_id":null},{"id":10846987003,"name":"Southern Connecticut State University","priority":1165,"external_id":null},{"id":10846988003,"name":"Southern Illinois University - Edwardsville","priority":1166,"external_id":null},{"id":10846989003,"name":"Southern Nazarene University","priority":1167,"external_id":null},{"id":10846990003,"name":"Southern New Hampshire University","priority":1168,"external_id":null},{"id":10846991003,"name":"Southern Oregon University","priority":1169,"external_id":null},{"id":10846992003,"name":"Southern Polytechnic State University","priority":1170,"external_id":null},{"id":10846993003,"name":"Southern University - New Orleans","priority":1171,"external_id":null},{"id":10846994003,"name":"Southern Vermont College","priority":1172,"external_id":null},{"id":10846995003,"name":"Southern Wesleyan University","priority":1173,"external_id":null},{"id":10846996003,"name":"Southwest Baptist University","priority":1174,"external_id":null},{"id":10846997003,"name":"Southwest Minnesota State University","priority":1175,"external_id":null},{"id":10846998003,"name":"Southwest University of Visual Arts","priority":1176,"external_id":null},{"id":10846999003,"name":"Southwestern Adventist University","priority":1177,"external_id":null},{"id":10847000003,"name":"Southwestern Assemblies of God University","priority":1178,"external_id":null},{"id":10847001003,"name":"Southwestern Christian College","priority":1179,"external_id":null},{"id":10847002003,"name":"Southwestern Christian University","priority":1180,"external_id":null},{"id":10847003003,"name":"Southwestern College","priority":1181,"external_id":null},{"id":10847004003,"name":"Southwestern Oklahoma State University","priority":1182,"external_id":null},{"id":10847005003,"name":"Southwestern University","priority":1183,"external_id":null},{"id":10847006003,"name":"Spalding University","priority":1184,"external_id":null},{"id":10847007003,"name":"Spelman College","priority":1185,"external_id":null},{"id":10847008003,"name":"Spring Arbor University","priority":1186,"external_id":null},{"id":10847009003,"name":"Spring Hill College","priority":1187,"external_id":null},{"id":10847010003,"name":"Springfield College","priority":1188,"external_id":null},{"id":10847011003,"name":"St. Ambrose University","priority":1189,"external_id":null},{"id":10847012003,"name":"St. Anselm College","priority":1190,"external_id":null},{"id":10847013003,"name":"St. Anthony College of Nursing","priority":1191,"external_id":null},{"id":10847014003,"name":"St. Augustine College","priority":1192,"external_id":null},{"id":10847015003,"name":"St. Augustine's University","priority":1193,"external_id":null},{"id":10847016003,"name":"St. Bonaventure University","priority":1194,"external_id":null},{"id":10847017003,"name":"St. Catharine College","priority":1195,"external_id":null},{"id":10847018003,"name":"St. Catherine University","priority":1196,"external_id":null},{"id":10847019003,"name":"St. Charles Borromeo Seminary","priority":1197,"external_id":null},{"id":10847020003,"name":"St. Cloud State University","priority":1198,"external_id":null},{"id":10847021003,"name":"St. Edward's University","priority":1199,"external_id":null},{"id":10847022003,"name":"St. Francis College","priority":1200,"external_id":null},{"id":10847023003,"name":"St. Francis Medical Center College of Nursing","priority":1201,"external_id":null},{"id":10847024003,"name":"St. Gregory's University","priority":1202,"external_id":null},{"id":10847025003,"name":"St. John Fisher College","priority":1203,"external_id":null},{"id":10847026003,"name":"St. John Vianney College Seminary","priority":1204,"external_id":null},{"id":10847027003,"name":"St. John's College","priority":1205,"external_id":null},{"id":10847028003,"name":"St. John's University","priority":1206,"external_id":null},{"id":10847029003,"name":"St. Joseph Seminary College","priority":1207,"external_id":null},{"id":10847030003,"name":"St. Joseph's College","priority":1208,"external_id":null},{"id":10847031003,"name":"St. Joseph's College New York","priority":1209,"external_id":null},{"id":10847032003,"name":"St. Joseph's University","priority":1210,"external_id":null},{"id":10847033003,"name":"St. Lawrence University","priority":1211,"external_id":null},{"id":10847034003,"name":"St. Leo University","priority":1212,"external_id":null},{"id":10847035003,"name":"Southern University and A&M College","priority":1213,"external_id":null},{"id":10847036003,"name":"Southern Methodist University","priority":1214,"external_id":null},{"id":10847037003,"name":"Southeast Missouri State University","priority":1215,"external_id":null},{"id":10847038003,"name":"Southern Utah University","priority":1216,"external_id":null},{"id":10847039003,"name":"South Dakota State University","priority":1217,"external_id":null},{"id":10847040003,"name":"St. Francis University","priority":1218,"external_id":null},{"id":10847041003,"name":"Southeastern Louisiana University","priority":1219,"external_id":null},{"id":10847042003,"name":"Southern Illinois University - Carbondale","priority":1220,"external_id":null},{"id":10847043003,"name":"St. Louis College of Pharmacy","priority":1221,"external_id":null},{"id":10847044003,"name":"St. Louis University","priority":1222,"external_id":null},{"id":10847045003,"name":"St. Luke's College of Health Sciences","priority":1223,"external_id":null},{"id":10847046003,"name":"St. Martin's University","priority":1224,"external_id":null},{"id":10847047003,"name":"St. Mary's College","priority":1225,"external_id":null},{"id":10847048003,"name":"St. Mary's College of California","priority":1226,"external_id":null},{"id":10847049003,"name":"St. Mary's College of Maryland","priority":1227,"external_id":null},{"id":10847050003,"name":"St. Mary's Seminary and University","priority":1228,"external_id":null},{"id":10847051003,"name":"St. Mary's University of Minnesota","priority":1229,"external_id":null},{"id":10847052003,"name":"St. Mary's University of San Antonio","priority":1230,"external_id":null},{"id":10847053003,"name":"St. Mary-of-the-Woods College","priority":1231,"external_id":null},{"id":10847054003,"name":"St. Michael's College","priority":1232,"external_id":null},{"id":10847055003,"name":"St. Norbert College","priority":1233,"external_id":null},{"id":10847056003,"name":"St. Olaf College","priority":1234,"external_id":null},{"id":10847057003,"name":"St. Paul's College","priority":1235,"external_id":null},{"id":10847058003,"name":"St. Peter's University","priority":1236,"external_id":null},{"id":10847059003,"name":"St. Petersburg College","priority":1237,"external_id":null},{"id":10847060003,"name":"St. Thomas Aquinas College","priority":1238,"external_id":null},{"id":10847061003,"name":"St. Thomas University","priority":1239,"external_id":null},{"id":10847062003,"name":"St. Vincent College","priority":1240,"external_id":null},{"id":10847063003,"name":"St. Xavier University","priority":1241,"external_id":null},{"id":10847064003,"name":"Stephens College","priority":1242,"external_id":null},{"id":10847065003,"name":"Sterling College","priority":1243,"external_id":null},{"id":10847066003,"name":"Stevens Institute of Technology","priority":1244,"external_id":null},{"id":10847067003,"name":"Stevenson University","priority":1245,"external_id":null},{"id":10847068003,"name":"Stillman College","priority":1246,"external_id":null},{"id":10847069003,"name":"Stonehill College","priority":1247,"external_id":null},{"id":10847070003,"name":"Strayer University","priority":1248,"external_id":null},{"id":10847071003,"name":"Suffolk University","priority":1249,"external_id":null},{"id":10847072003,"name":"Sul Ross State University","priority":1250,"external_id":null},{"id":10847073003,"name":"Sullivan University","priority":1251,"external_id":null},{"id":10847074003,"name":"SUNY Buffalo State","priority":1252,"external_id":null},{"id":10847075003,"name":"SUNY College of Agriculture and Technology - Cobleskill","priority":1253,"external_id":null},{"id":10847076003,"name":"SUNY College of Environmental Science and Forestry","priority":1254,"external_id":null},{"id":10847077003,"name":"SUNY College of Technology - Alfred","priority":1255,"external_id":null},{"id":10847078003,"name":"SUNY College of Technology - Canton","priority":1256,"external_id":null},{"id":10847079003,"name":"SUNY College of Technology - Delhi","priority":1257,"external_id":null},{"id":10847080003,"name":"SUNY College - Cortland","priority":1258,"external_id":null},{"id":10847081003,"name":"SUNY College - Old Westbury","priority":1259,"external_id":null},{"id":10847082003,"name":"SUNY College - Oneonta","priority":1260,"external_id":null},{"id":10847083003,"name":"SUNY College - Potsdam","priority":1261,"external_id":null},{"id":10847084003,"name":"SUNY Downstate Medical Center","priority":1262,"external_id":null},{"id":10847085003,"name":"SUNY Empire State College","priority":1263,"external_id":null},{"id":10847086003,"name":"SUNY Institute of Technology - Utica/Rome","priority":1264,"external_id":null},{"id":10847087003,"name":"SUNY Maritime College","priority":1265,"external_id":null},{"id":10847088003,"name":"SUNY Upstate Medical University","priority":1266,"external_id":null},{"id":10847089003,"name":"SUNY - Fredonia","priority":1267,"external_id":null},{"id":10847090003,"name":"SUNY - Geneseo","priority":1268,"external_id":null},{"id":10847091003,"name":"SUNY - New Paltz","priority":1269,"external_id":null},{"id":10847092003,"name":"SUNY - Oswego","priority":1270,"external_id":null},{"id":10847093003,"name":"SUNY - Plattsburgh","priority":1271,"external_id":null},{"id":10847094003,"name":"Swarthmore College","priority":1272,"external_id":null},{"id":10847095003,"name":"Sweet Briar College","priority":1273,"external_id":null},{"id":10847096003,"name":"Tabor College","priority":1274,"external_id":null},{"id":10847097003,"name":"Talladega College","priority":1275,"external_id":null},{"id":10847098003,"name":"Tarleton State University","priority":1276,"external_id":null},{"id":10847099003,"name":"Taylor University","priority":1277,"external_id":null},{"id":10847100003,"name":"Tennessee Wesleyan College","priority":1278,"external_id":null},{"id":10847101003,"name":"Texas A&M International University","priority":1279,"external_id":null},{"id":10847102003,"name":"Texas A&M University - Commerce","priority":1280,"external_id":null},{"id":10847103003,"name":"Texas A&M University - Corpus Christi","priority":1281,"external_id":null},{"id":10847104003,"name":"Texas A&M University - Galveston","priority":1282,"external_id":null},{"id":10847105003,"name":"Texas A&M University - Kingsville","priority":1283,"external_id":null},{"id":10847106003,"name":"Texas A&M University - Texarkana","priority":1284,"external_id":null},{"id":10847107003,"name":"Texas College","priority":1285,"external_id":null},{"id":10847108003,"name":"Texas Lutheran University","priority":1286,"external_id":null},{"id":10847109003,"name":"Bucknell University","priority":1287,"external_id":null},{"id":10847110003,"name":"Butler University","priority":1288,"external_id":null},{"id":10847111003,"name":"Stephen F. Austin State University","priority":1289,"external_id":null},{"id":10847112003,"name":"Texas A&M University - College Station","priority":1290,"external_id":null},{"id":10847113003,"name":"Stanford University","priority":1291,"external_id":null},{"id":10847114003,"name":"Stetson University","priority":1292,"external_id":null},{"id":10847115003,"name":"Stony Brook University - SUNY","priority":1293,"external_id":null},{"id":10847116003,"name":"Syracuse University","priority":1294,"external_id":null},{"id":10847117003,"name":"Texas Christian University","priority":1295,"external_id":null},{"id":10847118003,"name":"Temple University","priority":1296,"external_id":null},{"id":10847119003,"name":"Clemson University","priority":1297,"external_id":null},{"id":10847120003,"name":"Texas Southern University","priority":1298,"external_id":null},{"id":10847121003,"name":"Austin Peay State University","priority":1299,"external_id":null},{"id":10847122003,"name":"Tennessee State University","priority":1300,"external_id":null},{"id":10847123003,"name":"Ball State University","priority":1301,"external_id":null},{"id":10847124003,"name":"Texas Tech University Health Sciences Center","priority":1302,"external_id":null},{"id":10847125003,"name":"Texas Wesleyan University","priority":1303,"external_id":null},{"id":10847126003,"name":"Texas Woman's University","priority":1304,"external_id":null},{"id":10847127003,"name":"The Catholic University of America","priority":1305,"external_id":null},{"id":10847128003,"name":"The Sage Colleges","priority":1306,"external_id":null},{"id":10847129003,"name":"Thiel College","priority":1307,"external_id":null},{"id":10847130003,"name":"Thomas Aquinas College","priority":1308,"external_id":null},{"id":10847131003,"name":"Thomas College","priority":1309,"external_id":null},{"id":10847132003,"name":"Thomas Edison State College","priority":1310,"external_id":null},{"id":10847133003,"name":"Thomas Jefferson University","priority":1311,"external_id":null},{"id":10847134003,"name":"Thomas More College","priority":1312,"external_id":null},{"id":10847135003,"name":"Thomas More College of Liberal Arts","priority":1313,"external_id":null},{"id":10847136003,"name":"Thomas University","priority":1314,"external_id":null},{"id":10847137003,"name":"Tiffin University","priority":1315,"external_id":null},{"id":10847138003,"name":"Tilburg University","priority":1316,"external_id":null},{"id":10847139003,"name":"Toccoa Falls College","priority":1317,"external_id":null},{"id":10847140003,"name":"Tougaloo College","priority":1318,"external_id":null},{"id":10847141003,"name":"Touro College","priority":1319,"external_id":null},{"id":10847142003,"name":"Transylvania University","priority":1320,"external_id":null},{"id":10847143003,"name":"Trent University","priority":1321,"external_id":null},{"id":10847144003,"name":"Trevecca Nazarene University","priority":1322,"external_id":null},{"id":10847145003,"name":"Trident University International","priority":1323,"external_id":null},{"id":10847146003,"name":"Trine University","priority":1324,"external_id":null},{"id":10847147003,"name":"Trinity Christian College","priority":1325,"external_id":null},{"id":10847148003,"name":"Trinity College","priority":1326,"external_id":null},{"id":10847149003,"name":"Trinity College of Nursing & Health Sciences","priority":1327,"external_id":null},{"id":10847150003,"name":"Trinity International University","priority":1328,"external_id":null},{"id":10847151003,"name":"Trinity Lutheran College","priority":1329,"external_id":null},{"id":10847152003,"name":"Trinity University","priority":1330,"external_id":null},{"id":10847153003,"name":"Trinity Western University","priority":1331,"external_id":null},{"id":10847154003,"name":"Truett McConnell College","priority":1332,"external_id":null},{"id":10847155003,"name":"Truman State University","priority":1333,"external_id":null},{"id":10847156003,"name":"Tufts University","priority":1334,"external_id":null},{"id":10847157003,"name":"Tusculum College","priority":1335,"external_id":null},{"id":10847158003,"name":"Tuskegee University","priority":1336,"external_id":null},{"id":10847159003,"name":"Union College","priority":1337,"external_id":null},{"id":10847160003,"name":"Union Institute and University","priority":1338,"external_id":null},{"id":10847161003,"name":"Union University","priority":1339,"external_id":null},{"id":10847162003,"name":"United States Coast Guard Academy","priority":1340,"external_id":null},{"id":10847163003,"name":"United States International University - Kenya","priority":1341,"external_id":null},{"id":10847164003,"name":"United States Merchant Marine Academy","priority":1342,"external_id":null},{"id":10847165003,"name":"United States Sports Academy","priority":1343,"external_id":null},{"id":10847166003,"name":"Unity College","priority":1344,"external_id":null},{"id":10847167003,"name":"Universidad Adventista de las Antillas","priority":1345,"external_id":null},{"id":10847168003,"name":"Universidad del Este","priority":1346,"external_id":null},{"id":10847169003,"name":"Universidad del Turabo","priority":1347,"external_id":null},{"id":10847170003,"name":"Universidad Metropolitana","priority":1348,"external_id":null},{"id":10847171003,"name":"Universidad Politecnica De Puerto Rico","priority":1349,"external_id":null},{"id":10847172003,"name":"University of Advancing Technology","priority":1350,"external_id":null},{"id":10847173003,"name":"University of Alabama - Huntsville","priority":1351,"external_id":null},{"id":10847174003,"name":"University of Alaska - Anchorage","priority":1352,"external_id":null},{"id":10847175003,"name":"University of Alaska - Fairbanks","priority":1353,"external_id":null},{"id":10847176003,"name":"University of Alaska - Southeast","priority":1354,"external_id":null},{"id":10847177003,"name":"University of Alberta","priority":1355,"external_id":null},{"id":10847178003,"name":"University of Arkansas for Medical Sciences","priority":1356,"external_id":null},{"id":10847179003,"name":"University of Arkansas - Fort Smith","priority":1357,"external_id":null},{"id":10847180003,"name":"University of Arkansas - Little Rock","priority":1358,"external_id":null},{"id":10847181003,"name":"University of Arkansas - Monticello","priority":1359,"external_id":null},{"id":10847182003,"name":"University of Baltimore","priority":1360,"external_id":null},{"id":10847183003,"name":"University of Bridgeport","priority":1361,"external_id":null},{"id":10847184003,"name":"University of British Columbia","priority":1362,"external_id":null},{"id":10847185003,"name":"University of Calgary","priority":1363,"external_id":null},{"id":10847186003,"name":"University of California - Riverside","priority":1364,"external_id":null},{"id":10847187003,"name":"Holy Cross College","priority":1365,"external_id":null},{"id":10847188003,"name":"Towson University","priority":1366,"external_id":null},{"id":10847189003,"name":"United States Military Academy","priority":1367,"external_id":null},{"id":10847190003,"name":"The Citadel","priority":1368,"external_id":null},{"id":10847191003,"name":"Troy University","priority":1369,"external_id":null},{"id":10847192003,"name":"University of California - Davis","priority":1370,"external_id":null},{"id":10847193003,"name":"Grambling State University","priority":1371,"external_id":null},{"id":10847194003,"name":"University at Albany - SUNY","priority":1372,"external_id":null},{"id":10847195003,"name":"University at Buffalo - SUNY","priority":1373,"external_id":null},{"id":10847196003,"name":"United States Naval Academy","priority":1374,"external_id":null},{"id":10847197003,"name":"University of Arizona","priority":1375,"external_id":null},{"id":10847198003,"name":"University of California - Los Angeles","priority":1376,"external_id":null},{"id":10847199003,"name":"Florida A&M University","priority":1377,"external_id":null},{"id":10847200003,"name":"Texas State University","priority":1378,"external_id":null},{"id":10847201003,"name":"University of Alabama - Birmingham","priority":1379,"external_id":null},{"id":10847202003,"name":"University of California - Santa Cruz","priority":1380,"external_id":null},{"id":10847203003,"name":"University of Central Missouri","priority":1381,"external_id":null},{"id":10847204003,"name":"University of Central Oklahoma","priority":1382,"external_id":null},{"id":10847205003,"name":"University of Charleston","priority":1383,"external_id":null},{"id":10847206003,"name":"University of Chicago","priority":1384,"external_id":null},{"id":10847207003,"name":"University of Cincinnati - UC Blue Ash College","priority":1385,"external_id":null},{"id":10847208003,"name":"University of Colorado - Colorado Springs","priority":1386,"external_id":null},{"id":10847209003,"name":"University of Colorado - Denver","priority":1387,"external_id":null},{"id":10847210003,"name":"University of Dallas","priority":1388,"external_id":null},{"id":10847211003,"name":"University of Denver","priority":1389,"external_id":null},{"id":10847212003,"name":"University of Detroit Mercy","priority":1390,"external_id":null},{"id":10847213003,"name":"University of Dubuque","priority":1391,"external_id":null},{"id":10847214003,"name":"University of Evansville","priority":1392,"external_id":null},{"id":10847215003,"name":"University of Findlay","priority":1393,"external_id":null},{"id":10847216003,"name":"University of Great Falls","priority":1394,"external_id":null},{"id":10847217003,"name":"University of Guam","priority":1395,"external_id":null},{"id":10847218003,"name":"University of Guelph","priority":1396,"external_id":null},{"id":10847219003,"name":"University of Hartford","priority":1397,"external_id":null},{"id":10847220003,"name":"University of Hawaii - Hilo","priority":1398,"external_id":null},{"id":10847221003,"name":"University of Hawaii - Maui College","priority":1399,"external_id":null},{"id":10847222003,"name":"University of Hawaii - West Oahu","priority":1400,"external_id":null},{"id":10847223003,"name":"University of Houston - Clear Lake","priority":1401,"external_id":null},{"id":10847224003,"name":"University of Houston - Downtown","priority":1402,"external_id":null},{"id":10847225003,"name":"University of Houston - Victoria","priority":1403,"external_id":null},{"id":10847226003,"name":"University of Illinois - Chicago","priority":1404,"external_id":null},{"id":10847227003,"name":"University of Illinois - Springfield","priority":1405,"external_id":null},{"id":10847228003,"name":"University of Indianapolis","priority":1406,"external_id":null},{"id":10847229003,"name":"University of Jamestown","priority":1407,"external_id":null},{"id":10847230003,"name":"University of La Verne","priority":1408,"external_id":null},{"id":10847231003,"name":"University of Maine - Augusta","priority":1409,"external_id":null},{"id":10847232003,"name":"University of Maine - Farmington","priority":1410,"external_id":null},{"id":10847233003,"name":"University of Maine - Fort Kent","priority":1411,"external_id":null},{"id":10847234003,"name":"University of Maine - Machias","priority":1412,"external_id":null},{"id":10847235003,"name":"University of Maine - Presque Isle","priority":1413,"external_id":null},{"id":10847236003,"name":"University of Mary","priority":1414,"external_id":null},{"id":10847237003,"name":"University of Mary Hardin-Baylor","priority":1415,"external_id":null},{"id":10847238003,"name":"University of Mary Washington","priority":1416,"external_id":null},{"id":10847239003,"name":"University of Maryland - Baltimore","priority":1417,"external_id":null},{"id":10847240003,"name":"University of Maryland - Baltimore County","priority":1418,"external_id":null},{"id":10847241003,"name":"University of Maryland - Eastern Shore","priority":1419,"external_id":null},{"id":10847242003,"name":"University of Maryland - University College","priority":1420,"external_id":null},{"id":10847243003,"name":"University of Massachusetts - Boston","priority":1421,"external_id":null},{"id":10847244003,"name":"University of Massachusetts - Dartmouth","priority":1422,"external_id":null},{"id":10847245003,"name":"University of Massachusetts - Lowell","priority":1423,"external_id":null},{"id":10847246003,"name":"University of Medicine and Dentistry of New Jersey","priority":1424,"external_id":null},{"id":10847247003,"name":"University of Michigan - Dearborn","priority":1425,"external_id":null},{"id":10847248003,"name":"University of Michigan - Flint","priority":1426,"external_id":null},{"id":10847249003,"name":"University of Minnesota - Crookston","priority":1427,"external_id":null},{"id":10847250003,"name":"University of Minnesota - Duluth","priority":1428,"external_id":null},{"id":10847251003,"name":"University of Minnesota - Morris","priority":1429,"external_id":null},{"id":10847252003,"name":"University of Mississippi Medical Center","priority":1430,"external_id":null},{"id":10847253003,"name":"University of Missouri - Kansas City","priority":1431,"external_id":null},{"id":10847254003,"name":"University of Missouri - St. Louis","priority":1432,"external_id":null},{"id":10847255003,"name":"University of Mobile","priority":1433,"external_id":null},{"id":10847256003,"name":"University of Montana - Western","priority":1434,"external_id":null},{"id":10847257003,"name":"University of Montevallo","priority":1435,"external_id":null},{"id":10847258003,"name":"University of Mount Union","priority":1436,"external_id":null},{"id":10847259003,"name":"University of Nebraska Medical Center","priority":1437,"external_id":null},{"id":10847260003,"name":"University of Nebraska - Kearney","priority":1438,"external_id":null},{"id":10847261003,"name":"University of Dayton","priority":1439,"external_id":null},{"id":10847262003,"name":"University of Delaware","priority":1440,"external_id":null},{"id":10847263003,"name":"University of Florida","priority":1441,"external_id":null},{"id":10847264003,"name":"University of Iowa","priority":1442,"external_id":null},{"id":10847265003,"name":"University of Idaho","priority":1443,"external_id":null},{"id":10847266003,"name":"University of Kentucky","priority":1444,"external_id":null},{"id":10847267003,"name":"University of Massachusetts - Amherst","priority":1445,"external_id":null},{"id":10847268003,"name":"University of Maine","priority":1446,"external_id":null},{"id":10847269003,"name":"University of Michigan - Ann Arbor","priority":1447,"external_id":null},{"id":10847270003,"name":"University of Cincinnati","priority":1448,"external_id":null},{"id":10847271003,"name":"University of Miami","priority":1449,"external_id":null},{"id":10847272003,"name":"University of Louisiana - Monroe","priority":1450,"external_id":null},{"id":10847273003,"name":"University of Missouri","priority":1451,"external_id":null},{"id":10847274003,"name":"University of Mississippi","priority":1452,"external_id":null},{"id":10847275003,"name":"University of Memphis","priority":1453,"external_id":null},{"id":10847276003,"name":"University of Houston","priority":1454,"external_id":null},{"id":10847277003,"name":"University of Colorado - Boulder","priority":1455,"external_id":null},{"id":10847278003,"name":"University of Nebraska - Omaha","priority":1456,"external_id":null},{"id":10847279003,"name":"University of New Brunswick","priority":1457,"external_id":null},{"id":10847280003,"name":"University of New England","priority":1458,"external_id":null},{"id":10847281003,"name":"University of New Haven","priority":1459,"external_id":null},{"id":10847282003,"name":"University of New Orleans","priority":1460,"external_id":null},{"id":10847283003,"name":"University of North Alabama","priority":1461,"external_id":null},{"id":10847284003,"name":"University of North Carolina School of the Arts","priority":1462,"external_id":null},{"id":10847285003,"name":"University of North Carolina - Asheville","priority":1463,"external_id":null},{"id":10847286003,"name":"University of North Carolina - Greensboro","priority":1464,"external_id":null},{"id":10847287003,"name":"University of North Carolina - Pembroke","priority":1465,"external_id":null},{"id":10847288003,"name":"University of North Carolina - Wilmington","priority":1466,"external_id":null},{"id":10847289003,"name":"University of North Florida","priority":1467,"external_id":null},{"id":10847290003,"name":"University of North Georgia","priority":1468,"external_id":null},{"id":10847291003,"name":"University of Northwestern Ohio","priority":1469,"external_id":null},{"id":10847292003,"name":"University of Northwestern - St. Paul","priority":1470,"external_id":null},{"id":10847293003,"name":"University of Ottawa","priority":1471,"external_id":null},{"id":10847294003,"name":"University of Phoenix","priority":1472,"external_id":null},{"id":10847295003,"name":"University of Pikeville","priority":1473,"external_id":null},{"id":10847296003,"name":"University of Portland","priority":1474,"external_id":null},{"id":10847297003,"name":"University of Prince Edward Island","priority":1475,"external_id":null},{"id":10847298003,"name":"University of Puerto Rico - Aguadilla","priority":1476,"external_id":null},{"id":10847299003,"name":"University of Puerto Rico - Arecibo","priority":1477,"external_id":null},{"id":10847300003,"name":"University of Puerto Rico - Bayamon","priority":1478,"external_id":null},{"id":10847301003,"name":"University of Puerto Rico - Cayey","priority":1479,"external_id":null},{"id":10847302003,"name":"University of Puerto Rico - Humacao","priority":1480,"external_id":null},{"id":10847303003,"name":"University of Puerto Rico - Mayaguez","priority":1481,"external_id":null},{"id":10847304003,"name":"University of Puerto Rico - Medical Sciences Campus","priority":1482,"external_id":null},{"id":10847305003,"name":"University of Puerto Rico - Ponce","priority":1483,"external_id":null},{"id":10847306003,"name":"University of Puerto Rico - Rio Piedras","priority":1484,"external_id":null},{"id":10847307003,"name":"University of Puget Sound","priority":1485,"external_id":null},{"id":10847308003,"name":"University of Redlands","priority":1486,"external_id":null},{"id":10847309003,"name":"University of Regina","priority":1487,"external_id":null},{"id":10847310003,"name":"University of Rio Grande","priority":1488,"external_id":null},{"id":10847311003,"name":"University of Rochester","priority":1489,"external_id":null},{"id":10847312003,"name":"University of San Francisco","priority":1490,"external_id":null},{"id":10847313003,"name":"University of Saskatchewan","priority":1491,"external_id":null},{"id":10847314003,"name":"University of Science and Arts of Oklahoma","priority":1492,"external_id":null},{"id":10847315003,"name":"University of Scranton","priority":1493,"external_id":null},{"id":10847316003,"name":"University of Sioux Falls","priority":1494,"external_id":null},{"id":10847317003,"name":"University of South Carolina - Aiken","priority":1495,"external_id":null},{"id":10847318003,"name":"University of South Carolina - Beaufort","priority":1496,"external_id":null},{"id":10847319003,"name":"University of South Carolina - Upstate","priority":1497,"external_id":null},{"id":10847320003,"name":"University of South Florida - St. Petersburg","priority":1498,"external_id":null},{"id":10847321003,"name":"University of Southern Indiana","priority":1499,"external_id":null},{"id":10847322003,"name":"University of Southern Maine","priority":1500,"external_id":null},{"id":10847323003,"name":"University of St. Francis","priority":1501,"external_id":null},{"id":10847324003,"name":"University of St. Joseph","priority":1502,"external_id":null},{"id":10847325003,"name":"University of St. Mary","priority":1503,"external_id":null},{"id":10847326003,"name":"University of St. Thomas","priority":1504,"external_id":null},{"id":10847327003,"name":"University of Tampa","priority":1505,"external_id":null},{"id":10847328003,"name":"University of Texas Health Science Center - Houston","priority":1506,"external_id":null},{"id":10847329003,"name":"University of Texas Health Science Center - San Antonio","priority":1507,"external_id":null},{"id":10847330003,"name":"University of Texas Medical Branch - Galveston","priority":1508,"external_id":null},{"id":10847331003,"name":"University of Texas of the Permian Basin","priority":1509,"external_id":null},{"id":10847332003,"name":"University of Texas - Arlington","priority":1510,"external_id":null},{"id":10847333003,"name":"University of Texas - Brownsville","priority":1511,"external_id":null},{"id":10847334003,"name":"University of Texas - Pan American","priority":1512,"external_id":null},{"id":10847335003,"name":"University of Oregon","priority":1513,"external_id":null},{"id":10847336003,"name":"University of New Mexico","priority":1514,"external_id":null},{"id":10847337003,"name":"University of Pennsylvania","priority":1515,"external_id":null},{"id":10847338003,"name":"University of North Dakota","priority":1516,"external_id":null},{"id":10847339003,"name":"University of Nevada - Reno","priority":1517,"external_id":null},{"id":10847340003,"name":"University of New Hampshire","priority":1518,"external_id":null},{"id":10847341003,"name":"University of Texas - Austin","priority":1519,"external_id":null},{"id":10847342003,"name":"University of Southern Mississippi","priority":1520,"external_id":null},{"id":10847343003,"name":"University of Rhode Island","priority":1521,"external_id":null},{"id":10847344003,"name":"University of South Dakota","priority":1522,"external_id":null},{"id":10847345003,"name":"University of Tennessee","priority":1523,"external_id":null},{"id":10847346003,"name":"University of North Texas","priority":1524,"external_id":null},{"id":10847347003,"name":"University of North Carolina - Charlotte","priority":1525,"external_id":null},{"id":10847348003,"name":"University of Texas - San Antonio","priority":1526,"external_id":null},{"id":10847349003,"name":"University of Notre Dame","priority":1527,"external_id":null},{"id":10847350003,"name":"University of Southern California","priority":1528,"external_id":null},{"id":10847351003,"name":"University of Texas - Tyler","priority":1529,"external_id":null},{"id":10847352003,"name":"University of the Arts","priority":1530,"external_id":null},{"id":10847353003,"name":"University of the Cumberlands","priority":1531,"external_id":null},{"id":10847354003,"name":"University of the District of Columbia","priority":1532,"external_id":null},{"id":10847355003,"name":"University of the Ozarks","priority":1533,"external_id":null},{"id":10847356003,"name":"University of the Pacific","priority":1534,"external_id":null},{"id":10847357003,"name":"University of the Sacred Heart","priority":1535,"external_id":null},{"id":10847358003,"name":"University of the Sciences","priority":1536,"external_id":null},{"id":10847359003,"name":"University of the Southwest","priority":1537,"external_id":null},{"id":10847360003,"name":"University of the Virgin Islands","priority":1538,"external_id":null},{"id":10847361003,"name":"University of the West","priority":1539,"external_id":null},{"id":10847362003,"name":"University of Toronto","priority":1540,"external_id":null},{"id":10847363003,"name":"University of Vermont","priority":1541,"external_id":null},{"id":10847364003,"name":"University of Victoria","priority":1542,"external_id":null},{"id":10847365003,"name":"University of Virginia - Wise","priority":1543,"external_id":null},{"id":10847366003,"name":"University of Waterloo","priority":1544,"external_id":null},{"id":10847367003,"name":"University of West Alabama","priority":1545,"external_id":null},{"id":10847368003,"name":"University of West Florida","priority":1546,"external_id":null},{"id":10847369003,"name":"University of West Georgia","priority":1547,"external_id":null},{"id":10847370003,"name":"University of Windsor","priority":1548,"external_id":null},{"id":10847371003,"name":"University of Winnipeg","priority":1549,"external_id":null},{"id":10847372003,"name":"University of Wisconsin - Eau Claire","priority":1550,"external_id":null},{"id":10847373003,"name":"University of Wisconsin - Green Bay","priority":1551,"external_id":null},{"id":10847374003,"name":"University of Wisconsin - La Crosse","priority":1552,"external_id":null},{"id":10847375003,"name":"University of Wisconsin - Milwaukee","priority":1553,"external_id":null},{"id":10847376003,"name":"University of Wisconsin - Oshkosh","priority":1554,"external_id":null},{"id":10847377003,"name":"University of Wisconsin - Parkside","priority":1555,"external_id":null},{"id":10847378003,"name":"University of Wisconsin - Platteville","priority":1556,"external_id":null},{"id":10847379003,"name":"University of Wisconsin - River Falls","priority":1557,"external_id":null},{"id":10847380003,"name":"University of Wisconsin - Stevens Point","priority":1558,"external_id":null},{"id":10847381003,"name":"University of Wisconsin - Stout","priority":1559,"external_id":null},{"id":10847382003,"name":"University of Wisconsin - Superior","priority":1560,"external_id":null},{"id":10847383003,"name":"University of Wisconsin - Whitewater","priority":1561,"external_id":null},{"id":10847384003,"name":"Upper Iowa University","priority":1562,"external_id":null},{"id":10847385003,"name":"Urbana University","priority":1563,"external_id":null},{"id":10847386003,"name":"Ursinus College","priority":1564,"external_id":null},{"id":10847387003,"name":"Ursuline College","priority":1565,"external_id":null},{"id":10847388003,"name":"Utah Valley University","priority":1566,"external_id":null},{"id":10847389003,"name":"Utica College","priority":1567,"external_id":null},{"id":10847390003,"name":"Valdosta State University","priority":1568,"external_id":null},{"id":10847391003,"name":"Valley City State University","priority":1569,"external_id":null},{"id":10847392003,"name":"Valley Forge Christian College","priority":1570,"external_id":null},{"id":10847393003,"name":"VanderCook College of Music","priority":1571,"external_id":null},{"id":10847394003,"name":"Vanguard University of Southern California","priority":1572,"external_id":null},{"id":10847395003,"name":"Vassar College","priority":1573,"external_id":null},{"id":10847396003,"name":"Vaughn College of Aeronautics and Technology","priority":1574,"external_id":null},{"id":10847397003,"name":"Vermont Technical College","priority":1575,"external_id":null},{"id":10847398003,"name":"Victory University","priority":1576,"external_id":null},{"id":10847399003,"name":"Vincennes University","priority":1577,"external_id":null},{"id":10847400003,"name":"Virginia Commonwealth University","priority":1578,"external_id":null},{"id":10847401003,"name":"Virginia Intermont College","priority":1579,"external_id":null},{"id":10847402003,"name":"Virginia State University","priority":1580,"external_id":null},{"id":10847403003,"name":"Virginia Union University","priority":1581,"external_id":null},{"id":10847404003,"name":"Virginia Wesleyan College","priority":1582,"external_id":null},{"id":10847405003,"name":"Viterbo University","priority":1583,"external_id":null},{"id":10847406003,"name":"Voorhees College","priority":1584,"external_id":null},{"id":10847407003,"name":"Wabash College","priority":1585,"external_id":null},{"id":10847408003,"name":"Walden University","priority":1586,"external_id":null},{"id":10847409003,"name":"Waldorf College","priority":1587,"external_id":null},{"id":10847410003,"name":"Walla Walla University","priority":1588,"external_id":null},{"id":10847411003,"name":"Walsh College of Accountancy and Business Administration","priority":1589,"external_id":null},{"id":10847412003,"name":"Walsh University","priority":1590,"external_id":null},{"id":10847413003,"name":"Warner Pacific College","priority":1591,"external_id":null},{"id":10847414003,"name":"Warner University","priority":1592,"external_id":null},{"id":10847415003,"name":"Warren Wilson College","priority":1593,"external_id":null},{"id":10847416003,"name":"Wartburg College","priority":1594,"external_id":null},{"id":10847417003,"name":"Washburn University","priority":1595,"external_id":null},{"id":10847418003,"name":"Washington Adventist University","priority":1596,"external_id":null},{"id":10847419003,"name":"Washington and Jefferson College","priority":1597,"external_id":null},{"id":10847420003,"name":"Washington and Lee University","priority":1598,"external_id":null},{"id":10847421003,"name":"Washington College","priority":1599,"external_id":null},{"id":10847422003,"name":"Washington University in St. Louis","priority":1600,"external_id":null},{"id":10847423003,"name":"Watkins College of Art, Design & Film","priority":1601,"external_id":null},{"id":10847424003,"name":"Wayland Baptist University","priority":1602,"external_id":null},{"id":10847425003,"name":"Wayne State College","priority":1603,"external_id":null},{"id":10847426003,"name":"Wayne State University","priority":1604,"external_id":null},{"id":10847427003,"name":"Waynesburg University","priority":1605,"external_id":null},{"id":10847428003,"name":"Valparaiso University","priority":1606,"external_id":null},{"id":10847429003,"name":"Villanova University","priority":1607,"external_id":null},{"id":10847430003,"name":"Virginia Tech","priority":1608,"external_id":null},{"id":10847431003,"name":"Washington State University","priority":1609,"external_id":null},{"id":10847432003,"name":"University of Toledo","priority":1610,"external_id":null},{"id":10847433003,"name":"Wagner College","priority":1611,"external_id":null},{"id":10847434003,"name":"University of Wyoming","priority":1612,"external_id":null},{"id":10847435003,"name":"University of Wisconsin - Madison","priority":1613,"external_id":null},{"id":10847436003,"name":"University of Tulsa","priority":1614,"external_id":null},{"id":10847437003,"name":"Webb Institute","priority":1615,"external_id":null},{"id":10847438003,"name":"Webber International University","priority":1616,"external_id":null},{"id":10847439003,"name":"Webster University","priority":1617,"external_id":null},{"id":10847440003,"name":"Welch College","priority":1618,"external_id":null},{"id":10847441003,"name":"Wellesley College","priority":1619,"external_id":null},{"id":10847442003,"name":"Wells College","priority":1620,"external_id":null},{"id":10847443003,"name":"Wentworth Institute of Technology","priority":1621,"external_id":null},{"id":10847444003,"name":"Wesley College","priority":1622,"external_id":null},{"id":10847445003,"name":"Wesleyan College","priority":1623,"external_id":null},{"id":10847446003,"name":"Wesleyan University","priority":1624,"external_id":null},{"id":10847447003,"name":"West Chester University of Pennsylvania","priority":1625,"external_id":null},{"id":10847448003,"name":"West Liberty University","priority":1626,"external_id":null},{"id":10847449003,"name":"West Texas A&M University","priority":1627,"external_id":null},{"id":10847450003,"name":"West Virginia State University","priority":1628,"external_id":null},{"id":10847451003,"name":"West Virginia University Institute of Technology","priority":1629,"external_id":null},{"id":10847452003,"name":"West Virginia University - Parkersburg","priority":1630,"external_id":null},{"id":10847453003,"name":"West Virginia Wesleyan College","priority":1631,"external_id":null},{"id":10847454003,"name":"Western Connecticut State University","priority":1632,"external_id":null},{"id":10847455003,"name":"Western Governors University","priority":1633,"external_id":null},{"id":10847456003,"name":"Western International University","priority":1634,"external_id":null},{"id":10847457003,"name":"Western Nevada College","priority":1635,"external_id":null},{"id":10847458003,"name":"Western New England University","priority":1636,"external_id":null},{"id":10847459003,"name":"Western New Mexico University","priority":1637,"external_id":null},{"id":10847460003,"name":"Western Oregon University","priority":1638,"external_id":null},{"id":10847461003,"name":"Western State Colorado University","priority":1639,"external_id":null},{"id":10847462003,"name":"Western University","priority":1640,"external_id":null},{"id":10847463003,"name":"Western Washington University","priority":1641,"external_id":null},{"id":10847464003,"name":"Westfield State University","priority":1642,"external_id":null},{"id":10847465003,"name":"Westminster College","priority":1643,"external_id":null},{"id":10847466003,"name":"Westmont College","priority":1644,"external_id":null},{"id":10847467003,"name":"Wheaton College","priority":1645,"external_id":null},{"id":10847468003,"name":"Wheeling Jesuit University","priority":1646,"external_id":null},{"id":10847469003,"name":"Wheelock College","priority":1647,"external_id":null},{"id":10847470003,"name":"Whitman College","priority":1648,"external_id":null},{"id":10847471003,"name":"Whittier College","priority":1649,"external_id":null},{"id":10847472003,"name":"Whitworth University","priority":1650,"external_id":null},{"id":10847473003,"name":"Wichita State University","priority":1651,"external_id":null},{"id":10847474003,"name":"Widener University","priority":1652,"external_id":null},{"id":10847475003,"name":"Wilberforce University","priority":1653,"external_id":null},{"id":10847476003,"name":"Wiley College","priority":1654,"external_id":null},{"id":10847477003,"name":"Wilkes University","priority":1655,"external_id":null},{"id":10847478003,"name":"Willamette University","priority":1656,"external_id":null},{"id":10847479003,"name":"William Carey University","priority":1657,"external_id":null},{"id":10847480003,"name":"William Jessup University","priority":1658,"external_id":null},{"id":10847481003,"name":"William Jewell College","priority":1659,"external_id":null},{"id":10847482003,"name":"William Paterson University of New Jersey","priority":1660,"external_id":null},{"id":10847483003,"name":"William Peace University","priority":1661,"external_id":null},{"id":10847484003,"name":"William Penn University","priority":1662,"external_id":null},{"id":10847485003,"name":"William Woods University","priority":1663,"external_id":null},{"id":10847486003,"name":"Williams Baptist College","priority":1664,"external_id":null},{"id":10847487003,"name":"Williams College","priority":1665,"external_id":null},{"id":10847488003,"name":"Wilmington College","priority":1666,"external_id":null},{"id":10847489003,"name":"Wilmington University","priority":1667,"external_id":null},{"id":10847490003,"name":"Wilson College","priority":1668,"external_id":null},{"id":10847491003,"name":"Wingate University","priority":1669,"external_id":null},{"id":10847492003,"name":"Winona State University","priority":1670,"external_id":null},{"id":10847493003,"name":"Winston-Salem State University","priority":1671,"external_id":null},{"id":10847494003,"name":"Winthrop University","priority":1672,"external_id":null},{"id":10847495003,"name":"Wisconsin Lutheran College","priority":1673,"external_id":null},{"id":10847496003,"name":"Wittenberg University","priority":1674,"external_id":null},{"id":10847497003,"name":"Woodbury University","priority":1675,"external_id":null},{"id":10847498003,"name":"Worcester Polytechnic Institute","priority":1676,"external_id":null},{"id":10847499003,"name":"Worcester State University","priority":1677,"external_id":null},{"id":10847500003,"name":"Wright State University","priority":1678,"external_id":null},{"id":10847501003,"name":"Xavier University","priority":1679,"external_id":null},{"id":10847502003,"name":"Xavier University of Louisiana","priority":1680,"external_id":null},{"id":10847503003,"name":"Yeshiva University","priority":1681,"external_id":null},{"id":10847504003,"name":"York College","priority":1682,"external_id":null},{"id":10847505003,"name":"York College of Pennsylvania","priority":1683,"external_id":null},{"id":10847506003,"name":"York University","priority":1684,"external_id":null},{"id":10847507003,"name":"University of Cambridge","priority":1685,"external_id":null},{"id":10847508003,"name":"UCL (University College London)","priority":1686,"external_id":null},{"id":10847509003,"name":"Imperial College London","priority":1687,"external_id":null},{"id":10847510003,"name":"University of Oxford","priority":1688,"external_id":null},{"id":10847511003,"name":"ETH Zurich (Swiss Federal Institute of Technology)","priority":1689,"external_id":null},{"id":10847512003,"name":"University of Edinburgh","priority":1690,"external_id":null},{"id":10847513003,"name":"Ecole Polytechnique Fédérale de Lausanne","priority":1691,"external_id":null},{"id":10847514003,"name":"King's College London (KCL)","priority":1692,"external_id":null},{"id":10847515003,"name":"National University of Singapore (NUS)","priority":1693,"external_id":null},{"id":10847516003,"name":"University of Hong Kong","priority":1694,"external_id":null},{"id":10847517003,"name":"Australian National University","priority":1695,"external_id":null},{"id":10847518003,"name":"Ecole normale supérieure, Paris","priority":1696,"external_id":null},{"id":10847519003,"name":"University of Bristol","priority":1697,"external_id":null},{"id":10847520003,"name":"The University of Melbourne","priority":1698,"external_id":null},{"id":10847521003,"name":"The University of Tokyo","priority":1699,"external_id":null},{"id":10847522003,"name":"The University of Manchester","priority":1700,"external_id":null},{"id":10847523003,"name":"Western Illinois University","priority":1701,"external_id":null},{"id":10847524003,"name":"Wofford College","priority":1702,"external_id":null},{"id":10847525003,"name":"Western Carolina University","priority":1703,"external_id":null},{"id":10847526003,"name":"West Virginia University","priority":1704,"external_id":null},{"id":10847527003,"name":"Yale University","priority":1705,"external_id":null},{"id":10847528003,"name":"The Hong Kong University of Science and Technology","priority":1706,"external_id":null},{"id":10847529003,"name":"Kyoto University","priority":1707,"external_id":null},{"id":10847530003,"name":"Seoul National University","priority":1708,"external_id":null},{"id":10847531003,"name":"The University of Sydney","priority":1709,"external_id":null},{"id":10847532003,"name":"The Chinese University of Hong Kong","priority":1710,"external_id":null},{"id":10847533003,"name":"Ecole Polytechnique","priority":1711,"external_id":null},{"id":10847534003,"name":"Nanyang Technological University (NTU)","priority":1712,"external_id":null},{"id":10847535003,"name":"The University of Queensland","priority":1713,"external_id":null},{"id":10847536003,"name":"University of Copenhagen","priority":1714,"external_id":null},{"id":10847537003,"name":"Peking University","priority":1715,"external_id":null},{"id":10847538003,"name":"Tsinghua University","priority":1716,"external_id":null},{"id":10847539003,"name":"Ruprecht-Karls-Universität Heidelberg","priority":1717,"external_id":null},{"id":10847540003,"name":"University of Glasgow","priority":1718,"external_id":null},{"id":10847541003,"name":"The University of New South Wales","priority":1719,"external_id":null},{"id":10847542003,"name":"Technische Universität München","priority":1720,"external_id":null},{"id":10847543003,"name":"Osaka University","priority":1721,"external_id":null},{"id":10847544003,"name":"University of Amsterdam","priority":1722,"external_id":null},{"id":10847545003,"name":"KAIST - Korea Advanced Institute of Science & Technology","priority":1723,"external_id":null},{"id":10847546003,"name":"Trinity College Dublin","priority":1724,"external_id":null},{"id":10847547003,"name":"University of Birmingham","priority":1725,"external_id":null},{"id":10847548003,"name":"The University of Warwick","priority":1726,"external_id":null},{"id":10847549003,"name":"Ludwig-Maximilians-Universität München","priority":1727,"external_id":null},{"id":10847550003,"name":"Tokyo Institute of Technology","priority":1728,"external_id":null},{"id":10847551003,"name":"Lund University","priority":1729,"external_id":null},{"id":10847552003,"name":"London School of Economics and Political Science (LSE)","priority":1730,"external_id":null},{"id":10847553003,"name":"Monash University","priority":1731,"external_id":null},{"id":10847554003,"name":"University of Helsinki","priority":1732,"external_id":null},{"id":10847555003,"name":"The University of Sheffield","priority":1733,"external_id":null},{"id":10847556003,"name":"University of Geneva","priority":1734,"external_id":null},{"id":10847557003,"name":"Leiden University","priority":1735,"external_id":null},{"id":10847558003,"name":"The University of Nottingham","priority":1736,"external_id":null},{"id":10847559003,"name":"Tohoku University","priority":1737,"external_id":null},{"id":10847560003,"name":"KU Leuven","priority":1738,"external_id":null},{"id":10847561003,"name":"University of Zurich","priority":1739,"external_id":null},{"id":10847562003,"name":"Uppsala University","priority":1740,"external_id":null},{"id":10847563003,"name":"Utrecht University","priority":1741,"external_id":null},{"id":10847564003,"name":"National Taiwan University (NTU)","priority":1742,"external_id":null},{"id":10847565003,"name":"University of St Andrews","priority":1743,"external_id":null},{"id":10847566003,"name":"The University of Western Australia","priority":1744,"external_id":null},{"id":10847567003,"name":"University of Southampton","priority":1745,"external_id":null},{"id":10847568003,"name":"Fudan University","priority":1746,"external_id":null},{"id":10847569003,"name":"University of Oslo","priority":1747,"external_id":null},{"id":10847570003,"name":"Durham University","priority":1748,"external_id":null},{"id":10847571003,"name":"Aarhus University","priority":1749,"external_id":null},{"id":10847572003,"name":"Erasmus University Rotterdam","priority":1750,"external_id":null},{"id":10847573003,"name":"Université de Montréal","priority":1751,"external_id":null},{"id":10847574003,"name":"The University of Auckland","priority":1752,"external_id":null},{"id":10847575003,"name":"Delft University of Technology","priority":1753,"external_id":null},{"id":10847576003,"name":"University of Groningen","priority":1754,"external_id":null},{"id":10847577003,"name":"University of Leeds","priority":1755,"external_id":null},{"id":10847578003,"name":"Nagoya University","priority":1756,"external_id":null},{"id":10847579003,"name":"Universität Freiburg","priority":1757,"external_id":null},{"id":10847580003,"name":"City University of Hong Kong","priority":1758,"external_id":null},{"id":10847581003,"name":"The University of Adelaide","priority":1759,"external_id":null},{"id":10847582003,"name":"Pohang University of Science And Technology (POSTECH)","priority":1760,"external_id":null},{"id":10847583003,"name":"Freie Universität Berlin","priority":1761,"external_id":null},{"id":10847584003,"name":"University of Basel","priority":1762,"external_id":null},{"id":10847585003,"name":"University of Lausanne","priority":1763,"external_id":null},{"id":10847586003,"name":"Université Pierre et Marie Curie (UPMC)","priority":1764,"external_id":null},{"id":10847587003,"name":"Yonsei University","priority":1765,"external_id":null},{"id":10847588003,"name":"University of York","priority":1766,"external_id":null},{"id":10847589003,"name":"Queen Mary, University of London (QMUL)","priority":1767,"external_id":null},{"id":10847590003,"name":"Karlsruhe Institute of Technology (KIT)","priority":1768,"external_id":null},{"id":10847591003,"name":"KTH, Royal Institute of Technology","priority":1769,"external_id":null},{"id":10847592003,"name":"Lomonosov Moscow State University","priority":1770,"external_id":null},{"id":10847593003,"name":"Maastricht University","priority":1771,"external_id":null},{"id":10847594003,"name":"University of Ghent","priority":1772,"external_id":null},{"id":10847595003,"name":"Shanghai Jiao Tong University","priority":1773,"external_id":null},{"id":10847596003,"name":"Humboldt-Universität zu Berlin","priority":1774,"external_id":null},{"id":10847597003,"name":"Universidade de São Paulo (USP)","priority":1775,"external_id":null},{"id":10847598003,"name":"Georg-August-Universität Göttingen","priority":1776,"external_id":null},{"id":10847599003,"name":"Newcastle University","priority":1777,"external_id":null},{"id":10847600003,"name":"University of Liverpool","priority":1778,"external_id":null},{"id":10847601003,"name":"Kyushu University","priority":1779,"external_id":null},{"id":10847602003,"name":"Eberhard Karls Universität Tübingen","priority":1780,"external_id":null},{"id":10847603003,"name":"Technical University of Denmark","priority":1781,"external_id":null},{"id":10847604003,"name":"Cardiff University","priority":1782,"external_id":null},{"id":10847605003,"name":"Université Catholique de Louvain (UCL)","priority":1783,"external_id":null},{"id":10847606003,"name":"University College Dublin","priority":1784,"external_id":null},{"id":10847607003,"name":"McMaster University","priority":1785,"external_id":null},{"id":10847608003,"name":"Hebrew University of Jerusalem","priority":1786,"external_id":null},{"id":10847609003,"name":"Radboud University Nijmegen","priority":1787,"external_id":null},{"id":10847610003,"name":"Hokkaido University","priority":1788,"external_id":null},{"id":10847611003,"name":"Korea University","priority":1789,"external_id":null},{"id":10847612003,"name":"University of Cape Town","priority":1790,"external_id":null},{"id":10847613003,"name":"Rheinisch-Westfälische Technische Hochschule Aachen","priority":1791,"external_id":null},{"id":10847614003,"name":"University of Aberdeen","priority":1792,"external_id":null},{"id":10847615003,"name":"Wageningen University","priority":1793,"external_id":null},{"id":10847616003,"name":"University of Bergen","priority":1794,"external_id":null},{"id":10847617003,"name":"University of Bern","priority":1795,"external_id":null},{"id":10847618003,"name":"University of Otago","priority":1796,"external_id":null},{"id":10847619003,"name":"Lancaster University","priority":1797,"external_id":null},{"id":10847620003,"name":"Eindhoven University of Technology","priority":1798,"external_id":null},{"id":10847621003,"name":"Ecole Normale Supérieure de Lyon","priority":1799,"external_id":null},{"id":10847622003,"name":"University of Vienna","priority":1800,"external_id":null},{"id":10847623003,"name":"The Hong Kong Polytechnic University","priority":1801,"external_id":null},{"id":10847624003,"name":"Sungkyunkwan University","priority":1802,"external_id":null},{"id":10847625003,"name":"Rheinische Friedrich-Wilhelms-Universität Bonn","priority":1803,"external_id":null},{"id":10847626003,"name":"Universidad Nacional Autónoma de México (UNAM)","priority":1804,"external_id":null},{"id":10847627003,"name":"Zhejiang University","priority":1805,"external_id":null},{"id":10847628003,"name":"Pontificia Universidad Católica de Chile","priority":1806,"external_id":null},{"id":10847629003,"name":"Universiti Malaya (UM)","priority":1807,"external_id":null},{"id":10847630003,"name":"Université Libre de Bruxelles (ULB)","priority":1808,"external_id":null},{"id":10847631003,"name":"University of Exeter","priority":1809,"external_id":null},{"id":10847632003,"name":"Stockholm University","priority":1810,"external_id":null},{"id":10847633003,"name":"Queen's University of Belfast","priority":1811,"external_id":null},{"id":10847634003,"name":"Vrije Universiteit Brussel (VUB)","priority":1812,"external_id":null},{"id":10847635003,"name":"University of Science and Technology of China","priority":1813,"external_id":null},{"id":10847636003,"name":"Nanjing University","priority":1814,"external_id":null},{"id":10847637003,"name":"Universitat Autónoma de Barcelona","priority":1815,"external_id":null},{"id":10847638003,"name":"University of Barcelona","priority":1816,"external_id":null},{"id":10847639003,"name":"VU University Amsterdam","priority":1817,"external_id":null},{"id":10847640003,"name":"Technion - Israel Institute of Technology","priority":1818,"external_id":null},{"id":10847641003,"name":"Technische Universität Berlin","priority":1819,"external_id":null},{"id":10847642003,"name":"University of Antwerp","priority":1820,"external_id":null},{"id":10847643003,"name":"Universität Hamburg","priority":1821,"external_id":null},{"id":10847644003,"name":"University of Bath","priority":1822,"external_id":null},{"id":10847645003,"name":"University of Bologna","priority":1823,"external_id":null},{"id":10847646003,"name":"Queen's University, Ontario","priority":1824,"external_id":null},{"id":10847647003,"name":"Université Paris-Sud 11","priority":1825,"external_id":null},{"id":10847648003,"name":"Keio University","priority":1826,"external_id":null},{"id":10847649003,"name":"University of Sussex","priority":1827,"external_id":null},{"id":10847650003,"name":"Universidad Autónoma de Madrid","priority":1828,"external_id":null},{"id":10847651003,"name":"Aalto University","priority":1829,"external_id":null},{"id":10847652003,"name":"Sapienza University of Rome","priority":1830,"external_id":null},{"id":10847653003,"name":"Tel Aviv University","priority":1831,"external_id":null},{"id":10847654003,"name":"National Tsing Hua University","priority":1832,"external_id":null},{"id":10847655003,"name":"Chalmers University of Technology","priority":1833,"external_id":null},{"id":10847656003,"name":"University of Leicester","priority":1834,"external_id":null},{"id":10847657003,"name":"Université Paris Diderot - Paris 7","priority":1835,"external_id":null},{"id":10847658003,"name":"University of Gothenburg","priority":1836,"external_id":null},{"id":10847659003,"name":"University of Turku","priority":1837,"external_id":null},{"id":10847660003,"name":"Universität Frankfurt am Main","priority":1838,"external_id":null},{"id":10847661003,"name":"Universidad de Buenos Aires","priority":1839,"external_id":null},{"id":10847662003,"name":"University College Cork","priority":1840,"external_id":null},{"id":10847663003,"name":"University of Tsukuba","priority":1841,"external_id":null},{"id":10847664003,"name":"University of Reading","priority":1842,"external_id":null},{"id":10847665003,"name":"Sciences Po Paris","priority":1843,"external_id":null},{"id":10847666003,"name":"Universidade Estadual de Campinas","priority":1844,"external_id":null},{"id":10847667003,"name":"King Fahd University of Petroleum & Minerals","priority":1845,"external_id":null},{"id":10847668003,"name":"University Complutense Madrid","priority":1846,"external_id":null},{"id":10847669003,"name":"Université Paris-Sorbonne (Paris IV)","priority":1847,"external_id":null},{"id":10847670003,"name":"University of Dundee","priority":1848,"external_id":null},{"id":10847671003,"name":"Université Joseph Fourier - Grenoble 1","priority":1849,"external_id":null},{"id":10847672003,"name":"Waseda University","priority":1850,"external_id":null},{"id":10847673003,"name":"Indian Institute of Technology Delhi (IITD)","priority":1851,"external_id":null},{"id":10847674003,"name":"Universidad de Chile","priority":1852,"external_id":null},{"id":10847675003,"name":"Université Paris 1 Panthéon-Sorbonne","priority":1853,"external_id":null},{"id":10847676003,"name":"Université de Strasbourg","priority":1854,"external_id":null},{"id":10847677003,"name":"University of Twente","priority":1855,"external_id":null},{"id":10847678003,"name":"University of East Anglia (UEA)","priority":1856,"external_id":null},{"id":10847679003,"name":"National Chiao Tung University","priority":1857,"external_id":null},{"id":10847680003,"name":"Politecnico di Milano","priority":1858,"external_id":null},{"id":10847681003,"name":"Charles University","priority":1859,"external_id":null},{"id":10847682003,"name":"Indian Institute of Technology Bombay (IITB)","priority":1860,"external_id":null},{"id":10847683003,"name":"University of Milano","priority":1861,"external_id":null},{"id":10847684003,"name":"Westfälische Wilhelms-Universität Münster","priority":1862,"external_id":null},{"id":10847685003,"name":"University of Canterbury","priority":1863,"external_id":null},{"id":10847686003,"name":"Chulalongkorn University","priority":1864,"external_id":null},{"id":10847687003,"name":"Saint-Petersburg State University","priority":1865,"external_id":null},{"id":10847688003,"name":"University of Liege","priority":1866,"external_id":null},{"id":10847689003,"name":"Universität zu Köln","priority":1867,"external_id":null},{"id":10847690003,"name":"Loughborough University","priority":1868,"external_id":null},{"id":10847691003,"name":"National Cheng Kung University","priority":1869,"external_id":null},{"id":10847692003,"name":"Universität Stuttgart","priority":1870,"external_id":null},{"id":10847693003,"name":"Hanyang University","priority":1871,"external_id":null},{"id":10847694003,"name":"American University of Beirut (AUB)","priority":1872,"external_id":null},{"id":10847695003,"name":"Norwegian University of Science And Technology","priority":1873,"external_id":null},{"id":10847696003,"name":"Beijing Normal University","priority":1874,"external_id":null},{"id":10847697003,"name":"King Saud University","priority":1875,"external_id":null},{"id":10847698003,"name":"University of Oulu","priority":1876,"external_id":null},{"id":10847699003,"name":"Kyung Hee University","priority":1877,"external_id":null},{"id":10847700003,"name":"University of Strathclyde","priority":1878,"external_id":null},{"id":10847701003,"name":"Universität Ulm","priority":1879,"external_id":null},{"id":10847702003,"name":"University of Pisa","priority":1880,"external_id":null},{"id":10847703003,"name":"Technische Universität Darmstadt","priority":1881,"external_id":null},{"id":10847704003,"name":"Technische Universität Dresden","priority":1882,"external_id":null},{"id":10847705003,"name":"Macquarie University","priority":1883,"external_id":null},{"id":10847706003,"name":"Vienna University of Technology","priority":1884,"external_id":null},{"id":10847707003,"name":"Royal Holloway University of London","priority":1885,"external_id":null},{"id":10847708003,"name":"Victoria University of Wellington","priority":1886,"external_id":null},{"id":10847709003,"name":"University of Padua","priority":1887,"external_id":null},{"id":10847710003,"name":"Universiti Kebangsaan Malaysia (UKM)","priority":1888,"external_id":null},{"id":10847711003,"name":"University of Technology, Sydney","priority":1889,"external_id":null},{"id":10847712003,"name":"Universität Konstanz","priority":1890,"external_id":null},{"id":10847713003,"name":"Universidad de Los Andes Colombia","priority":1891,"external_id":null},{"id":10847714003,"name":"Université Paris Descartes","priority":1892,"external_id":null},{"id":10847715003,"name":"Tokyo Medical and Dental University","priority":1893,"external_id":null},{"id":10847716003,"name":"University of Wollongong","priority":1894,"external_id":null},{"id":10847717003,"name":"Universität Erlangen-Nürnberg","priority":1895,"external_id":null},{"id":10847718003,"name":"Queensland University of Technology","priority":1896,"external_id":null},{"id":10847719003,"name":"Tecnológico de Monterrey (ITESM)","priority":1897,"external_id":null},{"id":10847720003,"name":"Universität Mannheim","priority":1898,"external_id":null},{"id":10847721003,"name":"Universitat Pompeu Fabra","priority":1899,"external_id":null},{"id":10847722003,"name":"Mahidol University","priority":1900,"external_id":null},{"id":10847723003,"name":"Curtin University","priority":1901,"external_id":null},{"id":10847724003,"name":"National University of Ireland, Galway","priority":1902,"external_id":null},{"id":10847725003,"name":"Universidade Federal do Rio de Janeiro","priority":1903,"external_id":null},{"id":10847726003,"name":"University of Surrey","priority":1904,"external_id":null},{"id":10847727003,"name":"Hong Kong Baptist University","priority":1905,"external_id":null},{"id":10847728003,"name":"Umeå University","priority":1906,"external_id":null},{"id":10847729003,"name":"Universität Innsbruck","priority":1907,"external_id":null},{"id":10847730003,"name":"RMIT University","priority":1908,"external_id":null},{"id":10847731003,"name":"University of Eastern Finland","priority":1909,"external_id":null},{"id":10847732003,"name":"Christian-Albrechts-Universität zu Kiel","priority":1910,"external_id":null},{"id":10847733003,"name":"Indian Institute of Technology Kanpur (IITK)","priority":1911,"external_id":null},{"id":10847734003,"name":"National Yang Ming University","priority":1912,"external_id":null},{"id":10847735003,"name":"Johannes Gutenberg Universität Mainz","priority":1913,"external_id":null},{"id":10847736003,"name":"The University of Newcastle","priority":1914,"external_id":null},{"id":10847737003,"name":"Al-Farabi Kazakh National University","priority":1915,"external_id":null},{"id":10847738003,"name":"École des Ponts ParisTech","priority":1916,"external_id":null},{"id":10847739003,"name":"University of Jyväskylä","priority":1917,"external_id":null},{"id":10847740003,"name":"L.N. Gumilyov Eurasian National University","priority":1918,"external_id":null},{"id":10847741003,"name":"Kobe University","priority":1919,"external_id":null},{"id":10847742003,"name":"University of Tromso","priority":1920,"external_id":null},{"id":10847743003,"name":"Hiroshima University","priority":1921,"external_id":null},{"id":10847744003,"name":"Université Bordeaux 1, Sciences Technologies","priority":1922,"external_id":null},{"id":10847745003,"name":"University of Indonesia","priority":1923,"external_id":null},{"id":10847746003,"name":"Universität Leipzig","priority":1924,"external_id":null},{"id":10847747003,"name":"University of Southern Denmark","priority":1925,"external_id":null},{"id":10847748003,"name":"Indian Institute of Technology Madras (IITM)","priority":1926,"external_id":null},{"id":10847749003,"name":"University of The Witwatersrand","priority":1927,"external_id":null},{"id":10847750003,"name":"University of Navarra","priority":1928,"external_id":null},{"id":10847751003,"name":"Universidad Austral - Argentina","priority":1929,"external_id":null},{"id":10847752003,"name":"Universidad Carlos III de Madrid","priority":1930,"external_id":null},{"id":10847753003,"name":"Università¡ degli Studi di Roma - Tor Vergata","priority":1931,"external_id":null},{"id":10847754003,"name":"Pontificia Universidad Católica Argentina Santa María de los Buenos Aires","priority":1932,"external_id":null},{"id":10847755003,"name":"UCA","priority":1933,"external_id":null},{"id":10847756003,"name":"Julius-Maximilians-Universität Würzburg","priority":1934,"external_id":null},{"id":10847757003,"name":"Universidad Nacional de Colombia","priority":1935,"external_id":null},{"id":10847758003,"name":"Laval University","priority":1936,"external_id":null},{"id":10847759003,"name":"Ben Gurion University of The Negev","priority":1937,"external_id":null},{"id":10847760003,"name":"Linköping University","priority":1938,"external_id":null},{"id":10847761003,"name":"Aalborg University","priority":1939,"external_id":null},{"id":10847762003,"name":"Bauman Moscow State Technical University","priority":1940,"external_id":null},{"id":10847763003,"name":"Ecole Normale Supérieure de Cachan","priority":1941,"external_id":null},{"id":10847764003,"name":"SOAS - School of Oriental and African Studies, University of London","priority":1942,"external_id":null},{"id":10847765003,"name":"University of Essex","priority":1943,"external_id":null},{"id":10847766003,"name":"University of Warsaw","priority":1944,"external_id":null},{"id":10847767003,"name":"Griffith University","priority":1945,"external_id":null},{"id":10847768003,"name":"University of South Australia","priority":1946,"external_id":null},{"id":10847769003,"name":"Massey University","priority":1947,"external_id":null},{"id":10847770003,"name":"University of Porto","priority":1948,"external_id":null},{"id":10847771003,"name":"Universitat Politècnica de Catalunya","priority":1949,"external_id":null},{"id":10847772003,"name":"Indian Institute of Technology Kharagpur (IITKGP)","priority":1950,"external_id":null},{"id":10847773003,"name":"City University London","priority":1951,"external_id":null},{"id":10847774003,"name":"Dublin City University","priority":1952,"external_id":null},{"id":10847775003,"name":"Pontificia Universidad Javeriana","priority":1953,"external_id":null},{"id":10847776003,"name":"James Cook University","priority":1954,"external_id":null},{"id":10847777003,"name":"Novosibirsk State University","priority":1955,"external_id":null},{"id":10847778003,"name":"Universidade Nova de Lisboa","priority":1956,"external_id":null},{"id":10847779003,"name":"Université Aix-Marseille","priority":1957,"external_id":null},{"id":10847780003,"name":"Universiti Sains Malaysia (USM)","priority":1958,"external_id":null},{"id":10847781003,"name":"Universiti Teknologi Malaysia (UTM)","priority":1959,"external_id":null},{"id":10847782003,"name":"Université Paris Dauphine","priority":1960,"external_id":null},{"id":10847783003,"name":"University of Coimbra","priority":1961,"external_id":null},{"id":10847784003,"name":"Brunel University","priority":1962,"external_id":null},{"id":10847785003,"name":"King Abdul Aziz University (KAU)","priority":1963,"external_id":null},{"id":10847786003,"name":"Ewha Womans University","priority":1964,"external_id":null},{"id":10847787003,"name":"Nankai University","priority":1965,"external_id":null},{"id":10847788003,"name":"Taipei Medical University","priority":1966,"external_id":null},{"id":10847789003,"name":"Universität Jena","priority":1967,"external_id":null},{"id":10847790003,"name":"Ruhr-Universität Bochum","priority":1968,"external_id":null},{"id":10847791003,"name":"Heriot-Watt University","priority":1969,"external_id":null},{"id":10847792003,"name":"Politecnico di Torino","priority":1970,"external_id":null},{"id":10847793003,"name":"Universität Bremen","priority":1971,"external_id":null},{"id":10847794003,"name":"Xi'an Jiaotong University","priority":1972,"external_id":null},{"id":10847795003,"name":"Birkbeck College, University of London","priority":1973,"external_id":null},{"id":10847796003,"name":"Oxford Brookes University","priority":1974,"external_id":null},{"id":10847797003,"name":"Jagiellonian University","priority":1975,"external_id":null},{"id":10847798003,"name":"University of Tampere","priority":1976,"external_id":null},{"id":10847799003,"name":"University of Florence","priority":1977,"external_id":null},{"id":10847800003,"name":"Deakin University","priority":1978,"external_id":null},{"id":10847801003,"name":"University of the Philippines","priority":1979,"external_id":null},{"id":10847802003,"name":"Universitat Politècnica de València","priority":1980,"external_id":null},{"id":10847803003,"name":"Sun Yat-sen University","priority":1981,"external_id":null},{"id":10847804003,"name":"Université Montpellier 2, Sciences et Techniques du Languedoc","priority":1982,"external_id":null},{"id":10847805003,"name":"Moscow State Institute of International Relations (MGIMO-University)","priority":1983,"external_id":null},{"id":10847806003,"name":"Stellenbosch University","priority":1984,"external_id":null},{"id":10847807003,"name":"Politécnica de Madrid","priority":1985,"external_id":null},{"id":10847808003,"name":"Instituto Tecnológico de Buenos Aires (ITBA)","priority":1986,"external_id":null},{"id":10847809003,"name":"La Trobe University","priority":1987,"external_id":null},{"id":10847810003,"name":"Université Paul Sabatier Toulouse III","priority":1988,"external_id":null},{"id":10847811003,"name":"Karl-Franzens-Universität Graz","priority":1989,"external_id":null},{"id":10847812003,"name":"Universität Düsseldorf","priority":1990,"external_id":null},{"id":10847813003,"name":"University of Naples - Federico Ii","priority":1991,"external_id":null},{"id":10847814003,"name":"Aston University","priority":1992,"external_id":null},{"id":10847815003,"name":"University of Turin","priority":1993,"external_id":null},{"id":10847816003,"name":"Beihang University (former BUAA)","priority":1994,"external_id":null},{"id":10847817003,"name":"Indian Institute of Technology Roorkee (IITR)","priority":1995,"external_id":null},{"id":10847818003,"name":"National Central University","priority":1996,"external_id":null},{"id":10847819003,"name":"Sogang University","priority":1997,"external_id":null},{"id":10847820003,"name":"Universität Regensburg","priority":1998,"external_id":null},{"id":10847821003,"name":"Université Lille 1, Sciences et Technologie","priority":1999,"external_id":null},{"id":10847822003,"name":"University of Tasmania","priority":2000,"external_id":null},{"id":10847823003,"name":"University of Waikato","priority":2001,"external_id":null},{"id":10847824003,"name":"Wuhan University","priority":2002,"external_id":null},{"id":10847825003,"name":"National Taiwan University of Science And Technology","priority":2003,"external_id":null},{"id":10847826003,"name":"Universidade Federal de São Paulo (UNIFESP)","priority":2004,"external_id":null},{"id":10847827003,"name":"Università degli Studi di Pavia","priority":2005,"external_id":null},{"id":10847828003,"name":"Universität Bayreuth","priority":2006,"external_id":null},{"id":10847829003,"name":"Université Claude Bernard Lyon 1","priority":2007,"external_id":null},{"id":10847830003,"name":"Université du Québec","priority":2008,"external_id":null},{"id":10847831003,"name":"Universiti Putra Malaysia (UPM)","priority":2009,"external_id":null},{"id":10847832003,"name":"University of Kent","priority":2010,"external_id":null},{"id":10847833003,"name":"University of St Gallen (HSG)","priority":2011,"external_id":null},{"id":10847834003,"name":"Bond University","priority":2012,"external_id":null},{"id":10847835003,"name":"United Arab Emirates University","priority":2013,"external_id":null},{"id":10847836003,"name":"Universidad de San Andrés","priority":2014,"external_id":null},{"id":10847837003,"name":"Universidad Nacional de La Plata","priority":2015,"external_id":null},{"id":10847838003,"name":"Universität des Saarlandes","priority":2016,"external_id":null},{"id":10847839003,"name":"American University of Sharjah (AUS)","priority":2017,"external_id":null},{"id":10847840003,"name":"Bilkent University","priority":2018,"external_id":null},{"id":10847841003,"name":"Flinders University","priority":2019,"external_id":null},{"id":10847842003,"name":"Hankuk (Korea) University of Foreign Studies","priority":2020,"external_id":null},{"id":10847843003,"name":"Middle East Technical University","priority":2021,"external_id":null},{"id":10847844003,"name":"Philipps-Universität Marburg","priority":2022,"external_id":null},{"id":10847845003,"name":"Swansea University","priority":2023,"external_id":null},{"id":10847846003,"name":"Tampere University of Technology","priority":2024,"external_id":null},{"id":10847847003,"name":"Universität Bielefeld","priority":2025,"external_id":null},{"id":10847848003,"name":"University of Manitoba","priority":2026,"external_id":null},{"id":10847849003,"name":"Chiba University","priority":2027,"external_id":null},{"id":10847850003,"name":"Moscow Institute of Physics and Technology State University","priority":2028,"external_id":null},{"id":10847851003,"name":"Tallinn University of Technology","priority":2029,"external_id":null},{"id":10847852003,"name":"Taras Shevchenko National University of Kyiv","priority":2030,"external_id":null},{"id":10847853003,"name":"Tokyo University of Science","priority":2031,"external_id":null},{"id":10847854003,"name":"University of Salamanca","priority":2032,"external_id":null},{"id":10847855003,"name":"University of Trento","priority":2033,"external_id":null},{"id":10847856003,"name":"Université de Sherbrooke","priority":2034,"external_id":null},{"id":10847857003,"name":"Université Panthéon-Assas (Paris 2)","priority":2035,"external_id":null},{"id":10847858003,"name":"University of Delhi","priority":2036,"external_id":null},{"id":10847859003,"name":"Abo Akademi University","priority":2037,"external_id":null},{"id":10847860003,"name":"Czech Technical University In Prague","priority":2038,"external_id":null},{"id":10847861003,"name":"Leibniz Universität Hannover","priority":2039,"external_id":null},{"id":10847862003,"name":"Pusan National University","priority":2040,"external_id":null},{"id":10847863003,"name":"Shanghai University","priority":2041,"external_id":null},{"id":10847864003,"name":"St. Petersburg State Politechnical University","priority":2042,"external_id":null},{"id":10847865003,"name":"Università Cattolica del Sacro Cuore","priority":2043,"external_id":null},{"id":10847866003,"name":"University of Genoa","priority":2044,"external_id":null},{"id":10847867003,"name":"Bandung Institute of Technology (ITB)","priority":2045,"external_id":null},{"id":10847868003,"name":"Bogazici University","priority":2046,"external_id":null},{"id":10847869003,"name":"Goldsmiths, University of London","priority":2047,"external_id":null},{"id":10847870003,"name":"National Sun Yat-sen University","priority":2048,"external_id":null},{"id":10847871003,"name":"Renmin (People’s) University of China","priority":2049,"external_id":null},{"id":10847872003,"name":"Universidad de Costa Rica","priority":2050,"external_id":null},{"id":10847873003,"name":"Universidad de Santiago de Chile - USACH","priority":2051,"external_id":null},{"id":10847874003,"name":"University of Tartu","priority":2052,"external_id":null},{"id":10847875003,"name":"Aristotle University of Thessaloniki","priority":2053,"external_id":null},{"id":10847876003,"name":"Auckland University of Technology","priority":2054,"external_id":null},{"id":10847877003,"name":"Bangor University","priority":2055,"external_id":null},{"id":10847878003,"name":"Charles Darwin University","priority":2056,"external_id":null},{"id":10847879003,"name":"Kingston University, London","priority":2057,"external_id":null},{"id":10847880003,"name":"Universitat de Valencia","priority":2058,"external_id":null},{"id":10847881003,"name":"Université Montpellier 1","priority":2059,"external_id":null},{"id":10847882003,"name":"University of Pretoria","priority":2060,"external_id":null},{"id":10847883003,"name":"Lincoln University","priority":2061,"external_id":null},{"id":10847884003,"name":"National Taiwan Normal University","priority":2062,"external_id":null},{"id":10847885003,"name":"National University of Sciences And Technology (NUST) Islamabad","priority":2063,"external_id":null},{"id":10847886003,"name":"Swinburne University of Technology","priority":2064,"external_id":null},{"id":10847887003,"name":"Tongji University","priority":2065,"external_id":null},{"id":10847888003,"name":"Universidad de Zaragoza","priority":2066,"external_id":null},{"id":10847889003,"name":"Universidade Federal de Minas Gerais","priority":2067,"external_id":null},{"id":10847890003,"name":"Universität Duisburg-Essen","priority":2068,"external_id":null},{"id":10847891003,"name":"Al-Imam Mohamed Ibn Saud Islamic University","priority":2069,"external_id":null},{"id":10847892003,"name":"Harbin Institute of Technology","priority":2070,"external_id":null},{"id":10847893003,"name":"People's Friendship University of Russia","priority":2071,"external_id":null},{"id":10847894003,"name":"Universidade Estadual PaulistaJúlio de Mesquita Filho' (UNESP)","priority":2072,"external_id":null},{"id":10847895003,"name":"Université Nice Sophia-Antipolis","priority":2073,"external_id":null},{"id":10847896003,"name":"University of Crete","priority":2074,"external_id":null},{"id":10847897003,"name":"University of Milano-Bicocca","priority":2075,"external_id":null},{"id":10847898003,"name":"Ateneo de Manila University","priority":2076,"external_id":null},{"id":10847899003,"name":"Beijing Institute of Technology","priority":2077,"external_id":null},{"id":10847900003,"name":"Chang Gung University","priority":2078,"external_id":null},{"id":10847901003,"name":"hung-Ang University","priority":2079,"external_id":null},{"id":10847902003,"name":"Dublin Institute of Technology","priority":2080,"external_id":null},{"id":10847903003,"name":"Huazhong University of Science and Technology","priority":2081,"external_id":null},{"id":10847904003,"name":"International Islamic University Malaysia (IIUM)","priority":2082,"external_id":null},{"id":10847905003,"name":"Johannes Kepler University Linz","priority":2083,"external_id":null},{"id":10847906003,"name":"Justus-Liebig-Universität Gießen","priority":2084,"external_id":null},{"id":10847907003,"name":"Kanazawa University","priority":2085,"external_id":null},{"id":10847908003,"name":"Keele University","priority":2086,"external_id":null},{"id":10847909003,"name":"Koc University","priority":2087,"external_id":null},{"id":10847910003,"name":"National and Kapodistrian University of Athens","priority":2088,"external_id":null},{"id":10847911003,"name":"National Research University – Higher School of Economics (HSE)","priority":2089,"external_id":null},{"id":10847912003,"name":"National Technical University of Athens","priority":2090,"external_id":null},{"id":10847913003,"name":"Okayama University","priority":2091,"external_id":null},{"id":10847914003,"name":"Sabanci University","priority":2092,"external_id":null},{"id":10847915003,"name":"Southeast University","priority":2093,"external_id":null},{"id":10847916003,"name":"Sultan Qaboos University","priority":2094,"external_id":null},{"id":10847917003,"name":"Technische Universität Braunschweig","priority":2095,"external_id":null},{"id":10847918003,"name":"Technische Universität Dortmund","priority":2096,"external_id":null},{"id":10847919003,"name":"The Catholic University of Korea","priority":2097,"external_id":null},{"id":10847920003,"name":"Tianjin University","priority":2098,"external_id":null},{"id":10847921003,"name":"Tokyo Metropolitan University","priority":2099,"external_id":null},{"id":10847922003,"name":"Universidad de Antioquia","priority":2100,"external_id":null},{"id":10847923003,"name":"University of Granada","priority":2101,"external_id":null},{"id":10847924003,"name":"Universidad de Palermo","priority":2102,"external_id":null},{"id":10847925003,"name":"Universidad Nacional de Córdoba","priority":2103,"external_id":null},{"id":10847926003,"name":"Universidade de Santiago de Compostela","priority":2104,"external_id":null},{"id":10847927003,"name":"Universidade Federal do Rio Grande Do Sul","priority":2105,"external_id":null},{"id":10847928003,"name":"University of Siena","priority":2106,"external_id":null},{"id":10847929003,"name":"University of Trieste","priority":2107,"external_id":null},{"id":10847930003,"name":"Universitas Gadjah Mada","priority":2108,"external_id":null},{"id":10847931003,"name":"Université de Lorraine","priority":2109,"external_id":null},{"id":10847932003,"name":"Université de Rennes 1","priority":2110,"external_id":null},{"id":10847933003,"name":"University of Bradford","priority":2111,"external_id":null},{"id":10847934003,"name":"University of Hull","priority":2112,"external_id":null},{"id":10847935003,"name":"University of Kwazulu-Natal","priority":2113,"external_id":null},{"id":10847936003,"name":"University of Limerick","priority":2114,"external_id":null},{"id":10847937003,"name":"University of Stirling","priority":2115,"external_id":null},{"id":10847938003,"name":"University of Szeged","priority":2116,"external_id":null},{"id":10847939003,"name":"Ural Federal University","priority":2117,"external_id":null},{"id":10847940003,"name":"Xiamen University","priority":2118,"external_id":null},{"id":10847941003,"name":"Yokohama City University","priority":2119,"external_id":null},{"id":10847942003,"name":"Aberystwyth University","priority":2120,"external_id":null},{"id":10847943003,"name":"Belarus State University","priority":2121,"external_id":null},{"id":10847944003,"name":"Cairo University","priority":2122,"external_id":null},{"id":10847945003,"name":"Chiang Mai University","priority":2123,"external_id":null},{"id":10847946003,"name":"Chonbuk National University","priority":2124,"external_id":null},{"id":10847947003,"name":"Eötvös Loránd University","priority":2125,"external_id":null},{"id":10847948003,"name":"Inha University","priority":2126,"external_id":null},{"id":10847949003,"name":"Instituto Politécnico Nacional (IPN)","priority":2127,"external_id":null},{"id":10847950003,"name":"Istanbul Technical University","priority":2128,"external_id":null},{"id":10847951003,"name":"Kumamoto University","priority":2129,"external_id":null},{"id":10847952003,"name":"Kyungpook National University","priority":2130,"external_id":null},{"id":10847953003,"name":"Lingnan University (Hong Kong)","priority":2131,"external_id":null},{"id":10847954003,"name":"Masaryk University","priority":2132,"external_id":null},{"id":10847955003,"name":"Murdoch University","priority":2133,"external_id":null},{"id":10847956003,"name":"Nagasaki University","priority":2134,"external_id":null},{"id":10847957003,"name":"National Chung Hsing University","priority":2135,"external_id":null},{"id":10847958003,"name":"National Taipei University of Technology","priority":2136,"external_id":null},{"id":10847959003,"name":"National University of Ireland Maynooth","priority":2137,"external_id":null},{"id":10847960003,"name":"Osaka City University","priority":2138,"external_id":null},{"id":10847961003,"name":"Pontificia Universidad Católica del Perú","priority":2139,"external_id":null},{"id":10847962003,"name":"Pontificia Universidade Católica de São Paulo (PUC -SP)","priority":2140,"external_id":null},{"id":10847963003,"name":"Pontificia Universidade Católica do Rio de Janeiro (PUC - Rio)","priority":2141,"external_id":null},{"id":10847964003,"name":"Qatar University","priority":2142,"external_id":null},{"id":10847965003,"name":"Rhodes University","priority":2143,"external_id":null},{"id":10847966003,"name":"Tokyo University of Agriculture and Technology","priority":2144,"external_id":null},{"id":10847967003,"name":"Tomsk Polytechnic University","priority":2145,"external_id":null},{"id":10847968003,"name":"Tomsk State University","priority":2146,"external_id":null},{"id":10847969003,"name":"Umm Al-Qura University","priority":2147,"external_id":null},{"id":10847970003,"name":"Universidad Católica Andrés Bello - UCAB","priority":2148,"external_id":null},{"id":10847971003,"name":"Universidad Central de Venezuela - UCV","priority":2149,"external_id":null},{"id":10847972003,"name":"Universidad de Belgrano","priority":2150,"external_id":null},{"id":10847973003,"name":"Universidad de Concepción","priority":2151,"external_id":null},{"id":10847974003,"name":"Universidad de Sevilla","priority":2152,"external_id":null},{"id":10847975003,"name":"Universidade Catolica Portuguesa, Lisboa","priority":2153,"external_id":null},{"id":10847976003,"name":"Universidade de Brasilia (UnB)","priority":2154,"external_id":null},{"id":10847977003,"name":"University of Lisbon","priority":2155,"external_id":null},{"id":10847978003,"name":"University of Ljubljana","priority":2156,"external_id":null},{"id":10847979003,"name":"University of Seoul","priority":2157,"external_id":null},{"id":10847980003,"name":"Abu Dhabi University","priority":2158,"external_id":null},{"id":10847981003,"name":"Ain Shams University","priority":2159,"external_id":null},{"id":10847982003,"name":"Ajou University","priority":2160,"external_id":null},{"id":10847983003,"name":"De La Salle University","priority":2161,"external_id":null},{"id":10847984003,"name":"Dongguk University","priority":2162,"external_id":null},{"id":10847985003,"name":"Gifu University","priority":2163,"external_id":null},{"id":10847986003,"name":"Hacettepe University","priority":2164,"external_id":null},{"id":10847987003,"name":"Indian Institute of Technology Guwahati (IITG)","priority":2165,"external_id":null},{"id":10847988003,"name":"Jilin University","priority":2166,"external_id":null},{"id":10847989003,"name":"Kazan Federal University","priority":2167,"external_id":null},{"id":10847990003,"name":"King Khalid University","priority":2168,"external_id":null},{"id":10847991003,"name":"Martin-Luther-Universität Halle-Wittenberg","priority":2169,"external_id":null},{"id":10847992003,"name":"National Chengchi University","priority":2170,"external_id":null},{"id":10847993003,"name":"National Technical University of UkraineKyiv Polytechnic Institute'","priority":2171,"external_id":null},{"id":10847994003,"name":"Niigata University","priority":2172,"external_id":null},{"id":10847995003,"name":"Osaka Prefecture University","priority":2173,"external_id":null},{"id":10847996003,"name":"Paris Lodron University of Salzburg","priority":2174,"external_id":null},{"id":10847997003,"name":"Sharif University of Technology","priority":2175,"external_id":null},{"id":10847998003,"name":"Southern Federal University","priority":2176,"external_id":null},{"id":10847999003,"name":"Thammasat University","priority":2177,"external_id":null},{"id":10848000003,"name":"Universidad de Guadalajara (UDG)","priority":2178,"external_id":null},{"id":10848001003,"name":"Universidad de la República (UdelaR)","priority":2179,"external_id":null},{"id":10848002003,"name":"Universidad Iberoamericana (UIA)","priority":2180,"external_id":null},{"id":10848003003,"name":"Universidad Torcuato Di Tella","priority":2181,"external_id":null},{"id":10848004003,"name":"Universidade Federal da Bahia","priority":2182,"external_id":null},{"id":10848005003,"name":"Universidade Federal de São Carlos","priority":2183,"external_id":null},{"id":10848006003,"name":"Universidade Federal de Viçosa","priority":2184,"external_id":null},{"id":10848007003,"name":"Perugia University","priority":2185,"external_id":null},{"id":10848008003,"name":"Université de Nantes","priority":2186,"external_id":null},{"id":10848009003,"name":"Université Saint-Joseph de Beyrouth","priority":2187,"external_id":null},{"id":10848010003,"name":"University of Canberra","priority":2188,"external_id":null},{"id":10848011003,"name":"University of Debrecen","priority":2189,"external_id":null},{"id":10848012003,"name":"University of Johannesburg","priority":2190,"external_id":null},{"id":10848013003,"name":"University of Mumbai","priority":2191,"external_id":null},{"id":10848014003,"name":"University of Patras","priority":2192,"external_id":null},{"id":10848015003,"name":"University of Tehran","priority":2193,"external_id":null},{"id":10848016003,"name":"University of Ulsan","priority":2194,"external_id":null},{"id":10848017003,"name":"University of Ulster","priority":2195,"external_id":null},{"id":10848018003,"name":"University of Zagreb","priority":2196,"external_id":null},{"id":10848019003,"name":"Vilnius University","priority":2197,"external_id":null},{"id":10848020003,"name":"Warsaw University of Technology","priority":2198,"external_id":null},{"id":10848021003,"name":"Al Azhar University","priority":2199,"external_id":null},{"id":10848022003,"name":"Bar-Ilan University","priority":2200,"external_id":null},{"id":10848023003,"name":"Brno University of Technology","priority":2201,"external_id":null},{"id":10848024003,"name":"Chonnam National University","priority":2202,"external_id":null},{"id":10848025003,"name":"Chungnam National University","priority":2203,"external_id":null},{"id":10848026003,"name":"Corvinus University of Budapest","priority":2204,"external_id":null},{"id":10848027003,"name":"Gunma University","priority":2205,"external_id":null},{"id":10848028003,"name":"Hallym University","priority":2206,"external_id":null},{"id":10848029003,"name":"Instituto Tecnológico Autonomo de México (ITAM)","priority":2207,"external_id":null},{"id":10848030003,"name":"Istanbul University","priority":2208,"external_id":null},{"id":10848031003,"name":"Jordan University of Science & Technology","priority":2209,"external_id":null},{"id":10848032003,"name":"Kasetsart University","priority":2210,"external_id":null},{"id":10848033003,"name":"Kazakh-British Technical University","priority":2211,"external_id":null},{"id":10848034003,"name":"Khazar University","priority":2212,"external_id":null},{"id":10848035003,"name":"London Metropolitan University","priority":2213,"external_id":null},{"id":10848036003,"name":"Middlesex University","priority":2214,"external_id":null},{"id":10848037003,"name":"Universidad Industrial de Santander","priority":2215,"external_id":null},{"id":10848038003,"name":"Pontificia Universidad Católica de Valparaíso","priority":2216,"external_id":null},{"id":10848039003,"name":"Pontificia Universidade Católica do Rio Grande do Sul","priority":2217,"external_id":null},{"id":10848040003,"name":"Qafqaz University","priority":2218,"external_id":null},{"id":10848041003,"name":"Ritsumeikan University","priority":2219,"external_id":null},{"id":10848042003,"name":"Shandong University","priority":2220,"external_id":null},{"id":10848043003,"name":"University of St. Kliment Ohridski","priority":2221,"external_id":null},{"id":10848044003,"name":"South Kazakhstan State University (SKSU)","priority":2222,"external_id":null},{"id":10848045003,"name":"Universidad Adolfo Ibáñez","priority":2223,"external_id":null},{"id":10848046003,"name":"Universidad Autónoma del Estado de México","priority":2224,"external_id":null},{"id":10848047003,"name":"Universidad Autónoma Metropolitana (UAM)","priority":2225,"external_id":null},{"id":10848048003,"name":"Universidad de Alcalá","priority":2226,"external_id":null},{"id":10848049003,"name":"Universidad Nacional Costa Rica","priority":2227,"external_id":null},{"id":10848050003,"name":"Universidad Nacional de Mar del Plata","priority":2228,"external_id":null},{"id":10848051003,"name":"Universidad Peruana Cayetano Heredia","priority":2229,"external_id":null},{"id":10848052003,"name":"Universidad Simón Bolívar Venezuela","priority":2230,"external_id":null},{"id":10848053003,"name":"Universidade Federal de Santa Catarina","priority":2231,"external_id":null},{"id":10848054003,"name":"Universidade Federal do Paraná (UFPR)","priority":2232,"external_id":null},{"id":10848055003,"name":"Universidade Federal Fluminense","priority":2233,"external_id":null},{"id":10848056003,"name":"University of Modena","priority":2234,"external_id":null},{"id":10848057003,"name":"Université Lumière Lyon 2","priority":2235,"external_id":null},{"id":10848058003,"name":"Université Toulouse 1, Capitole","priority":2236,"external_id":null},{"id":10848059003,"name":"University of Economics Prague","priority":2237,"external_id":null},{"id":10848060003,"name":"University of Hertfordshire","priority":2238,"external_id":null},{"id":10848061003,"name":"University of Plymouth","priority":2239,"external_id":null},{"id":10848062003,"name":"University of Salford","priority":2240,"external_id":null},{"id":10848063003,"name":"University of Science and Technology Beijing","priority":2241,"external_id":null},{"id":10848064003,"name":"University of Western Sydney","priority":2242,"external_id":null},{"id":10848065003,"name":"Yamaguchi University","priority":2243,"external_id":null},{"id":10848066003,"name":"Yokohama National University","priority":2244,"external_id":null},{"id":10848067003,"name":"Airlangga University","priority":2245,"external_id":null},{"id":10848068003,"name":"Alexandria University","priority":2246,"external_id":null},{"id":10848069003,"name":"Alexandru Ioan Cuza University","priority":2247,"external_id":null},{"id":10848070003,"name":"Alpen-Adria-Universität Klagenfurt","priority":2248,"external_id":null},{"id":10848071003,"name":"Aoyama Gakuin University","priority":2249,"external_id":null},{"id":10848072003,"name":"Athens University of Economy And Business","priority":2250,"external_id":null},{"id":10848073003,"name":"Babes-Bolyai University","priority":2251,"external_id":null},{"id":10848074003,"name":"Baku State University","priority":2252,"external_id":null},{"id":10848075003,"name":"Belarusian National Technical University","priority":2253,"external_id":null},{"id":10848076003,"name":"Benemérita Universidad Autónoma de Puebla","priority":2254,"external_id":null},{"id":10848077003,"name":"Bogor Agricultural University","priority":2255,"external_id":null},{"id":10848078003,"name":"Coventry University","priority":2256,"external_id":null},{"id":10848079003,"name":"Cukurova University","priority":2257,"external_id":null},{"id":10848080003,"name":"Diponegoro University","priority":2258,"external_id":null},{"id":10848081003,"name":"Donetsk National University","priority":2259,"external_id":null},{"id":10848082003,"name":"Doshisha University","priority":2260,"external_id":null},{"id":10848083003,"name":"E.A.Buketov Karaganda State University","priority":2261,"external_id":null},{"id":10848084003,"name":"Far Eastern Federal University","priority":2262,"external_id":null},{"id":10848085003,"name":"Fu Jen Catholic University","priority":2263,"external_id":null},{"id":10848086003,"name":"Kagoshima University","priority":2264,"external_id":null},{"id":10848087003,"name":"Kaunas University of Technology","priority":2265,"external_id":null},{"id":10848088003,"name":"Kazakh Ablai khan University of International Relations and World Languages","priority":2266,"external_id":null},{"id":10848089003,"name":"Kazakh National Pedagogical University Abai","priority":2267,"external_id":null},{"id":10848090003,"name":"Kazakh National Technical University","priority":2268,"external_id":null},{"id":10848091003,"name":"Khon Kaen University","priority":2269,"external_id":null},{"id":10848092003,"name":"King Faisal University","priority":2270,"external_id":null},{"id":10848093003,"name":"King Mongkut''s University of Technology Thonburi","priority":2271,"external_id":null},{"id":10848094003,"name":"Kuwait University","priority":2272,"external_id":null},{"id":10848095003,"name":"Lodz University","priority":2273,"external_id":null},{"id":10848096003,"name":"Manchester Metropolitan University","priority":2274,"external_id":null},{"id":10848097003,"name":"Lobachevsky State University of Nizhni Novgorod","priority":2275,"external_id":null},{"id":10848098003,"name":"National Technical UniversityKharkiv Polytechnic Institute'","priority":2276,"external_id":null},{"id":10848099003,"name":"Nicolaus Copernicus University","priority":2277,"external_id":null},{"id":10848100003,"name":"Northumbria University at Newcastle","priority":2278,"external_id":null},{"id":10848101003,"name":"Nottingham Trent University","priority":2279,"external_id":null},{"id":10848102003,"name":"Ochanomizu University","priority":2280,"external_id":null},{"id":10848103003,"name":"Plekhanov Russian University of Economics","priority":2281,"external_id":null},{"id":10848104003,"name":"Pontificia Universidad Catolica del Ecuador","priority":2282,"external_id":null},{"id":10848105003,"name":"Prince of Songkla University","priority":2283,"external_id":null},{"id":10848106003,"name":"S.Seifullin Kazakh Agro Technical University","priority":2284,"external_id":null},{"id":10848107003,"name":"Saitama University","priority":2285,"external_id":null},{"id":10848108003,"name":"Sepuluh Nopember Institute of Technology","priority":2286,"external_id":null},{"id":10848109003,"name":"Shinshu University","priority":2287,"external_id":null},{"id":10848110003,"name":"The Robert Gordon University","priority":2288,"external_id":null},{"id":10848111003,"name":"Tokai University","priority":2289,"external_id":null},{"id":10848112003,"name":"Universidad ANAHUAC","priority":2290,"external_id":null},{"id":10848113003,"name":"Universidad Austral de Chile","priority":2291,"external_id":null},{"id":10848114003,"name":"University Autónoma de Nuevo León (UANL)","priority":2292,"external_id":null},{"id":10848115003,"name":"Universidad de la Habana","priority":2293,"external_id":null},{"id":10848116003,"name":"Universidad de La Sabana","priority":2294,"external_id":null},{"id":10848117003,"name":"Universidad de las Américas Puebla (UDLAP)","priority":2295,"external_id":null},{"id":10848118003,"name":"Universidad de los Andes Mérida","priority":2296,"external_id":null},{"id":10848119003,"name":"University of Murcia","priority":2297,"external_id":null},{"id":10848120003,"name":"Universidad de Puerto Rico","priority":2298,"external_id":null},{"id":10848121003,"name":"Universidad de San Francisco de Quito","priority":2299,"external_id":null},{"id":10848122003,"name":"Universidad de Talca","priority":2300,"external_id":null},{"id":10848123003,"name":"Universidad del Norte","priority":2301,"external_id":null},{"id":10848124003,"name":"Universidad del Rosario","priority":2302,"external_id":null},{"id":10848125003,"name":"Universidad del Valle","priority":2303,"external_id":null},{"id":10848126003,"name":"Universidad Nacional de Cuyo","priority":2304,"external_id":null},{"id":10848127003,"name":"Universidad Nacional de Rosario","priority":2305,"external_id":null},{"id":10848128003,"name":"Universidad Nacional de Tucumán","priority":2306,"external_id":null},{"id":10848129003,"name":"Universidad Nacional del Sur","priority":2307,"external_id":null},{"id":10848130003,"name":"Universidad Nacional Mayor de San Marcos","priority":2308,"external_id":null},{"id":10848131003,"name":"Universidad Técnica Federico Santa María","priority":2309,"external_id":null},{"id":10848132003,"name":"Universidad Tecnológica Nacional (UTN)","priority":2310,"external_id":null},{"id":10848133003,"name":"Universidade do Estado do Rio de Janeiro (UERJ)","priority":2311,"external_id":null},{"id":10848134003,"name":"Universidade Estadual de Londrina (UEL)","priority":2312,"external_id":null},{"id":10848135003,"name":"Universidade Federal de Santa Maria","priority":2313,"external_id":null},{"id":10848136003,"name":"Universidade Federal do Ceará (UFC)","priority":2314,"external_id":null},{"id":10848137003,"name":"Universidade Federal do Pernambuco","priority":2315,"external_id":null},{"id":10848138003,"name":"Università Ca'' Foscari Venezia","priority":2316,"external_id":null},{"id":10848139003,"name":"Catania University","priority":2317,"external_id":null},{"id":10848140003,"name":"Università degli Studi Roma Tre","priority":2318,"external_id":null},{"id":10848141003,"name":"Université Charles-de-Gaulle Lille 3","priority":2319,"external_id":null},{"id":10848142003,"name":"Université de Caen Basse-Normandie","priority":2320,"external_id":null},{"id":10848143003,"name":"Université de Cergy-Pontoise","priority":2321,"external_id":null},{"id":10848144003,"name":"Université de Poitiers","priority":2322,"external_id":null},{"id":10848145003,"name":"Université Jean Moulin Lyon 3","priority":2323,"external_id":null},{"id":10848146003,"name":"Université Lille 2 Droit et Santé","priority":2324,"external_id":null},{"id":10848147003,"name":"Université Paris Ouest Nanterre La Défense","priority":2325,"external_id":null},{"id":10848148003,"name":"Université Paul-Valéry Montpellier 3","priority":2326,"external_id":null},{"id":10848149003,"name":"Université Pierre Mendès France - Grenoble 2","priority":2327,"external_id":null},{"id":10848150003,"name":"Université Stendhal Grenoble 3","priority":2328,"external_id":null},{"id":10848151003,"name":"Université Toulouse II, Le Mirail","priority":2329,"external_id":null},{"id":10848152003,"name":"Universiti Teknologi MARA - UiTM","priority":2330,"external_id":null},{"id":10848153003,"name":"University of Baghdad","priority":2331,"external_id":null},{"id":10848154003,"name":"University of Bahrain","priority":2332,"external_id":null},{"id":10848155003,"name":"University of Bari","priority":2333,"external_id":null},{"id":10848156003,"name":"University of Belgrade","priority":2334,"external_id":null},{"id":10848157003,"name":"University of Brawijaya","priority":2335,"external_id":null},{"id":10848158003,"name":"University of Brescia","priority":2336,"external_id":null},{"id":10848159003,"name":"University of Bucharest","priority":2337,"external_id":null},{"id":10848160003,"name":"University of Calcutta","priority":2338,"external_id":null},{"id":10848161003,"name":"University of Central Lancashire","priority":2339,"external_id":null},{"id":10848162003,"name":"University of Colombo","priority":2340,"external_id":null},{"id":10848163003,"name":"University of Dhaka","priority":2341,"external_id":null},{"id":10848164003,"name":"University of East London","priority":2342,"external_id":null},{"id":10848165003,"name":"University of Engineering & Technology (UET) Lahore","priority":2343,"external_id":null},{"id":10848166003,"name":"University of Greenwich","priority":2344,"external_id":null},{"id":10848167003,"name":"University of Jordan","priority":2345,"external_id":null},{"id":10848168003,"name":"University of Karachi","priority":2346,"external_id":null},{"id":10848169003,"name":"University of Lahore","priority":2347,"external_id":null},{"id":10848170003,"name":"University of Latvia","priority":2348,"external_id":null},{"id":10848171003,"name":"University of New England","priority":2349,"external_id":null},{"id":10848172003,"name":"University of Pune","priority":2350,"external_id":null},{"id":10848173003,"name":"University of Santo Tomas","priority":2351,"external_id":null},{"id":10848174003,"name":"University of Southern Queensland","priority":2352,"external_id":null},{"id":10848175003,"name":"University of Wroclaw","priority":2353,"external_id":null},{"id":10848176003,"name":"Verona University","priority":2354,"external_id":null},{"id":10848177003,"name":"Victoria University","priority":2355,"external_id":null},{"id":10848178003,"name":"Vilnius Gediminas Technical University","priority":2356,"external_id":null},{"id":10848179003,"name":"Voronezh State University","priority":2357,"external_id":null},{"id":10848180003,"name":"Vytautas Magnus University","priority":2358,"external_id":null},{"id":10848181003,"name":"West University of Timisoara","priority":2359,"external_id":null},{"id":10848182003,"name":"University of South Alabama","priority":2360,"external_id":null},{"id":10848183003,"name":"University of Arkansas","priority":2361,"external_id":null},{"id":10848184003,"name":"University of California - Berkeley","priority":2362,"external_id":null},{"id":10848185003,"name":"University of Connecticut","priority":2363,"external_id":null},{"id":10848186003,"name":"University of South Florida","priority":2364,"external_id":null},{"id":10848187003,"name":"University of Georgia","priority":2365,"external_id":null},{"id":10848188003,"name":"University of Hawaii - Manoa","priority":2366,"external_id":null},{"id":10848189003,"name":"Iowa State University","priority":2367,"external_id":null},{"id":10848190003,"name":"Murray State University","priority":2368,"external_id":null},{"id":10848191003,"name":"University of Louisville","priority":2369,"external_id":null},{"id":10848192003,"name":"Western Kentucky University","priority":2370,"external_id":null},{"id":10848193003,"name":"Louisiana State University - Baton Rouge","priority":2371,"external_id":null},{"id":10848194003,"name":"University of Maryland - College Park","priority":2372,"external_id":null},{"id":10848195003,"name":"University of Minnesota - Twin Cities","priority":2373,"external_id":null},{"id":10848196003,"name":"University of Montana","priority":2374,"external_id":null},{"id":10848197003,"name":"East Carolina University","priority":2375,"external_id":null},{"id":10848198003,"name":"University of North Carolina - Chapel Hill","priority":2376,"external_id":null},{"id":10848199003,"name":"Wake Forest University","priority":2377,"external_id":null},{"id":10848200003,"name":"University of Nebraska - Lincoln","priority":2378,"external_id":null},{"id":10848201003,"name":"New Mexico State University","priority":2379,"external_id":null},{"id":10848202003,"name":"Ohio State University - Columbus","priority":2380,"external_id":null},{"id":10848203003,"name":"University of Oklahoma","priority":2381,"external_id":null},{"id":10848204003,"name":"Pennsylvania State University - University Park","priority":2382,"external_id":null},{"id":10848205003,"name":"University of Pittsburgh","priority":2383,"external_id":null},{"id":10848206003,"name":"University of Tennessee - Chattanooga","priority":2384,"external_id":null},{"id":10848207003,"name":"Vanderbilt University","priority":2385,"external_id":null},{"id":10848208003,"name":"Rice University","priority":2386,"external_id":null},{"id":10848209003,"name":"University of Utah","priority":2387,"external_id":null},{"id":10848210003,"name":"University of Richmond","priority":2388,"external_id":null},{"id":10848211003,"name":"University of Arkansas - Pine Bluff","priority":2389,"external_id":null},{"id":10848212003,"name":"University of Central Florida","priority":2390,"external_id":null},{"id":10848213003,"name":"Florida Atlantic University","priority":2391,"external_id":null},{"id":10848214003,"name":"Hampton University","priority":2392,"external_id":null},{"id":10848215003,"name":"Liberty University","priority":2393,"external_id":null},{"id":10848216003,"name":"Mercer University","priority":2394,"external_id":null},{"id":10848217003,"name":"Middle Tennessee State University","priority":2395,"external_id":null},{"id":10848218003,"name":"University of Nevada - Las Vegas","priority":2396,"external_id":null},{"id":10848219003,"name":"South Carolina State University","priority":2397,"external_id":null},{"id":10848220003,"name":"University of Tennessee - Martin","priority":2398,"external_id":null},{"id":10848221003,"name":"Weber State University","priority":2399,"external_id":null},{"id":10848222003,"name":"Youngstown State University","priority":2400,"external_id":null},{"id":10848223003,"name":"University of the Incarnate Word","priority":2401,"external_id":null},{"id":10848224003,"name":"University of Washington","priority":2402,"external_id":null},{"id":10848225003,"name":"University of Louisiana - Lafayette","priority":2403,"external_id":null},{"id":10848226003,"name":"Coastal Carolina University","priority":2404,"external_id":null},{"id":10848227003,"name":"Utah State University","priority":2405,"external_id":null},{"id":10848228003,"name":"University of Alabama","priority":2406,"external_id":null},{"id":10848229003,"name":"University of Illinois - Urbana-Champaign","priority":2407,"external_id":null},{"id":10848230003,"name":"United States Air Force Academy","priority":2408,"external_id":null},{"id":10848231003,"name":"University of Akron","priority":2409,"external_id":null},{"id":10848232003,"name":"University of Central Arkansas","priority":2410,"external_id":null},{"id":10848233003,"name":"University of Kansas","priority":2411,"external_id":null},{"id":10848234003,"name":"University of Northern Colorado","priority":2412,"external_id":null},{"id":10848235003,"name":"University of Northern Iowa","priority":2413,"external_id":null},{"id":10848236003,"name":"University of South Carolina","priority":2414,"external_id":null},{"id":10848237003,"name":"Tennessee Technological University","priority":2415,"external_id":null},{"id":10848238003,"name":"University of Texas - El Paso","priority":2416,"external_id":null},{"id":10848239003,"name":"Texas Tech University","priority":2417,"external_id":null},{"id":10848240003,"name":"Tulane University","priority":2418,"external_id":null},{"id":10848241003,"name":"Virginia Military Institute","priority":2419,"external_id":null},{"id":10848242003,"name":"Western Michigan University","priority":2420,"external_id":null},{"id":10848243003,"name":"Wilfrid Laurier University","priority":2421,"external_id":null},{"id":10848244003,"name":"University of San Diego","priority":2422,"external_id":null},{"id":10848245003,"name":"University of California - San Diego","priority":2423,"external_id":null},{"id":10848246003,"name":"Brooks Institute of Photography","priority":2424,"external_id":null},{"id":10848247003,"name":"Acupuncture and Integrative Medicine College - Berkeley","priority":2425,"external_id":null},{"id":10848248003,"name":"Southern Alberta Institute of Technology","priority":2426,"external_id":null},{"id":10848249003,"name":"Susquehanna University","priority":2427,"external_id":null},{"id":10848250003,"name":"University of Texas - Dallas","priority":2428,"external_id":null},{"id":10848251003,"name":"Thunderbird School of Global Management","priority":2429,"external_id":null},{"id":10848252003,"name":"Presidio Graduate School","priority":2430,"external_id":null},{"id":10848253003,"name":"École supérieure de commerce de Dijon","priority":2431,"external_id":null},{"id":10848254003,"name":"University of California - San Francisco","priority":2432,"external_id":null},{"id":10848255003,"name":"Hack Reactor","priority":2433,"external_id":null},{"id":10848256003,"name":"St. Mary''s College of California","priority":2434,"external_id":null},{"id":10848257003,"name":"New England Law","priority":2435,"external_id":null},{"id":10848258003,"name":"University of California, Merced","priority":2436,"external_id":null},{"id":10848259003,"name":"University of California, Hastings College of the Law","priority":2437,"external_id":null},{"id":10848260003,"name":"V.N. Karazin Kharkiv National University","priority":2438,"external_id":null},{"id":10848261003,"name":"SIM University (UniSIM)","priority":2439,"external_id":null},{"id":10848262003,"name":"Singapore Management University (SMU)","priority":2440,"external_id":null},{"id":10848263003,"name":"Singapore University of Technology and Design (SUTD)","priority":2441,"external_id":null},{"id":10848264003,"name":"Singapore Institute of Technology (SIT)","priority":2442,"external_id":null},{"id":10848265003,"name":"Nanyang Polytechnic (NYP)","priority":2443,"external_id":null},{"id":10848266003,"name":"Ngee Ann Polytechnic (NP)","priority":2444,"external_id":null},{"id":10848267003,"name":"Republic Polytechnic (RP)","priority":2445,"external_id":null},{"id":10848268003,"name":"Singapore Polytechnic (SP)","priority":2446,"external_id":null},{"id":10848269003,"name":"Temasek Polytechnic (TP)","priority":2447,"external_id":null},{"id":10848270003,"name":"INSEAD","priority":2448,"external_id":null},{"id":10848271003,"name":"Fundação Getúlio Vargas","priority":2449,"external_id":null},{"id":10848272003,"name":"Acharya Nagarjuna University","priority":2450,"external_id":null},{"id":10848273003,"name":"University of California - Santa Barbara","priority":2451,"external_id":null},{"id":10848274003,"name":"University of California - Irvine","priority":2452,"external_id":null},{"id":10848275003,"name":"California State University - Long Beach","priority":2453,"external_id":null},{"id":10848276003,"name":"Robert Morris University Illinois","priority":2454,"external_id":null},{"id":10848277003,"name":"Harold Washington College - City Colleges of Chicago","priority":2455,"external_id":null},{"id":10848278003,"name":"Harry S Truman College - City Colleges of Chicago","priority":2456,"external_id":null},{"id":10848279003,"name":"Kennedy-King College - City Colleges of Chicago","priority":2457,"external_id":null},{"id":10848280003,"name":"Malcolm X College - City Colleges of Chicago","priority":2458,"external_id":null},{"id":10848281003,"name":"Olive-Harvey College - City Colleges of Chicago","priority":2459,"external_id":null},{"id":10848282003,"name":"Richard J Daley College - City Colleges of Chicago","priority":2460,"external_id":null},{"id":10848283003,"name":"Wilbur Wright College - City Colleges of Chicago","priority":2461,"external_id":null},{"id":10848284003,"name":"Abertay University","priority":2462,"external_id":null},{"id":10848285003,"name":"Pontifícia Universidade Católica de Minas Gerais","priority":2463,"external_id":null},{"id":10848286003,"name":"Other","priority":2464,"external_id":null}]},"emitted_at":1660156526565} +{"stream":"custom_fields","data":{"id":4680899003,"name":"Degree","active":true,"field_type":"candidate","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"degree","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10848287003,"name":"High School","priority":0,"external_id":null},{"id":10848288003,"name":"Associate's Degree","priority":1,"external_id":null},{"id":10848289003,"name":"Bachelor's Degree","priority":2,"external_id":null},{"id":10848290003,"name":"Master's Degree","priority":3,"external_id":null},{"id":10848291003,"name":"Master of Business Administration (M.B.A.)","priority":4,"external_id":null},{"id":10848292003,"name":"Juris Doctor (J.D.)","priority":5,"external_id":null},{"id":10848293003,"name":"Doctor of Medicine (M.D.)","priority":6,"external_id":null},{"id":10848294003,"name":"Doctor of Philosophy (Ph.D.)","priority":7,"external_id":null},{"id":10848295003,"name":"Engineer's Degree","priority":8,"external_id":null},{"id":10848296003,"name":"Other","priority":9,"external_id":null}]},"emitted_at":1660156526606} +{"stream":"custom_fields","data":{"id":4680900003,"name":"Discipline","active":true,"field_type":"candidate","priority":2,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"discipline","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10848297003,"name":"Accounting","priority":0,"external_id":null},{"id":10848298003,"name":"African Studies","priority":1,"external_id":null},{"id":10848299003,"name":"Agriculture","priority":2,"external_id":null},{"id":10848300003,"name":"Anthropology","priority":3,"external_id":null},{"id":10848301003,"name":"Applied Health Services","priority":4,"external_id":null},{"id":10848302003,"name":"Architecture","priority":5,"external_id":null},{"id":10848303003,"name":"Art","priority":6,"external_id":null},{"id":10848304003,"name":"Asian Studies","priority":7,"external_id":null},{"id":10848305003,"name":"Biology","priority":8,"external_id":null},{"id":10848306003,"name":"Business","priority":9,"external_id":null},{"id":10848307003,"name":"Business Administration","priority":10,"external_id":null},{"id":10848308003,"name":"Chemistry","priority":11,"external_id":null},{"id":10848309003,"name":"Classical Languages","priority":12,"external_id":null},{"id":10848310003,"name":"Communications & Film","priority":13,"external_id":null},{"id":10848311003,"name":"Computer Science","priority":14,"external_id":null},{"id":10848312003,"name":"Dentistry","priority":15,"external_id":null},{"id":10848313003,"name":"Developing Nations","priority":16,"external_id":null},{"id":10848314003,"name":"Discipline Unknown","priority":17,"external_id":null},{"id":10848315003,"name":"Earth Sciences","priority":18,"external_id":null},{"id":10848316003,"name":"Economics","priority":19,"external_id":null},{"id":10848317003,"name":"Education","priority":20,"external_id":null},{"id":10848318003,"name":"Electronics","priority":21,"external_id":null},{"id":10848319003,"name":"Engineering","priority":22,"external_id":null},{"id":10848320003,"name":"English Studies","priority":23,"external_id":null},{"id":10848321003,"name":"Environmental Studies","priority":24,"external_id":null},{"id":10848322003,"name":"European Studies","priority":25,"external_id":null},{"id":10848323003,"name":"Fashion","priority":26,"external_id":null},{"id":10848324003,"name":"Finance","priority":27,"external_id":null},{"id":10848325003,"name":"Fine Arts","priority":28,"external_id":null},{"id":10848326003,"name":"General Studies","priority":29,"external_id":null},{"id":10848327003,"name":"Health Services","priority":30,"external_id":null},{"id":10848328003,"name":"History","priority":31,"external_id":null},{"id":10848329003,"name":"Human Resources Management","priority":32,"external_id":null},{"id":10848330003,"name":"Humanities","priority":33,"external_id":null},{"id":10848331003,"name":"Industrial Arts & Carpentry","priority":34,"external_id":null},{"id":10848332003,"name":"Information Systems","priority":35,"external_id":null},{"id":10848333003,"name":"International Relations","priority":36,"external_id":null},{"id":10848334003,"name":"Journalism","priority":37,"external_id":null},{"id":10848335003,"name":"Languages","priority":38,"external_id":null},{"id":10848336003,"name":"Latin American Studies","priority":39,"external_id":null},{"id":10848337003,"name":"Law","priority":40,"external_id":null},{"id":10848338003,"name":"Linguistics","priority":41,"external_id":null},{"id":10848339003,"name":"Manufacturing & Mechanics","priority":42,"external_id":null},{"id":10848340003,"name":"Mathematics","priority":43,"external_id":null},{"id":10848341003,"name":"Medicine","priority":44,"external_id":null},{"id":10848342003,"name":"Middle Eastern Studies","priority":45,"external_id":null},{"id":10848343003,"name":"Naval Science","priority":46,"external_id":null},{"id":10848344003,"name":"North American Studies","priority":47,"external_id":null},{"id":10848345003,"name":"Nuclear Technics","priority":48,"external_id":null},{"id":10848346003,"name":"Operations Research & Strategy","priority":49,"external_id":null},{"id":10848347003,"name":"Organizational Theory","priority":50,"external_id":null},{"id":10848348003,"name":"Philosophy","priority":51,"external_id":null},{"id":10848349003,"name":"Physical Education","priority":52,"external_id":null},{"id":10848350003,"name":"Physical Sciences","priority":53,"external_id":null},{"id":10848351003,"name":"Physics","priority":54,"external_id":null},{"id":10848352003,"name":"Political Science","priority":55,"external_id":null},{"id":10848353003,"name":"Psychology","priority":56,"external_id":null},{"id":10848354003,"name":"Public Policy","priority":57,"external_id":null},{"id":10848355003,"name":"Public Service","priority":58,"external_id":null},{"id":10848356003,"name":"Religious Studies","priority":59,"external_id":null},{"id":10848357003,"name":"Russian & Soviet Studies","priority":60,"external_id":null},{"id":10848358003,"name":"Scandinavian Studies","priority":61,"external_id":null},{"id":10848359003,"name":"Science","priority":62,"external_id":null},{"id":10848360003,"name":"Slavic Studies","priority":63,"external_id":null},{"id":10848361003,"name":"Social Science","priority":64,"external_id":null},{"id":10848362003,"name":"Social Sciences","priority":65,"external_id":null},{"id":10848363003,"name":"Sociology","priority":66,"external_id":null},{"id":10848364003,"name":"Speech","priority":67,"external_id":null},{"id":10848365003,"name":"Statistics & Decision Theory","priority":68,"external_id":null},{"id":10848366003,"name":"Urban Studies","priority":69,"external_id":null},{"id":10848367003,"name":"Veterinary Medicine","priority":70,"external_id":null},{"id":10848368003,"name":"Other","priority":71,"external_id":null}]},"emitted_at":1660156526607} +{"stream":"custom_fields","data":{"id":4680901003,"name":"Employment Type","active":true,"field_type":"job","priority":0,"value_type":"single_select","private":false,"required":false,"require_approval":true,"trigger_new_version":false,"name_key":"employment_type","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845796003,"name":"Full-time","priority":0,"external_id":null},{"id":10845797003,"name":"Part-time","priority":1,"external_id":null},{"id":10845798003,"name":"Intern","priority":2,"external_id":null},{"id":10845799003,"name":"Contract","priority":3,"external_id":null},{"id":10845800003,"name":"Temporary","priority":4,"external_id":null}]},"emitted_at":1660156526608} +{"stream":"custom_fields","data":{"id":4680902003,"name":"Start Date","active":true,"field_type":"offer","priority":0,"value_type":"date","private":true,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"start_date","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526608} +{"stream":"custom_fields","data":{"id":4680903003,"name":"Employment Type","active":true,"field_type":"offer","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":true,"name_key":"employment_type","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845801003,"name":"Full-time","priority":0,"external_id":null},{"id":10845802003,"name":"Part-time","priority":1,"external_id":null},{"id":10845803003,"name":"Intern","priority":2,"external_id":null},{"id":10845804003,"name":"Contract","priority":3,"external_id":null},{"id":10845805003,"name":"Temporary","priority":4,"external_id":null}]},"emitted_at":1660156526608} +{"stream":"custom_fields","data":{"id":4680904003,"name":"Offer Documents","active":true,"field_type":"offer","priority":2,"value_type":"short_text","private":true,"required":false,"require_approval":false,"trigger_new_version":true,"name_key":"offer_documents","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526608} +{"stream":"custom_fields","data":{"id":4680905003,"name":"Relationship","active":true,"field_type":"referral_question","priority":0,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"relationship","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845806003,"name":"Coworker","priority":0,"external_id":null},{"id":10845807003,"name":"School","priority":1,"external_id":null},{"id":10845808003,"name":"Manager","priority":2,"external_id":null},{"id":10845809003,"name":"Reported","priority":3,"external_id":null},{"id":10845810003,"name":"Friend","priority":4,"external_id":null},{"id":10845811003,"name":"Do not know","priority":5,"external_id":null}]},"emitted_at":1660156526608} +{"stream":"custom_fields","data":{"id":4680906003,"name":"Work History","active":true,"field_type":"referral_question","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"work_history","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845812003,"name":"0-1","priority":0,"external_id":null},{"id":10845813003,"name":"2-5","priority":1,"external_id":null},{"id":10845814003,"name":"5+","priority":2,"external_id":null}]},"emitted_at":1660156526609} +{"stream":"custom_fields","data":{"id":4680907003,"name":"Rating","active":true,"field_type":"referral_question","priority":2,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"rating","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845815003,"name":"Superstar","priority":0,"external_id":null},{"id":10845816003,"name":"Top 5%","priority":1,"external_id":null},{"id":10845817003,"name":"Top 10%","priority":2,"external_id":null},{"id":10845818003,"name":"Top 25%","priority":3,"external_id":null},{"id":10845819003,"name":"Top 50%","priority":4,"external_id":null}]},"emitted_at":1660156526609} +{"stream":"custom_fields","data":{"id":4680908003,"name":"When we reach out","active":true,"field_type":"referral_question","priority":3,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"when_we_reach_out","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845820003,"name":"You may mention me","priority":0,"external_id":null},{"id":10845821003,"name":"I wish to remain anonymous","priority":1,"external_id":null}]},"emitted_at":1660156526609} +{"stream":"custom_fields","data":{"id":4680909003,"name":"They know they're being referred","active":true,"field_type":"referral_question","priority":4,"value_type":"yes_no","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"they_know_they_re_being_referred","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526609} +{"stream":"custom_fields","data":{"id":4680910003,"name":"Referral Notes","active":true,"field_type":"referral_question","priority":5,"value_type":"long_text","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"referral_notes","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526609} +{"stream":"custom_fields","data":{"id":7431124003,"name":"Test User","active":true,"field_type":"agency_question","priority":0,"value_type":"yes_no","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"test_user","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526610} +{"stream":"custom_fields","data":{"id":7431125003,"name":"Test User","active":true,"field_type":"agency_question","priority":1,"value_type":"short_text","private":false,"required":true,"require_approval":false,"trigger_new_version":false,"name_key":"test_user_agency_question_1633884465.559642","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526610} +{"stream":"custom_fields","data":{"id":7431126003,"name":"Test User","active":true,"field_type":"referral_question","priority":6,"value_type":"yes_no","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"test_user","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526610} +{"stream":"demographics_question_sets","data":{"title":"Test Question Set 1","id":4000197003,"description":"

Test Question Set 1 description

","active":true},"emitted_at":1660156526996} +{"stream":"demographics_question_sets","data":{"title":"Test Question Set 2","id":4000198003,"description":"

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

","active":true},"emitted_at":1660156526998} +{"stream":"demographics_question_sets","data":{"title":"U.S. Standard Demographic Questions","id":4002702003,"description":"We invite applicants to share their demographic background. If you choose to complete this survey, your responses may be used to identify areas of improvement in our hiring process.","active":true},"emitted_at":1660156526998} +{"stream":"demographics_questions","data":{"translations":[{"name":"q1","language":"en"}],"required":false,"name":"q1","id":4000714003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527371} +{"stream":"demographics_questions","data":{"translations":[{"name":"q2","language":"en"}],"required":false,"name":"q2","id":4000715003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527375} +{"stream":"demographics_questions","data":{"translations":[{"name":"question1","language":"en"}],"required":false,"name":"question1","id":4000716003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527375} +{"stream":"demographics_questions","data":{"translations":[{"name":"question2","language":"en"}],"required":true,"name":"question2","id":4000717003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} +{"stream":"demographics_questions","data":{"translations":[{"name":"Are you a veteran or active member of the United States Armed Forces? (select one)","language":"en"}],"required":false,"name":"Are you a veteran or active member of the United States Armed Forces? (select one)","id":4015594003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} +{"stream":"demographics_questions","data":{"translations":[{"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning? (select one)","language":"en"}],"required":false,"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning? (select one)","id":4015596003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} +{"stream":"demographics_questions","data":{"translations":[{"name":"Do you identify as transgender? (select one)","language":"en"}],"required":false,"name":"Do you identify as transgender? (select one)","id":4015598003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} +{"stream":"demographics_questions","data":{"translations":[{"name":"How would you describe your sexual orientation? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your sexual orientation? (mark all that apply)","id":4015599003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527389} +{"stream":"demographics_questions","data":{"translations":[{"name":"How would you describe your racial/ethnic background? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your racial/ethnic background? (mark all that apply)","id":4015601003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527389} +{"stream":"demographics_questions","data":{"translations":[{"name":"How would you describe your gender identity? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your gender identity? (mark all that apply)","id":4015603003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527390} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"a1","language":"en"}],"name":"a1","id":4004258003,"free_form":false,"demographic_question_id":4000714003,"active":true},"emitted_at":1660156527772} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"a2","language":"en"}],"name":"a2","id":4004259003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156527775} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"a3","language":"en"}],"name":"a3","id":4004260003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156527775} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"s1","language":"en"}],"name":"s1","id":4004261003,"free_form":true,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156527775} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004262003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156527775} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer-self1","language":"en"}],"name":"answer-self1","id":4004263003,"free_form":true,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156527775} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004264003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156527775} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004265003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156527776} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4004266003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156527776} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004267003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156527776} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093930003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527776} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093931003,"free_form":true,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527776} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"No, I am not a veteran or active member","language":"en"}],"name":"No, I am not a veteran or active member","id":4093932003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527776} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Yes, I am a veteran or active member","language":"en"}],"name":"Yes, I am a veteran or active member","id":4093934003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527777} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093937003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093939003,"free_form":true,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093940003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093941003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093944003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527777} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093946003,"free_form":true,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527778} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093948003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527778} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093950003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527779} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093953003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527779} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093955003,"free_form":true,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527779} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Queer","language":"en"}],"name":"Queer","id":4093956003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527781} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Lesbian","language":"en"}],"name":"Lesbian","id":4093957003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527781} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Heterosexual","language":"en"}],"name":"Heterosexual","id":4093959003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527782} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Gay","language":"en"}],"name":"Gay","id":4093961003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527783} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Bisexual and/or pansexual","language":"en"}],"name":"Bisexual and/or pansexual","id":4093963003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527784} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Asexual","language":"en"}],"name":"Asexual","id":4093965003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527784} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093971003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527785} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093973003,"free_form":true,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527785} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"White or European","language":"en"}],"name":"White or European","id":4093975003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527785} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Southeast Asian","language":"en"}],"name":"Southeast Asian","id":4093976003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527786} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"South Asian","language":"en"}],"name":"South Asian","id":4093977003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527786} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Native Hawaiian or Pacific Islander","language":"en"}],"name":"Native Hawaiian or Pacific Islander","id":4093979003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527786} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Middle Eastern or North African","language":"en"}],"name":"Middle Eastern or North African","id":4093981003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527787} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Indigenous, American Indian or Alaska Native","language":"en"}],"name":"Indigenous, American Indian or Alaska Native","id":4093983003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527787} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Hispanic, Latinx or of Spanish Origin","language":"en"}],"name":"Hispanic, Latinx or of Spanish Origin","id":4093985003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527788} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"East Asian","language":"en"}],"name":"East Asian","id":4093986003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527789} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Black or of African descent","language":"en"}],"name":"Black or of African descent","id":4093988003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527789} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093989003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527789} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093990003,"free_form":true,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Woman","language":"en"}],"name":"Woman","id":4093991003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Non-binary","language":"en"}],"name":"Non-binary","id":4093993003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} +{"stream":"demographics_answer_options","data":{"translations":[{"name":"Man","language":"en"}],"name":"Man","id":4093995003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} +{"stream":"demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.248Z","id":9308815003,"free_form_text":null,"demographic_question_id":4000716003,"demographic_answer_option_id":4004262003,"created_at":"2021-11-03T19:56:07.248Z","application_id":47459993003},"emitted_at":1660156528190} +{"stream":"demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.252Z","id":9308816003,"free_form_text":"custom answer","demographic_question_id":4000716003,"demographic_answer_option_id":4004263003,"created_at":"2021-11-03T19:56:07.252Z","application_id":47459993003},"emitted_at":1660156528193} +{"stream":"demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.259Z","id":9308817003,"free_form_text":null,"demographic_question_id":4000717003,"demographic_answer_option_id":4004266003,"created_at":"2021-11-03T19:56:07.259Z","application_id":47459993003},"emitted_at":1660156528193} +{"stream":"applications_demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.248Z","id":9308815003,"free_form_text":null,"demographic_question_id":4000716003,"demographic_answer_option_id":4004262003,"created_at":"2021-11-03T19:56:07.248Z","application_id":47459993003},"emitted_at":1660156529467} +{"stream":"applications_demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.252Z","id":9308816003,"free_form_text":"custom answer","demographic_question_id":4000716003,"demographic_answer_option_id":4004263003,"created_at":"2021-11-03T19:56:07.252Z","application_id":47459993003},"emitted_at":1660156529468} +{"stream":"applications_demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.259Z","id":9308817003,"free_form_text":null,"demographic_question_id":4000717003,"demographic_answer_option_id":4004266003,"created_at":"2021-11-03T19:56:07.259Z","application_id":47459993003},"emitted_at":1660156529469} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"q1","language":"en"}],"required":false,"name":"q1","id":4000714003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531201} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"q2","language":"en"}],"required":false,"name":"q2","id":4000715003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531205} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"question1","language":"en"}],"required":false,"name":"question1","id":4000716003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531420} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"question2","language":"en"}],"required":true,"name":"question2","id":4000717003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531422} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"Are you a veteran or active member of the United States Armed Forces? (select one)","language":"en"}],"required":false,"name":"Are you a veteran or active member of the United States Armed Forces? (select one)","id":4015594003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531636} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning? (select one)","language":"en"}],"required":false,"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning? (select one)","id":4015596003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531636} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"Do you identify as transgender? (select one)","language":"en"}],"required":false,"name":"Do you identify as transgender? (select one)","id":4015598003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531636} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"How would you describe your sexual orientation? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your sexual orientation? (mark all that apply)","id":4015599003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531637} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"How would you describe your racial/ethnic background? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your racial/ethnic background? (mark all that apply)","id":4015601003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531637} +{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"How would you describe your gender identity? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your gender identity? (mark all that apply)","id":4015603003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531637} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"a1","language":"en"}],"name":"a1","id":4004258003,"free_form":false,"demographic_question_id":4000714003,"active":true},"emitted_at":1660156532379} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"a2","language":"en"}],"name":"a2","id":4004259003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156532578} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"a3","language":"en"}],"name":"a3","id":4004260003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156532579} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"s1","language":"en"}],"name":"s1","id":4004261003,"free_form":true,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156532579} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004262003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156532783} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer-self1","language":"en"}],"name":"answer-self1","id":4004263003,"free_form":true,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156532783} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004264003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156532783} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004265003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156532977} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4004266003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156532977} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004267003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156532977} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093930003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533171} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093931003,"free_form":true,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533171} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"No, I am not a veteran or active member","language":"en"}],"name":"No, I am not a veteran or active member","id":4093932003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533172} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Yes, I am a veteran or active member","language":"en"}],"name":"Yes, I am a veteran or active member","id":4093934003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533172} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093937003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533376} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093939003,"free_form":true,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533378} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093940003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533379} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093941003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533379} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093944003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533581} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093946003,"free_form":true,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533582} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093948003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533582} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093950003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533582} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093953003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533793} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093955003,"free_form":true,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533793} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Queer","language":"en"}],"name":"Queer","id":4093956003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533793} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Lesbian","language":"en"}],"name":"Lesbian","id":4093957003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Heterosexual","language":"en"}],"name":"Heterosexual","id":4093959003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Gay","language":"en"}],"name":"Gay","id":4093961003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Bisexual and/or pansexual","language":"en"}],"name":"Bisexual and/or pansexual","id":4093963003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Asexual","language":"en"}],"name":"Asexual","id":4093965003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533795} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093971003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534000} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093973003,"free_form":true,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534000} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"White or European","language":"en"}],"name":"White or European","id":4093975003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534000} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Southeast Asian","language":"en"}],"name":"Southeast Asian","id":4093976003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534001} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"South Asian","language":"en"}],"name":"South Asian","id":4093977003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534001} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Native Hawaiian or Pacific Islander","language":"en"}],"name":"Native Hawaiian or Pacific Islander","id":4093979003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534002} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Middle Eastern or North African","language":"en"}],"name":"Middle Eastern or North African","id":4093981003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534006} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Indigenous, American Indian or Alaska Native","language":"en"}],"name":"Indigenous, American Indian or Alaska Native","id":4093983003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534007} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Hispanic, Latinx or of Spanish Origin","language":"en"}],"name":"Hispanic, Latinx or of Spanish Origin","id":4093985003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534007} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"East Asian","language":"en"}],"name":"East Asian","id":4093986003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534008} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Black or of African descent","language":"en"}],"name":"Black or of African descent","id":4093988003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534008} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093989003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534199} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093990003,"free_form":true,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534199} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Woman","language":"en"}],"name":"Woman","id":4093991003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534200} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Non-binary","language":"en"}],"name":"Non-binary","id":4093993003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534200} +{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Man","language":"en"}],"name":"Man","id":4093995003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534201} +{"stream":"interviews","data":{"id":40387397003,"application_id":44937562003,"external_event_id":"123456789","start":{"date_time":"2021-12-12T13:15:00.000Z"},"end":{"date_time":"2021-12-12T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:21:44.107Z","updated_at":"2021-12-12T15:15:02.894Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156535168} +{"stream":"interviews","data":{"id":40387426003,"application_id":44937562003,"external_event_id":"12345678","start":{"date_time":"2021-12-13T13:15:00.000Z"},"end":{"date_time":"2021-12-13T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:04.561Z","updated_at":"2021-12-13T15:15:13.252Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156535172} +{"stream":"interviews","data":{"id":40387431003,"application_id":44937562003,"external_event_id":"1234567","start":{"date_time":"2021-12-14T13:15:00.000Z"},"end":{"date_time":"2021-12-14T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:13.681Z","updated_at":"2021-12-14T15:15:12.118Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156535173} +{"stream":"applications_interviews","data":{"id":40387397003,"application_id":44937562003,"external_event_id":"123456789","start":{"date_time":"2021-12-12T13:15:00.000Z"},"end":{"date_time":"2021-12-12T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:21:44.107Z","updated_at":"2021-12-12T15:15:02.894Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156536478} +{"stream":"applications_interviews","data":{"id":40387426003,"application_id":44937562003,"external_event_id":"12345678","start":{"date_time":"2021-12-13T13:15:00.000Z"},"end":{"date_time":"2021-12-13T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:04.561Z","updated_at":"2021-12-13T15:15:13.252Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156536483} +{"stream":"applications_interviews","data":{"id":40387431003,"application_id":44937562003,"external_event_id":"1234567","start":{"date_time":"2021-12-14T13:15:00.000Z"},"end":{"date_time":"2021-12-14T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:13.681Z","updated_at":"2021-12-14T15:15:12.118Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156536483} +{"stream":"sources","data":{"id":4000000003,"name":"Recurse","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537034} +{"stream":"sources","data":{"id":4000001003,"name":"cliquify","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537037} +{"stream":"sources","data":{"id":4000002003,"name":"ContactOut","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537038} +{"stream":"sources","data":{"id":4000003003,"name":"Crosschq","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537038} +{"stream":"sources","data":{"id":4000004003,"name":"Talentpair","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537038} +{"stream":"sources","data":{"id":4000005003,"name":"Sompani Talent Pools","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000006003,"name":"ScoutFor","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000007003,"name":"Gem","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000008003,"name":"Findem","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000009003,"name":"goldi staging","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000010003,"name":"MoBerries","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000011003,"name":"Onramp","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} +{"stream":"sources","data":{"id":4000012003,"name":"Knowledge Officer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000013003,"name":"Sourceress","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000014003,"name":"Resume Library","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000015003,"name":"Command E","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000016003,"name":"Attract","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000017003,"name":"WePow","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000018003,"name":"Planted","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} +{"stream":"sources","data":{"id":4000019003,"name":"Birch Local","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000020003,"name":"Birch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000021003,"name":"Consider","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000022003,"name":"Eightfold","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000023003,"name":"Google (Job Search)","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000024003,"name":"Hundred5","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000025003,"name":"Work4 Labs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} +{"stream":"sources","data":{"id":4000026003,"name":"Nudj","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000027003,"name":"Handshake","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000028003,"name":"goldi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000029003,"name":"Honeypot.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000030003,"name":"Joonko","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000031003,"name":"Untapped","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000032003,"name":"Bubblesort","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000033003,"name":"Fetcher","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} +{"stream":"sources","data":{"id":4000034003,"name":"WorksHub","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000035003,"name":"CareerBuilder Quick Apply","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000036003,"name":"BountyJobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000037003,"name":"SmartDreamers","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000038003,"name":"Gloat","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000039003,"name":"Selected","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000040003,"name":"SeekOut","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} +{"stream":"sources","data":{"id":4000041003,"name":"Jobmailer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} +{"stream":"sources","data":{"id":4000042003,"name":"MindMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} +{"stream":"sources","data":{"id":4000043003,"name":"Hackajob","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} +{"stream":"sources","data":{"id":4000044003,"name":"Snap.hr","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} +{"stream":"sources","data":{"id":4000045003,"name":"JamieAI","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} +{"stream":"sources","data":{"id":4000046003,"name":"Visage","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} +{"stream":"sources","data":{"id":4000047003,"name":"XING ReferralManager","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} +{"stream":"sources","data":{"id":4000048003,"name":"WorkShape.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} +{"stream":"sources","data":{"id":4000049003,"name":"Workey","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} +{"stream":"sources","data":{"id":4000050003,"name":"Uncommon","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} +{"stream":"sources","data":{"id":4000051003,"name":"Talentful","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537046} +{"stream":"sources","data":{"id":4000052003,"name":"TalentBin® by Monster","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537046} +{"stream":"sources","data":{"id":4000053003,"name":"Riviera Partners","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537047} +{"stream":"sources","data":{"id":4000054003,"name":"SingleSprout","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537047} +{"stream":"sources","data":{"id":4000055003,"name":"ScoutSavvy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537047} +{"stream":"sources","data":{"id":4000056003,"name":"RippleMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537048} +{"stream":"sources","data":{"id":4000057003,"name":"Project: “Odin”","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537048} +{"stream":"sources","data":{"id":4000058003,"name":"PowerToFly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537048} +{"stream":"sources","data":{"id":4000059003,"name":"OneWire","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} +{"stream":"sources","data":{"id":4000060003,"name":"OfferZen","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} +{"stream":"sources","data":{"id":4000061003,"name":"Netin","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} +{"stream":"sources","data":{"id":4000062003,"name":"Meritocracy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} +{"stream":"sources","data":{"id":4000063003,"name":"Jopwell","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} +{"stream":"sources","data":{"id":4000064003,"name":"Jobjet","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000065003,"name":"Interviewing.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000066003,"name":"Interseller","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000067003,"name":"HRMARKET","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000068003,"name":"hireEZ","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000069003,"name":"HeyJobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000070003,"name":"Hachi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} +{"stream":"sources","data":{"id":4000071003,"name":"getTalent","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000072003,"name":"Functional Works","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000073003,"name":"Firstbird","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000074003,"name":"Crowded","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000075003,"name":"CrediBLL","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000076003,"name":"Celential.ai","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000077003,"name":"AmazingHiring","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} +{"stream":"sources","data":{"id":4000078003,"name":"Teamable","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} +{"stream":"sources","data":{"id":4000079003,"name":"HubSpot Marketing","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} +{"stream":"sources","data":{"id":4000080003,"name":"VolkScience","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} +{"stream":"sources","data":{"id":4000081003,"name":"LeapMind","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} +{"stream":"sources","data":{"id":4000082003,"name":"Woo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} +{"stream":"sources","data":{"id":4000083003,"name":"ReferralMob","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537053} +{"stream":"sources","data":{"id":4000084003,"name":"Maildrop","type":{"id":4000004003,"name":"Other"}},"emitted_at":1660156537053} +{"stream":"sources","data":{"id":4000085003,"name":"Thumbtack Technology","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000086003,"name":"Wendy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000087003,"name":"Stella","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000088003,"name":"Resource","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000089003,"name":"Talentseer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000090003,"name":"Door of Clubs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000091003,"name":"untapt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} +{"stream":"sources","data":{"id":4000092003,"name":"vsource","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} +{"stream":"sources","data":{"id":4000093003,"name":"Ideal","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} +{"stream":"sources","data":{"id":4000094003,"name":"Indeed Prime","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} +{"stream":"sources","data":{"id":4000095003,"name":"Predikt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} +{"stream":"sources","data":{"id":4000096003,"name":"Beamery","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} +{"stream":"sources","data":{"id":4000097003,"name":"RippleHire","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537056} +{"stream":"sources","data":{"id":4000098003,"name":"LinkedIn (Prospecting)","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537056} +{"stream":"sources","data":{"id":4000099003,"name":"LinkedIn (Ad Posting)","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537056} +{"stream":"sources","data":{"id":4000100003,"name":"FirstJob","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537213} +{"stream":"sources","data":{"id":4000101003,"name":"Bsharp","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537214} +{"stream":"sources","data":{"id":4000102003,"name":"Landing.jobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537214} +{"stream":"sources","data":{"id":4000103003,"name":"Vettery","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537214} +{"stream":"sources","data":{"id":4000104003,"name":"RAKUNA Recruit","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537215} +{"stream":"sources","data":{"id":4000105003,"name":"E-SS portal","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537215} +{"stream":"sources","data":{"id":4000106003,"name":"Recsolu","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537215} +{"stream":"sources","data":{"id":4000107003,"name":"SocialReferral [DEV]","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537215} +{"stream":"sources","data":{"id":4000108003,"name":"WayUp","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537216} +{"stream":"sources","data":{"id":4000109003,"name":"SocialReferral","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537216} +{"stream":"sources","data":{"id":4000110003,"name":"HumanPredictions","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537216} +{"stream":"sources","data":{"id":4000111003,"name":"Gogohire","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} +{"stream":"sources","data":{"id":4000112003,"name":"TalentIQ","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} +{"stream":"sources","data":{"id":4000113003,"name":"DoWeKnowThisGuy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} +{"stream":"sources","data":{"id":4000114003,"name":"The Muse","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537217} +{"stream":"sources","data":{"id":4000115003,"name":"HackerRank","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} +{"stream":"sources","data":{"id":4000116003,"name":"Whitetruffle","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} +{"stream":"sources","data":{"id":4000117003,"name":"LinkedIn Limited Listing","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537218} +{"stream":"sources","data":{"id":4000118003,"name":"SpringRole","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} +{"stream":"sources","data":{"id":4000119003,"name":"StrongIntro","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} +{"stream":"sources","data":{"id":4000120003,"name":"CodeFights","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} +{"stream":"sources","data":{"id":4000121003,"name":"Citadel","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} +{"stream":"sources","data":{"id":4000122003,"name":"Savvy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} +{"stream":"sources","data":{"id":4000123003,"name":"SmashFly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} +{"stream":"sources","data":{"id":4000124003,"name":"Stack","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} +{"stream":"sources","data":{"id":4000125003,"name":"Network Monkey","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} +{"stream":"sources","data":{"id":4000126003,"name":"Triplebyte","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} +{"stream":"sources","data":{"id":4000127003,"name":"HireArt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} +{"stream":"sources","data":{"id":4000128003,"name":"Hirecanvas","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} +{"stream":"sources","data":{"id":4000129003,"name":"CloserIQ","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} +{"stream":"sources","data":{"id":4000130003,"name":"Codeity","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} +{"stream":"sources","data":{"id":4000131003,"name":"ZipRecruiter","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537220} +{"stream":"sources","data":{"id":4000132003,"name":"Drafted","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} +{"stream":"sources","data":{"id":4000133003,"name":"ROIKOI","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} +{"stream":"sources","data":{"id":4000134003,"name":"DoWeKnowThisGuy Staging","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000135003,"name":"AngelList","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000136003,"name":"Archively","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000137003,"name":"Hired","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000138003,"name":"EmployeeReferrals.com","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000139003,"name":"Aevy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000140003,"name":"Connectifier","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000141003,"name":"Simppler","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} +{"stream":"sources","data":{"id":4000142003,"name":"Internal Applicant","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000143003,"name":"Clinch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000144003,"name":"SwoopTalent","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000145003,"name":"Pymetrics","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000146003,"name":"YBorder","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000147003,"name":"Hirable","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000148003,"name":"RecruitiFi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} +{"stream":"sources","data":{"id":4000149003,"name":"RolePoint","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} +{"stream":"sources","data":{"id":4000150003,"name":"Entelo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} +{"stream":"sources","data":{"id":4000151003,"name":"Piazza","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} +{"stream":"sources","data":{"id":4000152003,"name":"Setter","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} +{"stream":"sources","data":{"id":4000153003,"name":"Other","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} +{"stream":"sources","data":{"id":4000154003,"name":"Coroflot","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537224} +{"stream":"sources","data":{"id":4000155003,"name":"Startuply","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537224} +{"stream":"sources","data":{"id":4000156003,"name":"Behance","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} +{"stream":"sources","data":{"id":4000157003,"name":"Dribbble","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} +{"stream":"sources","data":{"id":4000158003,"name":"Glassdoor","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} +{"stream":"sources","data":{"id":4000159003,"name":"Careers2.0 by StackOverflow","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} +{"stream":"sources","data":{"id":4000160003,"name":"LinkedIn (Social Media)","type":{"id":4000005003,"name":"Social media"}},"emitted_at":1660156537225} +{"stream":"sources","data":{"id":4000161003,"name":"Referral","type":{"id":4000002003,"name":"Referral"}},"emitted_at":1660156537226} +{"stream":"sources","data":{"id":4000162003,"name":"Beyond.com","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537226} +{"stream":"sources","data":{"id":4000163003,"name":"CareerBuilder","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537227} +{"stream":"sources","data":{"id":4000164003,"name":"Monster","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537227} +{"stream":"sources","data":{"id":4000165003,"name":"CareerBuilder","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537227} +{"stream":"sources","data":{"id":4000166003,"name":"Monster","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} +{"stream":"sources","data":{"id":4000167003,"name":"craigslist","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} +{"stream":"sources","data":{"id":4000168003,"name":"Dice","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} +{"stream":"sources","data":{"id":4000169003,"name":"GitHub Jobs","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} +{"stream":"sources","data":{"id":4000170003,"name":"Dribbble","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537228} +{"stream":"sources","data":{"id":4000171003,"name":"Social media presence","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537229} +{"stream":"sources","data":{"id":4000172003,"name":"Customer newsletter","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537229} +{"stream":"sources","data":{"id":4000173003,"name":"Use BountyJobs","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537229} +{"stream":"sources","data":{"id":4000174003,"name":"Google","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537229} +{"stream":"sources","data":{"id":4000175003,"name":"Indeed","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537229} +{"stream":"sources","data":{"id":4000176003,"name":"Meetups","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537229} +{"stream":"sources","data":{"id":4000177003,"name":"Jobs page on your website","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537230} +{"stream":"sources","data":{"id":4000178003,"name":"Twitter","type":{"id":4000005003,"name":"Social media"}},"emitted_at":1660156537230} +{"stream":"sources","data":{"id":4000179003,"name":"Facebook","type":{"id":4000005003,"name":"Social media"}},"emitted_at":1660156537230} +{"stream":"sources","data":{"id":4000180003,"name":"SimplyHired","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537230} +{"stream":"sources","data":{"id":4000181003,"name":"Indeed","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537230} +{"stream":"sources","data":{"id":4000182003,"name":"Job fairs/Conferences/Trade shows","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537230} +{"stream":"sources","data":{"id":4000183003,"name":"Campus recruiting","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537231} +{"stream":"sources","data":{"id":4000184003,"name":"Uncubed","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} +{"stream":"sources","data":{"id":4000185003,"name":"Ladders","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} +{"stream":"sources","data":{"id":4000186003,"name":"Splash","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} +{"stream":"sources","data":{"id":4000187003,"name":"Recruiter.AI","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} +{"stream":"sources","data":{"id":4000188003,"name":"Underdog.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} +{"stream":"sources","data":{"id":4000189003,"name":"UpScored","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000190003,"name":"LinkMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000191003,"name":"Jobbatical","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000192003,"name":"Upsider","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000195003,"name":"zealpath","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000198003,"name":"AppDirect Connector","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000213003,"name":"Helm","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000226003,"name":"Betts","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000228003,"name":"Circular","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} +{"stream":"sources","data":{"id":4000241003,"name":"Tempo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537233} +{"stream":"sources","data":{"id":4000264003,"name":"Woo Auto-sourcer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537233} +{"stream":"sources","data":{"id":4000344003,"name":"TopFunnel","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} +{"stream":"sources","data":{"id":4000538003,"name":"Greenhouse Test","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537379} +{"stream":"sources","data":{"id":4000619003,"name":"Wepow Staging","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} +{"stream":"sources","data":{"id":4001253003,"name":"Showcase Jobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} +{"stream":"sources","data":{"id":4001321003,"name":"Showcase QA","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} +{"stream":"sources","data":{"id":4001322003,"name":"Showcase Demo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} +{"stream":"sources","data":{"id":4001506003,"name":"Indeed - Sponsored","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4001507003,"name":"Indeed - Targeted Ad","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4002280003,"name":"Dash by Dashworks","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4002742003,"name":"Aleph","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4002891003,"name":"Xing","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4003381003,"name":"Talroo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4004993003,"name":"include.ai","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} +{"stream":"sources","data":{"id":4005607003,"name":"AppDirect","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4006015003,"name":"Otta","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4006251003,"name":"Revelo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4006550003,"name":"Rainmakers","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4007366003,"name":"Relode","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4007690003,"name":"JOIN","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4008420003,"name":"ERIN","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537381} +{"stream":"sources","data":{"id":4008511003,"name":"VentureBeat Careers, powered by Jobbio","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537382} +{"stream":"sources","data":{"id":4008617003,"name":"Monster Organic","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537382} +{"stream":"sources","data":{"id":4009581003,"name":"Sourcediv dev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537382} +{"stream":"sources","data":{"id":4009582003,"name":"Sourcediv","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537382} +{"stream":"sources","data":{"id":4009906003,"name":"PandoLogic","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537382} +{"stream":"sources","data":{"id":4010052003,"name":"JOBfindah","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537382} +{"stream":"sources","data":{"id":4010066003,"name":"Cord","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4010238003,"name":"purpose.jobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4010715003,"name":"Scout Hires","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4011228003,"name":"CBREX","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4012144003,"name":"Talent By Blind (dev)","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4012145003,"name":"Talent By Blind (test)","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4012387003,"name":"RecruitBot","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} +{"stream":"sources","data":{"id":4012474003,"name":"RepVue","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} +{"stream":"sources","data":{"id":4012953003,"name":"Jobplanner","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537384} +{"stream":"sources","data":{"id":4013544003,"name":"Test agency","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537384} +{"stream":"sources","data":{"id":4013712003,"name":"Jobstep","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} +{"stream":"sources","data":{"id":4014208003,"name":"Relyance","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} +{"stream":"sources","data":{"id":4014433003,"name":"Intrro","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} +{"stream":"sources","data":{"id":4014636003,"name":"DataFrenzy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4015128003,"name":"Talent By Blind","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4015567003,"name":"Real Links","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4016305003,"name":"Scouted","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4016428003,"name":"Talentry","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4016532003,"name":"Careerjet","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4018722003,"name":"ProvenBase","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} +{"stream":"sources","data":{"id":4018841003,"name":"Homi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537386} +{"stream":"sources","data":{"id":4019367003,"name":"10x10","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537386} +{"stream":"sources","data":{"id":4019617003,"name":"Secret Tel Aviv","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537387} +{"stream":"sources","data":{"id":4020294003,"name":"BuiltIn","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537387} +{"stream":"sources","data":{"id":4020736003,"name":"Tobu","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4020760003,"name":"Velents","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4021421003,"name":"Upward","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4022468003,"name":"ZoomInfo for Recruiters","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4023661003,"name":"Prentus","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4024133003,"name":"Arbeitnow","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4024421003,"name":"GuidedCompass","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} +{"stream":"sources","data":{"id":4024601003,"name":"Elpha","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4025746003,"name":"Indeed Hiring Platform","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4025759003,"name":"Elpha Dev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4026726003,"name":"Careerpuck","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4027676003,"name":"Appcast","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4027972003,"name":"SV Academy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4028644003,"name":"Tolstoy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4028649003,"name":"TolstoyDev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} +{"stream":"sources","data":{"id":4028668003,"name":"OutScout","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} +{"stream":"sources","data":{"id":4028669003,"name":"OutScoutDev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} +{"stream":"sources","data":{"id":4028843003,"name":"signNow","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} +{"stream":"sources","data":{"id":4029781003,"name":"TitanHouse","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} +{"stream":"sources","data":{"id":4030665003,"name":"Reprograma","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} +{"stream":"sources","data":{"id":4031825003,"name":"Trinsly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} +{"stream":"sources","data":{"id":4032805003,"name":"Brazen","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4034611003,"name":"Wednesday Talent","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4034771003,"name":"HRMarket 2","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4035349003,"name":"Phenom People","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4036053003,"name":"TalentMarketplace","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4036865003,"name":"Spleadly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4037066003,"name":"PathMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} +{"stream":"sources","data":{"id":4037068003,"name":"Bevov","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} +{"stream":"sources","data":{"id":4037490003,"name":"LinkedIn","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537392} +{"stream":"sources","data":{"id":4038632003,"name":"Reflr","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} +{"stream":"sources","data":{"id":4039580003,"name":"HeroHunt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} +{"stream":"sources","data":{"id":4040831003,"name":"Inclusively","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} +{"stream":"sources","data":{"id":4040946003,"name":"Us in Technology","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} +{"stream":"sources","data":{"id":4041406003,"name":"Retorio","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537393} +{"stream":"sources","data":{"id":4041695003,"name":"Waldo Labs Relay","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537393} +{"stream":"sources","data":{"id":4041720003,"name":"Supercharge","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537393} +{"stream":"sources","data":{"id":4041898003,"name":"matched.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537394} +{"stream":"rejection_reasons","data":{"id":4014678003,"name":"reason1","type":{"id":4000000003,"name":"We rejected them"}},"emitted_at":1660156537678} +{"stream":"rejection_reasons","data":{"id":4014679003,"name":"reason2","type":{"id":4000001003,"name":"They rejected us"}},"emitted_at":1660156537682} +{"stream":"rejection_reasons","data":{"id":4014680003,"name":"reason3","type":{"id":4000002003,"name":"None specified"}},"emitted_at":1660156537682} +{"stream":"jobs_openings","data":{"id":4320015003,"opening_id":"3-1","status":"open","opened_at":"2020-11-24T23:27:11.723Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539008} +{"stream":"jobs_openings","data":{"id":4320018003,"opening_id":"4-1","status":"open","opened_at":"2020-11-24T23:27:45.665Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539258} +{"stream":"jobs_openings","data":{"id":4926182003,"opening_id":"5-1","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} +{"stream":"jobs_openings","data":{"id":4928186003,"opening_id":"5-1","status":"open","opened_at":"2021-10-10T16:38:57.407Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} +{"stream":"jobs_openings","data":{"id":4928187003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:08.365Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} +{"stream":"jobs_openings","data":{"id":4928188003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:24.949Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} +{"stream":"jobs_openings","data":{"id":4926183003,"opening_id":"5-2","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} +{"stream":"jobs_openings","data":{"id":4928186003,"opening_id":"5-1","status":"open","opened_at":"2021-10-10T16:38:57.407Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539692} +{"stream":"jobs_openings","data":{"id":4928187003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:08.365Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539693} +{"stream":"jobs_openings","data":{"id":4928188003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:24.949Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539693} +{"stream":"jobs_openings","data":{"id":4970166003,"opening_id":"6-1","status":"open","opened_at":"2021-11-30T01:00:00.000Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156545037} +{"stream":"job_stages","data":{"id":5245803003,"name":"Application Review","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":0,"interviews":[{"id":5628614003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628609003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545600} +{"stream":"job_stages","data":{"id":5245804003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":1,"interviews":[{"id":5628615003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628610003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156545605} +{"stream":"job_stages","data":{"id":5245805003,"name":"Phone Interview","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":2,"interviews":[{"id":5628616003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628611003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545605} +{"stream":"job_stages","data":{"id":5245806003,"name":"Face to Face","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":3,"interviews":[{"id":5628617003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628612003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628618003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628613003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628619003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628614003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628620003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628615003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545606} +{"stream":"job_stages","data":{"id":5245807003,"name":"Reference Check","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":4,"interviews":[{"id":5628621003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628616003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156545606} +{"stream":"job_stages","data":{"id":5245808003,"name":"Offer","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":5,"interviews":[]},"emitted_at":1660156545607} +{"stream":"job_stages","data":{"id":5245818003,"name":"Application Review","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":0,"interviews":[{"id":5628634003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628629003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545607} +{"stream":"job_stages","data":{"id":5245819003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":1,"interviews":[{"id":5628635003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628630003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156545608} +{"stream":"job_stages","data":{"id":5245820003,"name":"Phone Interview","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":2,"interviews":[{"id":5628636003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628631003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545608} +{"stream":"job_stages","data":{"id":5245821003,"name":"Face to Face","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":3,"interviews":[{"id":5628637003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628632003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628638003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628633003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628639003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628634003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628640003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628635003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545609} +{"stream":"job_stages","data":{"id":5245822003,"name":"Reference Check","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":4,"interviews":[{"id":5628641003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628636003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156545609} +{"stream":"job_stages","data":{"id":5245823003,"name":"Offer","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":5,"interviews":[]},"emitted_at":1660156545610} +{"stream":"job_stages","data":{"id":7179760003,"name":"Application Review","created_at":"2021-10-08T08:19:42.584Z","updated_at":"2021-10-08T08:19:42.584Z","job_id":4446240003,"priority":0,"interviews":[{"id":8010560003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8010544003,"content":"

\r\n
Triage the inbox
\r\n
    \r\n
  • Quickly knock out any applications that are spam, duplicates, or clearly not worth your time
    • \r\n
  • Pass along qualified applicants to the next stage for assessment
    • \r\n
  • Reject any applicants who lack the necessary experience or required skills, or whose applications have poor grammar, spelling, or formatting 
    • \r\n
\r\n

","questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545610} +{"stream":"job_stages","data":{"id":7179761003,"name":"Preliminary Phone Screen","created_at":"2021-10-08T08:19:42.606Z","updated_at":"2021-10-08T08:19:42.606Z","job_id":4446240003,"priority":1,"interviews":[{"id":8010561003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":8010545003,"content":"
Purpose
\r\n
    \r\n
  • High-level screening call to make sure the candidate meets the basic requirements
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Explain the company and the job to the candidate: is it what he/she is looking for?
    • \r\n
  • Walk through his/her resume – are there any red flags?
    • \r\n
  • Find out what the candidate is looking for in his/her ideal role. Write this down so you can refer to this later!
    • \r\n
  • Get the candidate's desired salary range: is it in line with the compensation package?
    • \r\n
  • Tell the candidate what to expect next in your process
    • \r\n
\r\n

","questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156545610} +{"stream":"job_stages","data":{"id":7179762003,"name":"Phone Interview","created_at":"2021-10-08T08:19:42.620Z","updated_at":"2021-10-08T08:19:42.620Z","job_id":4446240003,"priority":2,"interviews":[{"id":8010562003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":8010546003,"content":"
Purpose
\r\n
    \r\n
  • Dig in deeper with a set of behavioral questions that target your desired skill set
    • \r\n
  • Specifically look for the following attributes: [Insert desired attributes here]
    • \r\n
  • Determine whether the candidate should be brought in for a face-to-face interview
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545611} +{"stream":"job_stages","data":{"id":7179763003,"name":"Face to Face","created_at":"2021-10-08T08:19:42.632Z","updated_at":"2021-10-08T08:19:42.632Z","job_id":4446240003,"priority":3,"interviews":[{"id":8010563003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":8010547003,"content":"
Purpose
\r\n
    \r\n
  • Determine whether or not the candidate would be a strong addition to the organization
    • \r\n
  • Do they live by your company values? [Insert Company Values Here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Ask the candidate for specific examples of times when they demonstrated your company values, e.g., “Tell me about a time when you took ownership of a project from start to finish”
    • \r\n
  • Ask about their motivations in their work - what excites them, what worries them, how do they work best, etc.
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010564003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":8010548003,"content":"
    \r\n
  • Ask the candidate behavioral questions that target the skills and personality traits you're looking for
    • \r\n
  • Specifically dig in on the following skills: [Insert skills and traits here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Is this someone who you want to work with and would add value to the team?
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010565003,"name":"Case Study","schedulable":true,"interview_kit":{"id":8010549003,"content":"

[FIll in your case study problem here]

\r\n

 

\r\n
    \r\n
  • Prepare a case study or problem for the candidate that mimics a real-world situation that he/she would face in the role
    • \r\n
  • Does the candidate approach the problem analytically and logically?
    • \r\n
  • Ask why he/she made certain decisions. Do they demonstrate a desired way of thinking?
    • \r\n
  • Ask follow-up questions to test the candidate further. (e.g., “What would you do this happened? How would you handle the following objection by a superior?”)
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":8010566003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":8010550003,"content":"
    \r\n
  • Unstructured conversation between Executive or Hiring Manager and the candidate
    • \r\n
  • Find out what motivates the candidate. What would they be most excited about if offered the position? What would they be most nervous about?
    • \r\n
  • Sell the candidate on the vision of the company and the quality of the team
    • \r\n
  • Answer any questions the candidate may have
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545611} +{"stream":"job_stages","data":{"id":7179764003,"name":"Reference Check","created_at":"2021-10-08T08:19:42.665Z","updated_at":"2021-10-08T08:19:42.665Z","job_id":4446240003,"priority":4,"interviews":[{"id":8010567003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":8010551003,"content":"
    \r\n
  • Have a quick chat with a former boss and confirm what the candidate said. Did the candidate accurately represent his/her responsibilities in their previous job?
    • \r\n
  • What does the boss think the candidate's biggest strengths are? What would the boss be most concerned about if hiring the candidate again?
    • \r\n
  • Ask about any red flags or questions you might have about the candidate
    • \r\n
","questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156545611} +{"stream":"job_stages","data":{"id":7179765003,"name":"Offer","created_at":"2021-10-08T08:19:42.679Z","updated_at":"2021-10-08T08:19:42.679Z","job_id":4446240003,"priority":5,"interviews":[]},"emitted_at":1660156545611} +{"stream":"job_stages","data":{"id":7332462003,"name":"Application Review","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":0,"interviews":[{"id":8202995003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8202979003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545612} +{"stream":"job_stages","data":{"id":7332463003,"name":"Offer","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":1,"interviews":[]},"emitted_at":1660156545612} +{"stream":"jobs_stages","data":{"id":5245803003,"name":"Application Review","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":0,"interviews":[{"id":5628614003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628609003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156546562} +{"stream":"jobs_stages","data":{"id":5245804003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":1,"interviews":[{"id":5628615003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628610003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156546566} +{"stream":"jobs_stages","data":{"id":5245805003,"name":"Phone Interview","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":2,"interviews":[{"id":5628616003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628611003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546566} +{"stream":"jobs_stages","data":{"id":5245806003,"name":"Face to Face","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":3,"interviews":[{"id":5628617003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628612003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628618003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628613003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628619003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628614003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628620003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628615003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546566} +{"stream":"jobs_stages","data":{"id":5245807003,"name":"Reference Check","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":4,"interviews":[{"id":5628621003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628616003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156546567} +{"stream":"jobs_stages","data":{"id":5245808003,"name":"Offer","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":5,"interviews":[]},"emitted_at":1660156546567} +{"stream":"jobs_stages","data":{"id":5245818003,"name":"Application Review","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":0,"interviews":[{"id":5628634003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628629003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156546749} +{"stream":"jobs_stages","data":{"id":5245819003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":1,"interviews":[{"id":5628635003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628630003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156546750} +{"stream":"jobs_stages","data":{"id":5245820003,"name":"Phone Interview","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":2,"interviews":[{"id":5628636003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628631003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546750} +{"stream":"jobs_stages","data":{"id":5245821003,"name":"Face to Face","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":3,"interviews":[{"id":5628637003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628632003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628638003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628633003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628639003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628634003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628640003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628635003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546750} +{"stream":"jobs_stages","data":{"id":5245822003,"name":"Reference Check","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":4,"interviews":[{"id":5628641003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628636003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156546751} +{"stream":"jobs_stages","data":{"id":5245823003,"name":"Offer","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":5,"interviews":[]},"emitted_at":1660156546751} +{"stream":"jobs_stages","data":{"id":7179760003,"name":"Application Review","created_at":"2021-10-08T08:19:42.584Z","updated_at":"2021-10-08T08:19:42.584Z","job_id":4446240003,"priority":0,"interviews":[{"id":8010560003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8010544003,"content":"

\r\n
Triage the inbox
\r\n
    \r\n
  • Quickly knock out any applications that are spam, duplicates, or clearly not worth your time
    • \r\n
  • Pass along qualified applicants to the next stage for assessment
    • \r\n
  • Reject any applicants who lack the necessary experience or required skills, or whose applications have poor grammar, spelling, or formatting 
    • \r\n
\r\n

","questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156546992} +{"stream":"jobs_stages","data":{"id":7179761003,"name":"Preliminary Phone Screen","created_at":"2021-10-08T08:19:42.606Z","updated_at":"2021-10-08T08:19:42.606Z","job_id":4446240003,"priority":1,"interviews":[{"id":8010561003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":8010545003,"content":"
Purpose
\r\n
    \r\n
  • High-level screening call to make sure the candidate meets the basic requirements
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Explain the company and the job to the candidate: is it what he/she is looking for?
    • \r\n
  • Walk through his/her resume – are there any red flags?
    • \r\n
  • Find out what the candidate is looking for in his/her ideal role. Write this down so you can refer to this later!
    • \r\n
  • Get the candidate's desired salary range: is it in line with the compensation package?
    • \r\n
  • Tell the candidate what to expect next in your process
    • \r\n
\r\n

","questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156546994} +{"stream":"jobs_stages","data":{"id":7179762003,"name":"Phone Interview","created_at":"2021-10-08T08:19:42.620Z","updated_at":"2021-10-08T08:19:42.620Z","job_id":4446240003,"priority":2,"interviews":[{"id":8010562003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":8010546003,"content":"
Purpose
\r\n
    \r\n
  • Dig in deeper with a set of behavioral questions that target your desired skill set
    • \r\n
  • Specifically look for the following attributes: [Insert desired attributes here]
    • \r\n
  • Determine whether the candidate should be brought in for a face-to-face interview
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546995} +{"stream":"jobs_stages","data":{"id":7179763003,"name":"Face to Face","created_at":"2021-10-08T08:19:42.632Z","updated_at":"2021-10-08T08:19:42.632Z","job_id":4446240003,"priority":3,"interviews":[{"id":8010563003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":8010547003,"content":"
Purpose
\r\n
    \r\n
  • Determine whether or not the candidate would be a strong addition to the organization
    • \r\n
  • Do they live by your company values? [Insert Company Values Here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Ask the candidate for specific examples of times when they demonstrated your company values, e.g., “Tell me about a time when you took ownership of a project from start to finish”
    • \r\n
  • Ask about their motivations in their work - what excites them, what worries them, how do they work best, etc.
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010564003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":8010548003,"content":"
    \r\n
  • Ask the candidate behavioral questions that target the skills and personality traits you're looking for
    • \r\n
  • Specifically dig in on the following skills: [Insert skills and traits here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Is this someone who you want to work with and would add value to the team?
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010565003,"name":"Case Study","schedulable":true,"interview_kit":{"id":8010549003,"content":"

[FIll in your case study problem here]

\r\n

 

\r\n
    \r\n
  • Prepare a case study or problem for the candidate that mimics a real-world situation that he/she would face in the role
    • \r\n
  • Does the candidate approach the problem analytically and logically?
    • \r\n
  • Ask why he/she made certain decisions. Do they demonstrate a desired way of thinking?
    • \r\n
  • Ask follow-up questions to test the candidate further. (e.g., “What would you do this happened? How would you handle the following objection by a superior?”)
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":8010566003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":8010550003,"content":"
    \r\n
  • Unstructured conversation between Executive or Hiring Manager and the candidate
    • \r\n
  • Find out what motivates the candidate. What would they be most excited about if offered the position? What would they be most nervous about?
    • \r\n
  • Sell the candidate on the vision of the company and the quality of the team
    • \r\n
  • Answer any questions the candidate may have
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546995} +{"stream":"jobs_stages","data":{"id":7179764003,"name":"Reference Check","created_at":"2021-10-08T08:19:42.665Z","updated_at":"2021-10-08T08:19:42.665Z","job_id":4446240003,"priority":4,"interviews":[{"id":8010567003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":8010551003,"content":"
    \r\n
  • Have a quick chat with a former boss and confirm what the candidate said. Did the candidate accurately represent his/her responsibilities in their previous job?
    • \r\n
  • What does the boss think the candidate's biggest strengths are? What would the boss be most concerned about if hiring the candidate again?
    • \r\n
  • Ask about any red flags or questions you might have about the candidate
    • \r\n
","questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156546996} +{"stream":"jobs_stages","data":{"id":7179765003,"name":"Offer","created_at":"2021-10-08T08:19:42.679Z","updated_at":"2021-10-08T08:19:42.679Z","job_id":4446240003,"priority":5,"interviews":[]},"emitted_at":1660156546996} +{"stream":"jobs_stages","data":{"id":7332462003,"name":"Application Review","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":0,"interviews":[{"id":8202995003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8202979003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156547151} +{"stream":"jobs_stages","data":{"id":7332463003,"name":"Offer","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":1,"interviews":[]},"emitted_at":1660156547152} diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index ebb8f91512d5..4f989f4d083a 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -16,7 +16,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1"], + install_requires=["airbyte-cdk~=0.1.74"], package_data={"": ["*.json", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml new file mode 100644 index 000000000000..28dc67fc49ad --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml @@ -0,0 +1,277 @@ +version: "0.1.0" + +definitions: + schema_loader: + file_path: "./source_greenhouse/schemas/{{ options['name'] }}.json" + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_pointer: [] + requester: + type: HttpRequester + name: "{{ options['name'] }}" + url_base: "https://harvest.greenhouse.io/v1/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['api_key'] }}" + retriever: + type: SimpleRetriever + name: "{{ options['name'] }}" + primary_key: "{{ options['primary_key'] }}" + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: LimitPaginator + page_size: 100 + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ headers['link']['next']['url'] }}" + stop_condition: "{{ 'next' not in headers['link'] }}" + limit_option: + field_name: "per_page" + inject_into: "request_parameter" + page_token_option: + inject_into: "path" + url_base: "*ref(definitions.requester.url_base)" + base_stream: + $options: + name: "applications" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + applications_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "applications" + path: "applications" + primary_key: "id" + candidates_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "candidates" + path: "candidates" + primary_key: "id" + close_reasons_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "close_reasons" + path: "close_reasons" + primary_key: "id" + degrees_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "degrees" + path: "degrees" + departments_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "departments" + path: "departments" + jobs_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "jobs" + path: "jobs" + jobs_openings_stream: + $options: + name: "jobs_openings" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "jobs/{{ stream_slice.parent_id }}/openings" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.jobs_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + applications_demographics_answers_stream: + $options: + name: "applications_demographics_answers" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "applications/{{ stream_slice.parent_id }}/demographics/answers" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.applications_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + applications_interviews_stream: + $options: + name: "applications_interviews" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "applications/{{ stream_slice.parent_id }}/scheduled_interviews" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.applications_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + custom_fields_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "custom_fields" + path: "custom_fields" + questions_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "demographics_questions" + path: "demographics/questions" + demographics_answers_answer_options_stream: + $options: + name: "demographics_answers_answer_options" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "demographics/questions/{{ stream_slice.parent_id }}/answer_options" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.questions_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + demographics_question_sets_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "demographics_question_sets" + path: "demographics/question_sets" + demographics_question_sets_questions_stream: + $options: + name: "demographics_question_sets_questions" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "demographics/question_sets/{{ stream_slice.parent_id }}/questions" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.demographics_question_sets_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + interviews_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "interviews" + path: "scheduled_interviews" + job_posts_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "job_posts" + path: "job_posts" + job_stages_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "job_stages" + path: "job_stages" + jobs_stages_stream: + $options: + name: "jobs_stages" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "jobs/{{ stream_slice.parent_id }}/stages" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.jobs_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + offers_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "offers" + path: "offers" + rejection_reasons_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "rejection_reasons" + path: "rejection_reasons" + scorecards_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "scorecards" + path: "scorecards" + sources_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "sources" + path: "sources" + users_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "users" + path: "users" + demographics_answers_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "demographics_answers" + path: "demographics/answers" + demographics_answer_options_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "demographics_answer_options" + path: "demographics/answer_options" +streams: + - "*ref(definitions.applications_stream)" + - "*ref(definitions.applications_demographics_answers_stream)" + - "*ref(definitions.applications_interviews_stream)" + - "*ref(definitions.candidates_stream)" + - "*ref(definitions.close_reasons_stream)" + - "*ref(definitions.custom_fields_stream)" + - "*ref(definitions.degrees_stream)" + - "*ref(definitions.demographics_answers_stream)" + - "*ref(definitions.demographics_answer_options_stream)" + - "*ref(definitions.questions_stream)" + - "*ref(definitions.demographics_answers_answer_options_stream)" + - "*ref(definitions.demographics_question_sets_stream)" + - "*ref(definitions.demographics_question_sets_questions_stream)" + - "*ref(definitions.departments_stream)" + - "*ref(definitions.jobs_stream)" + - "*ref(definitions.jobs_openings_stream)" + - "*ref(definitions.interviews_stream)" + - "*ref(definitions.job_posts_stream)" + - "*ref(definitions.job_stages_stream)" + - "*ref(definitions.jobs_stages_stream)" + - "*ref(definitions.offers_stream)" + - "*ref(definitions.rejection_reasons_stream)" + - "*ref(definitions.scorecards_stream)" + - "*ref(definitions.sources_stream)" + - "*ref(definitions.users_stream)" + +check: + type: CheckStream + stream_names: ["applications"] diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py index 0b4e92621782..bdc9933f3c6d 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py @@ -2,80 +2,16 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from requests.auth import HTTPBasicAuth -from source_greenhouse.streams import ( - Applications, - ApplicationsDemographicsAnswers, - ApplicationsInterviews, - Candidates, - CloseReasons, - CustomFields, - Degrees, - DemographicsAnswerOptions, - DemographicsAnswers, - DemographicsAnswersAnswerOptions, - DemographicsQuestions, - DemographicsQuestionSets, - DemographicsQuestionSetsQuestions, - Departments, - Interviews, - JobPosts, - Jobs, - JobsOpenings, - JobsStages, - JobStages, - Offers, - RejectionReasons, - Scorecards, - Sources, - Users, -) +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -class SourceGreenhouse(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - try: - auth = HTTPBasicAuth(config["api_key"], "") - users_gen = Users(authenticator=auth).read_records(sync_mode=SyncMode.full_refresh) - next(users_gen) - return True, None - except Exception as error: - return False, f"Unable to connect to Greenhouse API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = HTTPBasicAuth(config["api_key"], "") - streams = [ - Applications(authenticator=auth), - ApplicationsInterviews(authenticator=auth), - Candidates(authenticator=auth), - CloseReasons(authenticator=auth), - CustomFields(authenticator=auth), - Degrees(authenticator=auth), - Departments(authenticator=auth), - Interviews(authenticator=auth), - JobPosts(authenticator=auth), - JobStages(authenticator=auth), - Jobs(authenticator=auth), - JobsOpenings(authenticator=auth), - JobsStages(authenticator=auth), - Offers(authenticator=auth), - RejectionReasons(authenticator=auth), - Scorecards(authenticator=auth), - Sources(authenticator=auth), - Users(authenticator=auth), - ApplicationsDemographicsAnswers(authenticator=auth), - DemographicsAnswers(authenticator=auth), - DemographicsAnswerOptions(authenticator=auth), - DemographicsQuestions(authenticator=auth), - DemographicsAnswersAnswerOptions(authenticator=auth), - DemographicsQuestionSets(authenticator=auth), - DemographicsQuestionSetsQuestions(authenticator=auth), - ] - - return streams +# Declarative Source +class SourceGreenhouse(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "./source_greenhouse/greenhouse.yaml"}) diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json index 6cced6a87c2b..7d64621c2531 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json @@ -5,12 +5,14 @@ "title": "Greenhouse Spec", "type": "object", "required": ["api_key"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "api_key": { + "title": "API Key", "type": "string", "description": "Greenhouse API Key. See the docs for more information on how to generate this key.", - "airbyte_secret": true + "airbyte_secret": true, + "order": 0 } } } diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/streams.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/streams.py deleted file mode 100644 index 22cbaa82c010..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/streams.py +++ /dev/null @@ -1,250 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Type -from urllib import parse - -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -class GreenhouseStream(HttpStream, ABC): - url_base = "https://harvest.greenhouse.io/v1/" - page_size = 100 - primary_key = "id" - - def path(self, **kwargs) -> str: - # wrap with str() to pass mypy pre-commit validation - return str(self.name) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - # parsing response header links is the recommended pagination method https://developers.greenhouse.io/harvest.html#pagination - parsed_link = parse.urlparse(response.links.get("next", {}).get("url", "")) - if parsed_link: - query_params = dict(parse.parse_qsl(parsed_link.query)) - return query_params - return None - - def request_params(self, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]: - params = {"per_page": self.page_size} - if next_page_token: - params.update(next_page_token) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - - yield from response.json() - - -class GreenhouseSubStream(GreenhouseStream): - @property - @abstractmethod - def path_template(self) -> str: - """ - :return: sub stream path template - """ - - @property - @abstractmethod - def parent_stream(self) -> Type[GreenhouseStream]: - """ - :return: parent stream class - """ - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - items = self.parent_stream(authenticator=self._session.auth) - for item in items.read_records(sync_mode=SyncMode.full_refresh): - yield {"parent_id": item["id"]} - - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - stream_slice = stream_slice or {} - return self.path_template.format(parent_id=stream_slice["parent_id"]) - - -class Applications(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-applications - """ - - -class ApplicationsDemographicsAnswers(GreenhouseSubStream, GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-answers - """ - - parent_stream = Applications - path_template = "applications/{parent_id}/demographics/answers" - - -class ApplicationsInterviews(GreenhouseSubStream, GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-scheduled-interviews-for-application - """ - - parent_stream = Applications - path_template = "applications/{parent_id}/scheduled_interviews" - - -class Candidates(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-candidates - """ - - -class CloseReasons(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-close-reasons - """ - - -class CustomFields(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-custom-fields - """ - - -class Degrees(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-degrees - """ - - -class DemographicsAnswers(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-answers - """ - - def path(self, **kwargs) -> str: - return "demographics/answers" - - -class DemographicsAnswerOptions(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-answer-options - """ - - def path(self, **kwargs) -> str: - return "demographics/answer_options" - - -class DemographicsQuestions(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-questions - """ - - def path(self, **kwargs) -> str: - return "demographics/questions" - - -class DemographicsAnswersAnswerOptions(GreenhouseSubStream, GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-answer-options-for-demographic-question - """ - - parent_stream = DemographicsQuestions - path_template = "demographics/questions/{parent_id}/answer_options" - - -class DemographicsQuestionSets(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-question-sets - """ - - def path(self, **kwargs) -> str: - return "demographics/question_sets" - - -class DemographicsQuestionSetsQuestions(GreenhouseSubStream, GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-demographic-questions-for-demographic-question-set - """ - - parent_stream = DemographicsQuestionSets - path_template = "demographics/question_sets/{parent_id}/questions" - - -class Departments(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-departments - """ - - -class Interviews(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-scheduled-interviews - """ - - def path(self, **kwargs) -> str: - return "scheduled_interviews" - - -class JobPosts(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-job-posts - """ - - -class JobStages(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-job-stages - """ - - -class Jobs(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-jobs - """ - - -class JobsOpenings(GreenhouseSubStream, GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-job-openings - """ - - parent_stream = Jobs - path_template = "jobs/{parent_id}/openings" - - -class JobsStages(GreenhouseSubStream, GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-job-stages-for-job - """ - - parent_stream = Jobs - path_template = "jobs/{parent_id}/stages" - - -class Offers(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-offers - """ - - -class RejectionReasons(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-rejection-reasons - """ - - -class Scorecards(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-scorecards - """ - - -class Sources(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-sources - """ - - -class Users(GreenhouseStream): - """ - Docs: https://developers.greenhouse.io/harvest.html#get-list-users - """ diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py index 5a883f8b013d..4757ed989fc3 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py @@ -3,57 +3,60 @@ # import json -from json import JSONDecodeError -from types import GeneratorType import pytest import requests -from requests.auth import HTTPBasicAuth -from source_greenhouse.streams import Applications +from source_greenhouse.source import SourceGreenhouse @pytest.fixture def applications_stream(): - auth = HTTPBasicAuth("api_key", "") - stream = Applications(authenticator=auth) - return stream + source = SourceGreenhouse() + streams = source.streams({}) + return [s for s in streams if s.name == "applications"][0] -def test_next_page_token_has_next(applications_stream): + +def create_response(headers): response = requests.Response() - response.headers = { - "link": f'; rel="next"' - } - next_page_token = applications_stream.next_page_token(response=response) - assert next_page_token == {"per_page": str(Applications.page_size), "since_id": "123456789"} + response_body = {"next": "https://airbyte.io/next_url"} + response._content = json.dumps(response_body).encode("utf-8") + response.headers = headers + return response + + +def test_next_page_token_has_next(applications_stream): + headers = {"link": '; rel="next"'} + response = create_response(headers) + next_page_token = applications_stream.retriever.next_page_token(response=response) + assert next_page_token == {"next_page_token": "https://harvest.greenhouse.io/v1/applications?per_page=100&since_id=123456789"} def test_next_page_token_has_not_next(applications_stream): - response = requests.Response() - next_page_token = applications_stream.next_page_token(response=response) + response = create_response({}) + next_page_token = applications_stream.retriever.next_page_token(response=response) - assert next_page_token == {} + assert next_page_token is None def test_request_params_next_page_token_is_not_none(applications_stream): - response = requests.Response() - response.headers = { - "link": f'; rel="next"' - } - next_page_token = applications_stream.next_page_token(response=response) - request_params = applications_stream.request_params(next_page_token=next_page_token, stream_state={}) - - assert request_params == {"per_page": str(Applications.page_size), "since_id": "123456789"} + response = create_response({"link": f'; rel="next"'}) + next_page_token = applications_stream.retriever.next_page_token(response=response) + request_params = applications_stream.retriever.request_params(next_page_token=next_page_token, stream_state={}) + path = applications_stream.retriever.path(next_page_token=next_page_token, stream_state={}) + assert "applications?per_page=100&since_id=123456789" == path + assert request_params == {"per_page": 100} def test_request_params_next_page_token_is_none(applications_stream): - request_params = applications_stream.request_params(stream_state={}) + request_params = applications_stream.retriever.request_params(stream_state={}) - assert request_params == {"per_page": Applications.page_size} + assert request_params == {"per_page": 100} def test_parse_response_expected_response(applications_stream): response = requests.Response() + response.status_code = 200 response_content = b""" [ { @@ -135,29 +138,17 @@ def test_parse_response_expected_response(applications_stream): ] """ response._content = response_content - parsed_response = applications_stream.parse_response(response) + parsed_response = applications_stream.retriever.parse_response(response, stream_state={}) records = [record for record in parsed_response] - assert isinstance(parsed_response, GeneratorType) assert records == json.loads(response_content) def test_parse_response_empty_content(applications_stream): response = requests.Response() + response.status_code = 200 response._content = b"[]" - parsed_response = applications_stream.parse_response(response) + parsed_response = applications_stream.retriever.parse_response(response, stream_state={}) records = [record for record in parsed_response] - assert isinstance(parsed_response, GeneratorType) assert records == [] - - -def test_parse_response_invalid_content(applications_stream): - response = requests.Response() - response._content = b"not json" - parsed_response = applications_stream.parse_response(response) - - assert isinstance(parsed_response, GeneratorType) - with pytest.raises(JSONDecodeError): - for _ in parsed_response: - pass diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile index 357777ca5bd7..cd03eb7f3907 100644 --- a/airbyte-integrations/connectors/source-harvest/Dockerfile +++ b/airbyte-integrations/connectors/source-harvest/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json index cc071925d0ba..c557775fd69f 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "parent_id": { + "type": "integer" + }, "amount": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json index cc071925d0ba..c557775fd69f 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "parent_id": { + "type": "integer" + }, "amount": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json index 28fbab6eaf43..7d62b9f3ae37 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "parent_id": { + "type": "integer" + }, "sent_by": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json index f096b8c4a686..1767ab4292de 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "parent_id": { + "type": "integer" + }, "sent_by": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json index 7ab3a20b236a..ddd2b46666a2 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "parent_id": { + "type": "integer" + }, "amount": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json index ae8710b066c1..3019b6ce51bb 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "parent_id": { + "type": "integer" + }, "is_project_manager": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index dfc828a3abf9..390bba2f0663 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -101,7 +101,7 @@ def request_params( return params -class HarvestSubStream(HarvestStream): +class HarvestSubStream(HarvestStream, ABC): @property @abstractmethod def path_template(self) -> str: @@ -124,6 +124,11 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: return self.path_template.format(parent_id=stream_slice["parent_id"]) + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response, stream_slice=stream_slice, **kwargs): + record["parent_id"] = stream_slice["parent_id"] + yield record + class Contacts(IncrementalHarvestStream): """ diff --git a/airbyte-integrations/connectors/source-hubplanner/.dockerignore b/airbyte-integrations/connectors/source-hubplanner/.dockerignore new file mode 100644 index 000000000000..beb33d510fd4 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_hubplanner +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-hubplanner/Dockerfile b/airbyte-integrations/connectors/source-hubplanner/Dockerfile new file mode 100644 index 000000000000..ba50ba0758fb --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_hubplanner ./source_hubplanner +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-hubplanner diff --git a/airbyte-integrations/connectors/source-hubplanner/README.md b/airbyte-integrations/connectors/source-hubplanner/README.md new file mode 100644 index 000000000000..f38554dc5785 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/README.md @@ -0,0 +1,132 @@ +# Hubplanner Source + +This is the repository for the Hubplanner source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hubplanner). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubplanner) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubplanner/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubplanner test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-hubplanner:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-hubplanner:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hubplanner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml new file mode 100644 index 000000000000..7dece07aad48 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-hubplanner:dev +tests: + spec: + - spec_path: "source_hubplanner/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["holidays"] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-hubplanner/build.gradle b/airbyte-integrations/connectors/source-hubplanner/build.gradle new file mode 100644 index 000000000000..c75aea3d3c21 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_hubplanner' +} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..1404db601f4b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "stream": { + "name": "billing_rates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "bookings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "clients", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "holidays", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "resources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json new file mode 100644 index 000000000000..92a71d900e59 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "invalid-api-key" +} diff --git a/airbyte-integrations/connectors/source-hubplanner/main.py b/airbyte-integrations/connectors/source-hubplanner/main.py new file mode 100644 index 000000000000..14edc91deb05 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hubplanner import SourceHubplanner + +if __name__ == "__main__": + source = SourceHubplanner() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubplanner/requirements.txt b/airbyte-integrations/connectors/source-hubplanner/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json new file mode 100644 index 000000000000..4fac0c64d90b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json @@ -0,0 +1,137 @@ +{ + "streams": [ + { + "stream": { + "name": "billing_rates", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "bookings", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "clients", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "holidays", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "resources", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py new file mode 100644 index 000000000000..cd9e6accd855 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_hubplanner", + description="Source implementation for Hubplanner.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py new file mode 100644 index 000000000000..3bcd0c2b2b7a --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceHubplanner + +__all__ = ["SourceHubplanner"] diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json new file mode 100644 index 000000000000..e3b3e029ea4f --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + }, + "metadata": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "label": { + "type": "string" + }, + "currency": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json new file mode 100644 index 000000000000..d359bcf702a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json @@ -0,0 +1,156 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "allDay": { + "type": ["null", "boolean"] + }, + "scale": { + "type": ["null", "string"] + }, + "start": { + "type": "string" + }, + "end": { + "type": "string" + }, + "categoryTemplateId": { + "type": ["null", "string"] + }, + "categoryName": { + "type": ["null", "string"] + }, + "stateValue": { + "type": ["null", "integer"] + }, + "resource": { + "type": "string" + }, + "project": { + "type": "string" + }, + "note": { + "type": ["null", "string"] + }, + "details": { + "properties": { + "offDaysCount": { + "type": ["null", "integer"] + }, + "workDaysCount": { + "type": ["null", "integer"] + }, + "holidayCount": { + "type": ["null", "integer"] + }, + "workWeekDetails": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "bookedMinutes": { + "type": ["null", "integer"] + }, + "budgetBookedAmount": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "createdDate": { + "format": "date-time", + "type": ["null", "string"] + }, + "updatedDate": { + "format": "date-time", + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "string"] + }, + "customFields": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "templateId": { + "type": "string" + }, + "templateType": { + "type": "string" + }, + "templateLabel": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "choices": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "choiceId": { + "type": "string" + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "bookingRate": { + "properties": { + "external": { + "properties": { + "defaultRateId": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "internal": { + "properties": { + "defaultRateId": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "bookingCreatorId": { + "type": ["null", "string"] + }, + "lastUpdatedById": { + "type": ["null", "string"] + }, + "backgroundColor": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json new file mode 100644 index 000000000000..c40248dc84dd --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "name": { + "type": "string" + }, + "company": { + "type": ["null", "string"] + }, + "__v": { + "type": ["null", "integer"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json new file mode 100644 index 000000000000..8486b058ccbe --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "name": { + "type": "string" + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "backgroundColor": { + "type": ["null", "string"] + }, + "label": { + "type": "string" + }, + "metadata": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json new file mode 100644 index 000000000000..b774f5cf2d20 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "name": { + "type": "string" + }, + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "metadata": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json new file mode 100644 index 000000000000..af3395edabc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json @@ -0,0 +1,271 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "name": { + "type": "string" + }, + "links": { + "properties": {}, + "type": ["null", "object"] + }, + "notes": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "timeEntryEnabled": { + "type": ["null", "boolean"] + }, + "timeEntryLocked": { + "type": ["null", "boolean"] + }, + "timeEntryApproval": { + "type": ["null", "boolean"] + }, + "resourceRates": { + "items": { + "properties": { + "resource": { + "type": ["null", "string"] + }, + "companyBillingRateId": { + "type": ["null", "string"] + }, + "companyBillingRate": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "includeBookedTimeReports": { + "type": ["null", "boolean"] + }, + "includeBookedTimeGrid": { + "type": ["null", "boolean"] + }, + "projectManagers": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "resources": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "workDays": { + "items": { + "type": ["null", "boolean"] + }, + "type": ["null", "array"] + }, + "useProjectDays": { + "type": ["null", "boolean"] + }, + "budget": { + "properties": { + "hasBudget": { + "type": ["null", "boolean"] + }, + "projectHours": { + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "hours": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "cashAmount": { + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "amount": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "billingRate": { + "properties": { + "useDefault": { + "type": ["null", "boolean"] + }, + "rate": { + "type": ["null", "number"] + }, + "id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "companyBillingRateId": { + "type": ["null", "string"] + }, + "budgetHours": { + "type": ["null", "number"] + }, + "budgetCashAmount": { + "type": ["null", "number"] + }, + "budgetCurrency": { + "type": ["null", "string"] + }, + "useStatusColor": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + }, + "useProjectDuration": { + "type": ["null", "boolean"] + }, + "start": { + "type": ["null", "string"] + }, + "end": { + "type": ["null", "string"] + }, + "backgroundColor": { + "type": ["null", "string"] + }, + "projectCode": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "string"] + }, + "customFields": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "templateId": { + "type": "string" + }, + "templateType": { + "type": "string" + }, + "templateLabel": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "choices": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "choiceId": { + "type": "string" + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "timeEntryNoteRequired": { + "type": ["null", "boolean"] + }, + "projectRate": { + "properties": { + "external": { + "properties": { + "defaultRateId": { + "type": ["null", "string"] + }, + "customRates": { + "items": { + "properties": { + "_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "resourceId": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "internal": { + "properties": { + "customRates": { + "items": { + "properties": { + "_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "resourceId": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "defaultRateId": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "note": { + "type": ["null", "string"] + }, + "tags": { + "items": { + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json new file mode 100644 index 000000000000..1b123381b9cc --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "firstName": { + "type": "string" + }, + "lastName": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "links": { + "properties": {}, + "type": ["null", "object"] + }, + "billing": { + "properties": { + "useDefault": { + "type": ["null", "boolean"] + }, + "rate": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "useCustomAvailability": { + "type": ["null", "boolean"] + }, + "customFields": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "templateId": { + "type": "string" + }, + "templateType": { + "type": "string" + }, + "templateLabel": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "choices": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "choiceId": { + "type": "string" + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "resourceRates": { + "properties": { + "external": { + "items": { + "properties": {}, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "internal": { + "items": { + "properties": {}, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "isProjectManager": { + "type": ["null", "boolean"] + }, + "tags": { + "items": { + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py new file mode 100644 index 000000000000..3ca0496d538b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py @@ -0,0 +1,248 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator + + +# Basic full refresh stream +class HubplannerStream(HttpStream, ABC): + + url_base = "https://api.hubplanner.com/v1" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +# Basic incremental stream +class IncrementalHubplannerStream(HubplannerStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class HubplannerAuthenticator(HttpAuthenticator): + def __init__(self, token: str, auth_header: str = "Authorization"): + self.auth_header = auth_header + self._token = token + + def get_auth_header(self) -> Mapping[str, Any]: + return {self.auth_header: f"{self._token}"} + + +class BillingRates(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/billingRate" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Bookings(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/booking" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Clients(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/client" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Events(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/event" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Holidays(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/holiday" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Projects(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/project" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Resources(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/resource" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +# Source +class SourceHubplanner(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + + url_base = "https://api.hubplanner.com/v1" + + try: + url = f"{url_base}/project" + + authenticator = HubplannerAuthenticator(token=config["api_key"]) + + session = requests.get(url, headers=authenticator.get_auth_header()) + session.raise_for_status() + + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + authenticator = HubplannerAuthenticator(token=config["api_key"]) + return [ + BillingRates(authenticator=authenticator), + Bookings(authenticator=authenticator), + Clients(authenticator=authenticator), + Events(authenticator=authenticator), + Holidays(authenticator=authenticator), + Projects(authenticator=authenticator), + Resources(authenticator=authenticator), + ] diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json new file mode 100644 index 000000000000..aa061822e322 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json @@ -0,0 +1,17 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubplanner", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hubplanner Spec", + "type": "object", + "required": ["api_key"], + "additionalProperties": true, + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See https://github.com/hubplanner/API#authentication for more details.", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py new file mode 100644 index 000000000000..363717924500 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_hubplanner.source import SourceHubplanner + + +def test_streams(mocker): + source = SourceHubplanner() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 7 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py new file mode 100644 index 000000000000..808f40802b20 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_hubplanner.source import HubplannerStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(HubplannerStream, "path", "v0/example_endpoint") + mocker.patch.object(HubplannerStream, "primary_key", "test_primary_key") + mocker.patch.object(HubplannerStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = HubplannerStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = HubplannerStream() + inputs = {"response": MagicMock()} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = HubplannerStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = HubplannerStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = HubplannerStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = HubplannerStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = HubplannerStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index a20b4a89a5ae..dd570f8a1475 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -15,7 +15,7 @@ TEST_REQUIREMENTS = [ "pytest==6.1.2", "pytest-mock~=3.6", - "requests_mock==1.8.0", + "requests-mock~=1.9.3", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore b/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore new file mode 100644 index 000000000000..53de6536d1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_linkedin_pages +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile new file mode 100644 index 000000000000..926f0c238fb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_linkedin_pages ./source_linkedin_pages + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-linkedin-pages diff --git a/airbyte-integrations/connectors/source-linkedin-pages/README.md b/airbyte-integrations/connectors/source-linkedin-pages/README.md new file mode 100644 index 000000000000..5671f959d3ff --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/README.md @@ -0,0 +1,132 @@ +# Linkedin Pages Source + +This is the repository for the Linkedin Pages source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/linkedin-pages). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-pages) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_pages/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-pages test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-linkedin-pages:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-linkedin-pages:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-linkedin-pages:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml new file mode 100644 index 000000000000..b7023bc46b22 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-config.yml @@ -0,0 +1,19 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-linkedin-pages:dev +tests: + spec: + - spec_path: "source_linkedin_pages/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md b/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md new file mode 100644 index 000000000000..90e5cbd5eb5c --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/bootstrap.md @@ -0,0 +1,7 @@ +The LinkedIn Marketing Developer Platform API can be used to pull data from LinkedIn Organizations such as company page details, follower counts, follower statistics, and all sorts of organic content data. + +You must have a LinkedIn Developers' App created in order to request access to the Marketing Developer Platform API. API Access approval takes up to 72 hours after submitting a ~15-question form. + +The app also must be verified by an admin of the LinkedIn organization your app is created for. Once the app is "verified" and granted access to the Marketing Developer Platform API, you can use their easy-peasy OAuth Token Tools to generate access tokens **and** refresh tokens. + +You can access the `client id` and `client secret` in the **Auth** tab of the app dashboard to round out all of the authorization needs you may have. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-pages/build.gradle b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle new file mode 100644 index 000000000000..df9098d6060f --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_linkedin_pages' +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..8f9f142bd912 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/abnormal_state.json @@ -0,0 +1,23 @@ +{ + "organization_lookup": { + "lastModified": "2050-01-01" + }, + "follower_statistics": { + "lastModified": "2050-01-01" + }, + "page_statistics": { + "lastModified": "2050-01-01" + }, + "share_statistics": { + "lastModified": "2050-01-01" + }, + "shares": { + "lastModified": "2050-01-01" + }, + "total_follower_count": { + "end_date": "2050-01-01" + }, + "ugc_posts": { + "end_date": "2050-01-01" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py new file mode 100644 index 000000000000..1d66fbf1a331 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + yield diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..c0e203e60479 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/configured_catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "organization_lookup", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "follower_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "share_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "total_follower_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json new file mode 100644 index 000000000000..fd62f7bf71a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/invalid_config.json @@ -0,0 +1,7 @@ +{ + "org_id": 12345678, + "credentials": { + "auth_method": "access_token", + "access_token": "wrong_token_sra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQl" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json new file mode 100644 index 000000000000..c32e7dbe2010 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_config.json @@ -0,0 +1,7 @@ +{ + "org_id": 12345678, + "credentials": { + "auth_method": "access_token", + "access_token": "example_token_string123" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/main.py b/airbyte-integrations/connectors/source-linkedin-pages/main.py new file mode 100644 index 000000000000..459f9cbf2253 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_pages import SourceLinkedinPages + +if __name__ == "__main__": + source = SourceLinkedinPages() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py new file mode 100644 index 000000000000..06721a7d3f41 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum~=2.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_linkedin_pages", + description="Source implementation for Linkedin Company Pages.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py new file mode 100644 index 000000000000..e4157287bd16 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceLinkedinPages + +__all__ = ["SourceLinkedinPages"] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json new file mode 100644 index 000000000000..fa4cd1548b00 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "followerCountsByStaffCountRange": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "followerCountsByFunction": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "followerCountsByAssociationType": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "followerCountsBySeniority": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json new file mode 100644 index 000000000000..34d327587056 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -0,0 +1,278 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "vanityName": { + "type": ["null", "string"] + }, + "localizedName": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "foundedOn": { + "type": ["null", "object"], + "properties": { + "year": { + "type": ["null", "integer"] + } + } + }, + "groups": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "description": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "versionTag": { + "type": ["null", "string"] + }, + "coverPhotoV2": { + "type": ["null", "object"], + "properties": { + "cropped": { + "type": ["null", "string"] + }, + "original": { + "type": ["null", "string"] + }, + "cropInfo": { + "type": ["null", "object"], + "properties": { + "x": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "y": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + } + } + } + } + }, + "defaultLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "organizationType": { + "type": ["null", "string"] + }, + "alternativeNames": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "specialties": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "staffCountRange": { + "type": ["null", "string"] + }, + "localizedSpecialties": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "industries": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "name": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "primaryOrganizationType": { + "type": ["null", "string"] + }, + "locations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "locationType": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "address": { + "type": ["null", "object"], + "properties": { + "geographicArea": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + } + } + }, + "localizedDescription": { + "type": ["null", "string"] + }, + "geoLocation": { + "type": ["null", "string"] + }, + "streetAddressFieldState": { + "type": ["null", "string"] + } + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "localizedDescription": { + "type": ["null", "string"] + }, + "$URN": { + "type": ["null", "string"] + }, + "localizedWebsite": { + "type": ["null", "string"] + }, + "logoV2": { + "type": ["null", "object"], + "properties": { + "cropped": { + "type": ["null", "string"] + }, + "original": { + "type": ["null", "string"] + }, + "cropInfo": { + "type": ["null", "object"], + "properties": { + "x": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "y": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json new file mode 100644 index 000000000000..77129bc9d5b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "totalShareStatistics": { + "type": ["null", "object"], + "properties": { + "uniqueImpressionsCount": { + "type": ["null", "integer"] + }, + "clickCount": { + "type": ["null", "integer"] + }, + "engagement": { + "type": ["null", "number"] + }, + "likeCount": { + "type": ["null", "integer"] + }, + "commentCount": { + "type": ["null", "integer"] + }, + "shareCount": { + "type": ["null", "integer"] + }, + "commentMentionsCount": { + "type": ["null", "integer"] + }, + "impressionCount": { + "type": ["null", "integer"] + }, + "shareMentionsCount": { + "type": ["null", "integer"] + } + } + }, + "organizationalEntity": { "type": ["null", "string"] } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json new file mode 100644 index 000000000000..84387c6c37b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "firstDegreeSize": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py new file mode 100644 index 000000000000..e03bbeecbb50 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/source.py @@ -0,0 +1,154 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator + + +class LinkedinPagesStream(HttpStream, ABC): + + url_base = "https://api.linkedin.com/v2/" + primary_key = None + + def __init__(self, config): + super().__init__(authenticator=config.get("authenticator")) + self.config = config + + @property + def org(self): + """Property to return the user Organization Id from input""" + return self.config.get("org_id") + + def path(self, **kwargs) -> str: + """Returns the API endpoint path for stream, from `endpoint` class attribute.""" + return self.endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + return [response.json()] + + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 429: + error_message = ( + f"Stream {self.name}: LinkedIn API requests are rate limited. " + f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " + f"These limits reset at midnight UTC every day. " + f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-pages. " + f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." + ) + self.logger.error(error_message) + return super().should_retry(response) + + +class OrganizationLookup(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizations/{self.org}" + return path + + +class FollowerStatistics(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" + return path + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + yield from response.json().get("elements") + + +class ShareStatistics(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" + return path + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + yield from response.json().get("elements") + + +class TotalFollowerCount(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" + return path + + +class SourceLinkedinPages(AbstractSource): + """ + Abstract Source inheritance, provides: + - implementation for `check` connector's connectivity + - implementation to call each stream with it's input parameters. + """ + + @classmethod + def get_authenticator(cls, config: Mapping[str, Any]) -> TokenAuthenticator: + """ + Validate input parameters and generate a necessary Authentication object + This connectors support 2 auth methods: + 1) direct access token with TTL = 2 months + 2) refresh token (TTL = 1 year) which can be converted to access tokens + Every new refresh revokes all previous access tokens q + """ + auth_method = config.get("credentials", {}).get("auth_method") + if not auth_method or auth_method == "access_token": + # support of backward compatibility with old exists configs + access_token = config["credentials"]["access_token"] if auth_method else config["access_token"] + return TokenAuthenticator(token=access_token) + elif auth_method == "oAuth2.0": + return Oauth2Authenticator( + token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", + client_id=config["credentials"]["client_id"], + client_secret=config["credentials"]["client_secret"], + refresh_token=config["credentials"]["refresh_token"], + ) + raise Exception("incorrect input parameters") + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # RUN $ python main.py check --config secrets/config.json + + """ + Testing connection availability for the connector. + :: for this check method the Customer must have the "r_liteprofile" scope enabled. + :: more info: https://docs.microsoft.com/linkedin/consumer/integrations/self-serve/sign-in-with-linkedin + """ + + config["authenticator"] = self.get_authenticator(config) + stream = OrganizationLookup(config) + stream.records_limit = 1 + try: + next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + return True, None + except Exception as e: + return False, e + + # RUN: $ python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config["authenticator"] = self.get_authenticator(config) + return [ + OrganizationLookup(config), + FollowerStatistics(config), + ShareStatistics(config), + TotalFollowerCount(config), + ShareStatistics(config), + TotalFollowerCount(config), + ] diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json new file mode 100644 index 000000000000..a335440fc6f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/spec.json @@ -0,0 +1,79 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Linkedin Pages Spec", + "type": "object", + "required": ["org_id"], + "additionalProperties": true, + "properties": { + "org_id": { + "title": "Organization ID", + "type": "integer", + "airbyte_secret": true, + "description": "Specify the Organization ID", + "examples": ["123456789"] + }, + "credentials": { + "title": "Authentication *", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": ["client_id", "client_secret", "refresh_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "oAuth2.0" + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The client ID of the LinkedIn developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client secret", + "description": "The client secret of the LinkedIn developer application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + }, + { + "title": "Access token", + "type": "object", + "required": ["access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + "title": "Access token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["credentials", "0"], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["refresh_token"]] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py new file mode 100644 index 000000000000..4bb4685fd7a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/source_linkedin_pages/utils.py @@ -0,0 +1,324 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any, Dict, Iterable, List, Mapping + +import pendulum as pdm + + +def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: + """ + Outputs the Dict with key:value slices for the stream. + :: EXAMPLE: + Input: + records = [{dict}, {dict}, ...], + key_value_map = {: } + + Output: + { + : records..value, + } + """ + result = {} + for key in key_value_map: + value = record.get(key_value_map[key]) + if value: + result[key] = value + return result + + +def transform_change_audit_stamps( + record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `changeAuditStamps` input structure: + { + "changeAuditStamps": { + "created": {"time": 1629581275000}, + "lastModified": {"time": 1629664544760} + } + } + + :: EXAMPLE output: + { + "created": "2021-08-21 21:27:55", + "lastModified": "2021-08-22 20:35:44" + } + """ + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record[prop] = pdm.from_timestamp(target_dict.get(prop).get(field) / 1000).to_datetime_string() + record.pop(dict_key) + + return record + + +def date_str_from_date_range(record: Dict, prefix: str) -> str: + """ + Makes the ISO8601 format date string from the input . + + EXAMPLE: + Input: record + { + "start.year": 2021, "start.month": 8, "start.day": 1, + "end.year": 2021, "end.month": 9, "end.day": 31 + } + + EXAMPLE output: + With `prefix` = "start" + str: "2021-08-13", + + With `prefix` = "end" + str: "2021-09-31", + """ + + year = record.get(f"{prefix}.year") + month = record.get(f"{prefix}.month") + day = record.get(f"{prefix}.day") + return pdm.date(year, month, day).to_date_string() + + +def transform_date_range( + record: Dict, + dict_key: str = "dateRange", + props: List = ["start", "end"], + fields: List = ["year", "month", "day"], +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `dateRange` input structure in Analytics streams: + { + "dateRange": { + "start": {"month": 8, "day": 13, "year": 2021}, + "end": {"month": 8, "day": 13, "year": 2021} + } + } + :: EXAMPLE output: + { + "start_date": "2021-08-13", + "end_date": "2021-08-13" + } + """ + # define list of tmp keys for cleanup. + keys_to_remove = [dict_key, "start.day", "start.month", "start.year", "end.day", "end.month", "end.year", "start", "end"] + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record.update(**{f"{prop}.{field}": target_dict.get(prop).get(field)}) + # We build `start_date` & `end_date` fields from nested structure. + record.update(**{"start_date": date_str_from_date_range(record, "start"), "end_date": date_str_from_date_range(record, "end")}) + # Cleanup tmp fields & nested used parts + for key in keys_to_remove: + if key in record: + record.pop(key) + return record + + +def transform_targeting_criteria( + record: Dict, + dict_key: str = "targetingCriteria", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `targetingCriteria` input structure: + { + "targetingCriteria": { + "include": { + "and": [ + { + "or": { + "urn:li:adTargetingFacet:titles": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ] + } + }, + {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, + {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + ] + }, + "exclude": { + "or": { + "urn:li:adTargetingFacet:facet_Key1": [ + "facet_test1", + "facet_test2", + ], + "urn:li:adTargetingFacet:facet_Key2": [ + "facet_test3", + "facet_test4", + ], + } + } + } + + :: EXAMPLE output: + { + "targetingCriteria": { + "include": { + "and": [ + { + "type": "urn:li:adTargetingFacet:titles", + "values": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ], + }, + { + "type": "urn:li:adTargetingFacet:locations", + "values": ["urn:li:geo:103644278"], + }, + { + "type": "urn:li:adTargetingFacet:interfaceLocales", + "values": ["urn:li:locale:en_US"], + }, + ] + }, + "exclude": { + "or": [ + { + "type": "urn:li:adTargetingFacet:facet_Key1", + "values": ["facet_test1", "facet_test2"], + }, + { + "type": "urn:li:adTargetingFacet:facet_Key2", + "values": ["facet_test3", "facet_test4"], + }, + ] + }, + } + + """ + + def unnest_dict(nested_dict: Dict) -> Iterable[Dict]: + """ + Unnest the nested dict to simplify the normalization + + EXAMPLE OUTPUT: + [ + {"type": "some_key", "values": "some_values"}, + ..., + {"type": "some_other_key", "values": "some_other_values"} + ] + """ + + for key, value in nested_dict.items(): + values = [] + if isinstance(value, List): + if len(value) > 0: + if isinstance(value[0], str): + values = value + elif isinstance(value[0], Dict): + for v in value: + values.append(v) + elif isinstance(value, Dict): + values.append(value) + yield {"type": key, "values": values} + + # get the target dict from record + targeting_criteria = record.get(dict_key) + + # transform `include` + if "include" in targeting_criteria: + and_list = targeting_criteria.get("include").get("and") + updated_include = {"and": []} + for k in and_list: + or_dict = k.get("or") + for j in unnest_dict(or_dict): + updated_include["and"].append(j) + # Replace the original 'and' with updated_include + record["targetingCriteria"]["include"] = updated_include + + # transform `exclude` if present + if "exclude" in targeting_criteria: + or_dict = targeting_criteria.get("exclude").get("or") + updated_exclude = {"or": []} + for k in unnest_dict(or_dict): + updated_exclude["or"].append(k) + # Replace the original 'or' with updated_exclude + record["targetingCriteria"]["exclude"] = updated_exclude + + return record + + +def transform_variables( + record: Dict, + dict_key: str = "variables", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `variables` input: + { + "variables": { + "data": { + "com.linkedin.ads.SponsoredUpdateCreativeVariables": { + "activity": "urn:li:activity:1234", + "directSponsoredContent": 0, + "share": "urn:li:share:1234", + } + } + } + } + + :: EXAMPLE output: + { + "variables": { + "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", + "values": [ + {"key": "activity", "value": "urn:li:activity:1234"}, + {"key": "directSponsoredContent", "value": 0}, + {"key": "share", "value": "urn:li:share:1234"}, + ], + } + } + """ + + variables = record.get(dict_key).get("data") + for key, params in variables.items(): + record["variables"]["type"] = key + record["variables"]["values"] = [] + for key, value in params.items(): + # convert various datatypes of values into the string + record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)}) + # Clean the nested structure + record["variables"].pop("data") + return record + + +def transform_data(records: List) -> Iterable[Mapping]: + """ + We need to transform the nested complex data structures into simple key:value pair, + to be properly normalised in the destination. + """ + for record in records: + + if "changeAuditStamps" in record: + record = transform_change_audit_stamps(record) + + if "dateRange" in record: + record = transform_date_range(record) + + if "targetingCriteria" in record: + record = transform_targeting_criteria(record) + + if "variables" in record: + record = transform_variables(record) + + yield record diff --git a/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-pages/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 0881a4715a13..311b118461c7 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.12 +LABEL io.airbyte.version=0.4.15 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 4460af8ed78b..bdc7a3007ecb 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.13 +LABEL io.airbyte.version=0.4.15 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java index cc830c0f96c1..aca20541c9bb 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java @@ -68,7 +68,7 @@ public static MssqlCdcTargetPosition getTargetPosition(final JdbcDatabase databa try { final List jsonNodes = database .bufferedResultSetQuery(conn -> conn.createStatement().executeQuery( - "USE " + dbName + "; SELECT sys.fn_cdc_get_max_lsn() AS max_lsn;"), JdbcUtils.getDefaultSourceOperations()::rowToJson); + "USE [" + dbName + "]; SELECT sys.fn_cdc_get_max_lsn() AS max_lsn;"), JdbcUtils.getDefaultSourceOperations()::rowToJson); Preconditions.checkState(jsonNodes.size() == 1); if (jsonNodes.get(0).get("max_lsn") != null) { final Lsn maxLsn = Lsn.valueOf(jsonNodes.get(0).get("max_lsn").binaryValue()); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index cc998b59af85..370187835e0b 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.1 +LABEL io.airbyte.version=0.6.2 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index b9465054f44a..54ce574ace98 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.1 +LABEL io.airbyte.version=0.6.2 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 5cde4f85fd45..f8f770f71abf 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 9dfb9767a391..f35dada0802a 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java index 2e47c6bc1487..83c789664d58 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java @@ -13,7 +13,13 @@ public class PostgresCdcProperties { static Properties getDebeziumDefaultProperties(final JsonNode config) { final Properties props = commonProperties(); props.setProperty("plugin.name", PostgresUtils.getPluginValue(config.get("replication_method"))); - props.setProperty("snapshot.mode", "initial"); + if (config.has("snapshot_mode")) { + // The parameter `snapshot_mode` is passed in test to simulate reading the WAL Logs directly and + // skip initial snapshot + props.setProperty("snapshot.mode", config.get("snapshot_mode").asText()); + } else { + props.setProperty("snapshot.mode", "initial"); + } props.setProperty("slot.name", config.get("replication_method").get("replication_slot").asText()); props.setProperty("publication.name", config.get("replication_method").get("publication").asText()); @@ -29,6 +35,7 @@ private static Properties commonProperties() { props.setProperty("converters", "datetime"); props.setProperty("datetime.type", PostgresConverter.class.getName()); + props.setProperty("include.unknown.datatypes", "true"); return props; } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index 03a7a6d910e1..fd927593deea 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -4,8 +4,6 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.db.DataTypeUtils.TIMESTAMP_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIME_FORMATTER; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; @@ -20,6 +18,7 @@ import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.db.DataTypeUtils; +import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.protocol.models.JsonSchemaType; import java.math.BigDecimal; @@ -214,26 +213,17 @@ public void setJsonField(final ResultSet resultSet, final int colIndex, final Ob @Override protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - LocalDate date = getObject(resultSet, index, LocalDate.class); - if (isBce(date)) { - // java.time uses a year 0, but the standard AD/BC system does not. So we just subtract one to hack - // around this difference. - date = date.minusYears(1); - } - node.put(columnName, resolveEra(date, date.toString())); + node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class))); } @Override protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - final LocalTime time = getObject(resultSet, index, LocalTime.class); - node.put(columnName, time.format(TIME_FORMATTER)); + node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime.class))); } @Override protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - final LocalDateTime timestamp = getObject(resultSet, index, LocalDateTime.class); - final LocalDate date = timestamp.toLocalDate(); - node.put(columnName, resolveEra(date, timestamp.format(TIMESTAMP_FORMATTER))); + node.put(columnName, DateTimeConverter.convertToTimestamp(resultSet.getTimestamp(index))); } @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java index afe2854743aa..5ea93aded985 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java @@ -398,21 +398,6 @@ protected void initTests() { .build()); } - // time with time zone - for (final String fullSourceType : Set.of("timetz", "time with time zone")) { - addDataTypeTestData( - TestDataHolder.builder() - .sourceType("timetz") - .fullSourceDataType(fullSourceType) - .airbyteType(JsonSchemaType.STRING_TIME_WITH_TIMEZONE) - .addInsertValues("null", "'13:00:01'", "'13:00:00+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.012345Z+8'", "'13:00:06.00000Z-8'") - // A time value without time zone will use the time zone set on the database, which is Z-7, - // so 13:00:01 is returned as 13:00:01-07. - .addExpectedValues(null, "13:00:01.000000-07:00", "13:00:00.000000+08:00", "13:00:03.000000-08:00", "13:00:04.000000Z", - "13:00:05.012345-08:00", "13:00:06.000000+08:00") - .build()); - } - // timestamp without time zone for (final String fullSourceType : Set.of("timestamp", "timestamp without time zone")) { addDataTypeTestData( @@ -555,6 +540,25 @@ protected void initTests() { {"ISBN-13":"978-1449370000","weight":"11.2 ounces","paperback":"243","publisher":"postgresqltutorial.com","language":"English"}""", null) .build()); + + addTimeWithTimeZoneTest(); + } + + protected void addTimeWithTimeZoneTest() { + // time with time zone + for (final String fullSourceType : Set.of("timetz", "time with time zone")) { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("timetz") + .fullSourceDataType(fullSourceType) + .airbyteType(JsonSchemaType.STRING_TIME_WITH_TIMEZONE) + .addInsertValues("null", "'13:00:01'", "'13:00:00+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.012345Z+8'", "'13:00:06.00000Z-8'") + // A time value without time zone will use the time zone set on the database, which is Z-7, + // so 13:00:01 is returned as 13:00:01-07. + .addExpectedValues(null, "13:00:01.000000-07:00", "13:00:00.000000+08:00", "13:00:03.000000-08:00", "13:00:04.000000Z", + "13:00:05.012345-08:00", "13:00:06.000000+08:00") + .build()); + } } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java similarity index 96% rename from airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java rename to airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java index 37abb3a3c544..bc9ff87b0522 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java @@ -18,7 +18,7 @@ import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; -public class CdcPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { +public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { private static final String SCHEMA_NAME = "test"; private static final String SLOT_NAME_BASE = "debezium_slot"; @@ -55,6 +55,7 @@ protected Database setupDatabase() throws Exception { .put("replication_method", replicationMethod) .put("is_test", true) .put(JdbcUtils.SSL_KEY, false) + .put("snapshot_mode", "initial_only") .build()); dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java new file mode 100644 index 000000000000..773146fe5c89 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.standardtest.source.TestDataHolder; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.JsonSchemaType; +import java.util.List; +import java.util.Set; +import org.jooq.SQLDialect; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.MountableFile; + +public class CdcWalLogsPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { + + private static final String SCHEMA_NAME = "test"; + private static final String SLOT_NAME_BASE = "debezium_slot"; + private static final String PUBLICATION = "publication"; + private static final int INITIAL_WAITING_SECONDS = 5; + private JsonNode stateAfterFirstSync; + + @Override + protected List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { + if (stateAfterFirstSync == null) { + throw new RuntimeException("stateAfterFirstSync is null"); + } + return super.runRead(configuredCatalog, stateAfterFirstSync); + } + + @Override + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + final Database database = setupDatabase(); + initTests(); + for (final TestDataHolder test : testDataHolders) { + database.query(ctx -> { + ctx.fetch(test.getCreateSqlQuery()); + return null; + }); + } + + final ConfiguredAirbyteStream dummyTableWithData = createDummyTableWithData(database); + final ConfiguredAirbyteCatalog catalog = getConfiguredCatalog(); + catalog.getStreams().add(dummyTableWithData); + + final List allMessages = super.runRead(catalog); + if (allMessages.size() != 2) { + throw new RuntimeException("First sync should only generate 2 records"); + } + final List stateAfterFirstBatch = extractStateMessages(allMessages); + if (stateAfterFirstBatch == null || stateAfterFirstBatch.isEmpty()) { + throw new RuntimeException("stateAfterFirstBatch should not be null or empty"); + } + stateAfterFirstSync = Jsons.jsonNode(stateAfterFirstBatch); + if (stateAfterFirstSync == null) { + throw new RuntimeException("stateAfterFirstSync should not be null"); + } + for (final TestDataHolder test : testDataHolders) { + database.query(ctx -> { + test.getInsertSqlQueries().forEach(ctx::fetch); + return null; + }); + } + } + + @Override + protected Database setupDatabase() throws Exception { + + container = new PostgreSQLContainer<>("postgres:14-alpine") + .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), + "/etc/postgresql/postgresql.conf") + .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); + container.start(); + + /** + * The publication is not being set as part of the config and because of it + * {@link io.airbyte.integrations.source.postgres.PostgresSource#isCdc(JsonNode)} returns false, as + * a result no test in this class runs through the cdc path. + */ + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .put("replication_slot", SLOT_NAME_BASE) + .put("publication", PUBLICATION) + .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) + .build()); + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put("replication_method", replicationMethod) + .put("is_test", true) + .put(JdbcUtils.SSL_KEY, false) + .build()); + + dslContext = DSLContextFactory.create( + config.get(JdbcUtils.USERNAME_KEY).asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + container.getHost(), + container.getFirstMappedPort(), + config.get(JdbcUtils.DATABASE_KEY).asText()), + SQLDialect.POSTGRES); + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.execute( + "SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); + ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); + ctx.execute("CREATE EXTENSION hstore;"); + return null; + }); + + database.query(ctx -> ctx.fetch("CREATE SCHEMA TEST;")); + database.query(ctx -> ctx.fetch("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');")); + database.query(ctx -> ctx.fetch("CREATE TYPE inventory_item AS (\n" + + " name text,\n" + + " supplier_id integer,\n" + + " price numeric\n" + + ");")); + + database.query(ctx -> ctx.fetch("SET TIMEZONE TO 'MST'")); + return database; + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + dslContext.close(); + container.close(); + } + + public boolean testCatalog() { + return true; + } + + @Override + protected void addTimeWithTimeZoneTest() { + // time with time zone + for (final String fullSourceType : Set.of("timetz", "time with time zone")) { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("timetz") + .fullSourceDataType(fullSourceType) + .airbyteType(JsonSchemaType.STRING_TIME_WITH_TIMEZONE) + .addInsertValues("null", "'13:00:01'", "'13:00:00+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.012345Z+8'", "'13:00:06.00000Z-8'") + // A time value without time zone will use the time zone set on the database, which is Z-7, + // so 13:00:01 is returned as 13:00:01-07. + .addExpectedValues(null, "20:00:01.000000Z", "05:00:00.000000Z", "21:00:03.000000Z", "13:00:04.000000Z", "21:00:05.012345Z", + "05:00:06.000000Z") + .build()); + } + } + +} diff --git a/airbyte-integrations/connectors/source-recurly/Dockerfile b/airbyte-integrations/connectors/source-recurly/Dockerfile index 6518a6a5da91..51646e6746f5 100644 --- a/airbyte-integrations/connectors/source-recurly/Dockerfile +++ b/airbyte-integrations/connectors/source-recurly/Dockerfile @@ -34,5 +34,5 @@ COPY source_recurly ./source_recurly ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.0 +LABEL io.airbyte.version=0.4.1 LABEL io.airbyte.name=airbyte/source-recurly diff --git a/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py b/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py index 78de431fbbca..37c75bfefc68 100644 --- a/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py +++ b/airbyte-integrations/connectors/source-recurly/source_recurly/streams.py @@ -22,6 +22,8 @@ class BaseStream(Stream): + state_checkpoint_interval = 1000 + def __init__(self, client: Client, begin_time: str = None, end_time: str = None, **kwargs): super(Stream, self).__init__(**kwargs) diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index d2880e26a3cd..12370d9468b6 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -13,6 +13,7 @@ import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Iterator; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,10 +28,16 @@ public class StateDecoratingIterator extends AbstractIterator im private final JsonSchemaPrimitive cursorType; private final int stateEmissionFrequency; + private final String initialCursor; private String maxCursor; - private AirbyteMessage intermediateStateMessage; private boolean hasEmittedFinalState; - private int recordCount; + + // The intermediateStateMessage is set to the latest state message. + // For every stateEmissionFrequency messages, emitIntermediateState is set to true and + // the latest intermediateStateMessage will be emitted. + private int totalRecordCount = 0; + private boolean emitIntermediateState = false; + private AirbyteMessage intermediateStateMessage = null; /** * @param stateEmissionFrequency If larger than 0, intermediate states will be emitted for every @@ -49,6 +56,7 @@ public StateDecoratingIterator(final Iterator messageIterator, this.pair = pair; this.cursorField = cursorField; this.cursorType = cursorType; + this.initialCursor = initialCursor; this.maxCursor = initialCursor; this.stateEmissionFrequency = stateEmissionFrequency; } @@ -60,36 +68,41 @@ private String getCursorCandidate(final AirbyteMessage message) { @Override protected AirbyteMessage computeNext() { - if (intermediateStateMessage != null) { - final AirbyteMessage message = intermediateStateMessage; - intermediateStateMessage = null; - return message; - } else if (messageIterator.hasNext()) { - recordCount++; + if (messageIterator.hasNext()) { + if (emitIntermediateState && intermediateStateMessage != null) { + final AirbyteMessage message = intermediateStateMessage; + intermediateStateMessage = null; + emitIntermediateState = false; + return message; + } + + totalRecordCount++; final AirbyteMessage message = messageIterator.next(); if (message.getRecord().getData().hasNonNull(cursorField)) { final String cursorCandidate = getCursorCandidate(message); if (IncrementalUtils.compareCursors(maxCursor, cursorCandidate, cursorType) < 0) { + if (stateEmissionFrequency > 0 && !Objects.equals(maxCursor, initialCursor) && messageIterator.hasNext()) { + // Only emit an intermediate state when it is not the first or last record message, + // because the last state message will be taken care of in a different branch. + intermediateStateMessage = createStateMessage(false); + } maxCursor = cursorCandidate; } } - if (stateEmissionFrequency > 0 && recordCount % stateEmissionFrequency == 0) { - // Mark the state as final in case this intermediate state happens to be the last one. - // This is not necessary, but avoid sending the final states twice and prevent any edge case. - final boolean isFinalState = !messageIterator.hasNext(); - intermediateStateMessage = emitStateMessage(isFinalState); + if (stateEmissionFrequency > 0 && totalRecordCount % stateEmissionFrequency == 0) { + emitIntermediateState = true; } return message; } else if (!hasEmittedFinalState) { - return emitStateMessage(true); + return createStateMessage(true); } else { return endOfData(); } } - public AirbyteMessage emitStateMessage(final boolean isFinalState) { + public AirbyteMessage createStateMessage(final boolean isFinalState) { final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, maxCursor); LOGGER.info("State Report: stream name: {}, original cursor field: {}, original cursor value {}, cursor field: {}, new cursor value: {}", pair, diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/resources/db_models/db_models.yaml b/airbyte-integrations/connectors/source-relational-db/src/main/resources/db_models/db_models.yaml index d058b5e306ee..28af27c8046b 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/resources/db_models/db_models.yaml +++ b/airbyte-integrations/connectors/source-relational-db/src/main/resources/db_models/db_models.yaml @@ -27,7 +27,7 @@ definitions: "$ref": "#/definitions/DbStreamState" DbStreamState: type: object - additionalProperties: false + additionalProperties: true required: - stream_name - stream_namespace diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java index 8f16a7d5a11f..474d553d3a1d 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -45,10 +45,18 @@ class StateDecoratingIteratorTest { private static final AirbyteMessage RECORD_MESSAGE_2 = createRecordMessage(RECORD_VALUE_2); private static final AirbyteMessage STATE_MESSAGE_2 = createStateMessage(RECORD_VALUE_2); - private static final String RECORD_VALUE_3 = "xyz"; + private static final String RECORD_VALUE_3 = "ghi"; private static final AirbyteMessage RECORD_MESSAGE_3 = createRecordMessage(RECORD_VALUE_3); private static final AirbyteMessage STATE_MESSAGE_3 = createStateMessage(RECORD_VALUE_3); + private static final String RECORD_VALUE_4 = "jkl"; + private static final AirbyteMessage RECORD_MESSAGE_4 = createRecordMessage(RECORD_VALUE_4); + private static final AirbyteMessage STATE_MESSAGE_4 = createStateMessage(RECORD_VALUE_4); + + private static final String RECORD_VALUE_5 = "xyz"; + private static final AirbyteMessage RECORD_MESSAGE_5 = createRecordMessage(RECORD_VALUE_5); + private static final AirbyteMessage STATE_MESSAGE_5 = createStateMessage(RECORD_VALUE_5); + private static AirbyteMessage createRecordMessage(final String recordValue) { return new AirbyteMessage() .withType(Type.RECORD) @@ -73,6 +81,8 @@ void setup() { when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_1)).thenReturn(STATE_MESSAGE_1.getState()); when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_2)).thenReturn(STATE_MESSAGE_2.getState()); when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_3)).thenReturn(STATE_MESSAGE_3.getState()); + when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_4)).thenReturn(STATE_MESSAGE_4.getState()); + when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_5)).thenReturn(STATE_MESSAGE_5.getState()); when(stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); when(stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); @@ -106,13 +116,13 @@ void testWithInitialCursor() { stateManager, NAME_NAMESPACE_PAIR, UUID_FIELD_NAME, - RECORD_VALUE_3, + RECORD_VALUE_5, JsonSchemaPrimitive.STRING, 0); assertEquals(RECORD_MESSAGE_1, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); - assertEquals(STATE_MESSAGE_3, iterator.next()); + assertEquals(STATE_MESSAGE_5, iterator.next()); assertFalse(iterator.hasNext()); } @@ -179,8 +189,8 @@ void testUnicodeNull() { @Test @DisplayName("When initial cursor is null, and emit state for every record") - void testStateEmission1() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3); + void testStateEmissionFrequency1() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -191,19 +201,27 @@ void testStateEmission1() { 1); assertEquals(RECORD_MESSAGE_1, iterator1.next()); - assertEquals(STATE_MESSAGE_1, iterator1.next()); + // should emit state 1, but it is unclear whether there will be more + // records with the same cursor value, so no state is ready for emission assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); + // emit state 1 because it is the latest state ready for emission + assertEquals(STATE_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); - // final state message should only be emitted once + assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + // state 4 is not emitted because there is no more record and only + // the final state should be emitted at this point; also the final + // state should only be emitted once + assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } @Test @DisplayName("When initial cursor is null, and emit state for every 2 records") - void testStateEmission2() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3); + void testStateEmissionFrequency2() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -215,16 +233,74 @@ void testStateEmission2() { assertEquals(RECORD_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); + // emit state 1 because it is the latest state ready for emission + assertEquals(STATE_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); + // emit state 3 because it is the latest state ready for emission assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } @Test @DisplayName("When initial cursor is not null") - void testStateEmission3() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_2, RECORD_MESSAGE_3); + void testStateEmissionWhenInitialCursorIsNotNull() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); + final StateDecoratingIterator iterator1 = new StateDecoratingIterator( + messageIterator, + stateManager, + NAME_NAMESPACE_PAIR, + UUID_FIELD_NAME, + RECORD_VALUE_1, + JsonSchemaPrimitive.STRING, + 1); + + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); + assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_5, iterator1.next()); + assertFalse(iterator1.hasNext()); + } + + /** + * Incremental syncs will sort the table with the cursor field, and emit the max cursor for every N + * records. The purpose is to emit the states frequently, so that if any transient failure occurs + * during a long sync, the next run does not need to start from the beginning, but can resume from + * the last successful intermediate state committed on the destination. The next run will start with + * `cursorField > cursor`. However, it is possible that there are multiple records with the same + * cursor value. If the intermediate state is emitted before all these records have been synced to + * the destination, some of these records may be lost. + *

+ * Here is an example: + * + * 

+   * | Record ID | Cursor Field | Other Field | Note                          |
+   * | --------- | ------------ | ----------- | ----------------------------- |
+   * | 1         | F1=16        | F2="abc"    |                               |
+   * | 2         | F1=16        | F2="def"    | <- state emission and failure |
+   * | 3         | F1=16        | F2="ghi"    |                               |
+   *
+ * + * If the intermediate state is emitted for record 2 and the sync fails immediately such that the + * cursor value `16` is committed, but only record 1 and 2 are actually synced, the next run will + * start with `F1 > 16` and skip record 3. + *

+ * So intermediate state emission should only happen when all records with the same cursor value has + * been synced to destination. Reference: https://github.com/airbytehq/airbyte/issues/15427 + */ + @Test + @DisplayName("When there are multiple records with the same cursor value") + void testStateEmissionForRecordsSharingSameCursorValue() { + messageIterator = MoreIterators.of( + RECORD_MESSAGE_2, RECORD_MESSAGE_2, + RECORD_MESSAGE_3, RECORD_MESSAGE_3, RECORD_MESSAGE_3, + RECORD_MESSAGE_4, + RECORD_MESSAGE_5, RECORD_MESSAGE_5); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -235,9 +311,19 @@ void testStateEmission3() { 1); assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + // state 2 is the latest state ready for emission because + // all records with the same cursor value have been emitted assertEquals(STATE_MESSAGE_2, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_4, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index 1395a0442e5a..d68ba279dae9 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.12 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/README.md b/airbyte-integrations/connectors/source-salesforce/README.md index 474d96df01b7..ef03b9599b80 100644 --- a/airbyte-integrations/connectors/source-salesforce/README.md +++ b/airbyte-integrations/connectors/source-salesforce/README.md @@ -81,7 +81,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` -pip install .[tests] +pip install ".[tests]" ``` ### Unit Tests To run unit tests locally, from the connector directory run: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 22ec66f84191..7234f0894a55 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -149,8 +149,8 @@ def read_records( class BulkSalesforceStream(SalesforceStream): - page_size = 30000 - DEFAULT_WAIT_TIMEOUT_SECONDS = 600 + page_size = 15000 + DEFAULT_WAIT_TIMEOUT_SECONDS = 86400 # 24-hour bulk job running time MAX_CHECK_INTERVAL_SECONDS = 2.0 MAX_RETRY_NUMBER = 3 diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile index 80cb0bc4e57f..b1459933c7f1 100644 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ b/airbyte-integrations/connectors/source-sendgrid/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.8 +LABEL io.airbyte.version=0.2.9 LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index c5826e3411d8..1b84fed88cd9 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -11,6 +11,6 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2", "pytest-mock"], + install_requires=["airbyte-cdk>=0.1.74", "backoff", "requests", "pytest==6.1.2", "pytest-mock"], package_data={"": ["*.json", "schemas/*.json"]}, ) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml new file mode 100644 index 000000000000..d03e3d987701 --- /dev/null +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml @@ -0,0 +1,256 @@ +definitions: + page_size: 50 + step: "30d" + + schema_loader: + type: JsonSchema + file_path: "./source_sendgrid/schemas/{{ options.name }}.json" + + requester: + type: HttpRequester + name: "{{ options['name'] }}" + url_base: "https://api.sendgrid.com" + http_method: "GET" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + cursor_paginator: + type: LimitPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size: "*ref(definitions.page_size)" + limit_option: + inject_into: "request_parameter" + field_name: "page_size" + page_token_option: + inject_into: "path" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + offset_paginator: + type: LimitPaginator + $options: + url_base: "*ref(definitions.requester.url_base)" + page_size: "*ref(definitions.page_size)" + limit_option: + inject_into: "request_parameter" + field_name: "limit" + page_token_option: + inject_into: "request_parameter" + field_name: "offset" + pagination_strategy: + type: "OffsetIncrement" + retriever: + type: SimpleRetriever + name: "{{ options['name'] }}" + primary_key: "{{ options['primary_key'] }}" + stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_time'] }}" + datetime_format: "%s" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f%z" + step: "*ref(definitions.step)" + cursor_field: "{{ options.stream_cursor_field }}" + start_time_option: + field_name: "start_time" + inject_into: "request_parameter" + end_time_option: + field_name: "end_time" + inject_into: "request_parameter" + datetime_format: "%s" + messages_stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_time'] }}" + datetime_format: "%s" + end_datetime: + datetime: "{{ now_utc() }}}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f%z" + step: "*ref(definitions.step)" + cursor_field: "{{ options.stream_cursor_field }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + + base_stream: + type: DeclarativeStream + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + extractor: + field_pointer: [] + requester: + $ref: "*ref(definitions.requester)" + paginator: + type: NoPagination +streams: + - $ref: "*ref(definitions.base_stream)" + $options: + name: "lists" + primary_key: "id" + path: "/v3/marketing/lists" + field_pointer: ["result"] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.cursor_paginator)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "campaigns" + primary_key: "id" + path: "/v3/marketing/campaigns" + field_pointer: ["result"] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.cursor_paginator)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "contacts" + primary_key: "id" + path: "/v3/marketing/contacts" + field_pointer: ["result"] + - $ref: "*ref(definitions.base_stream)" + $options: + name: "stats_automations" + primary_key: "id" + path: "/v3/marketing/stats/automations" + field_pointer: ["results"] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.cursor_paginator)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "segments" + primary_key: "id" + path: "/v3/marketing/segments" + field_pointer: ["results"] + - $ref: "*ref(definitions.base_stream)" + $options: + name: "single_sends" + primary_key: "id" + path: "/v3/marketing/stats/singlesends" + field_pointer: ["results"] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.cursor_paginator)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "templates" + primary_key: "id" + path: "/v3/templates" + field_pointer: ["result"] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + requester: + $ref: "*ref(definitions.base_stream.retriever.requester)" + request_options_provider: + request_parameters: + generations: "legacy,dynamic" + paginator: + $ref: "*ref(definitions.cursor_paginator)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "bounces" + primary_key: "email" + stream_cursor_field: "created" + path: "/v3/suppression/bounces" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "global_suppressions" + primary_key: "email" + stream_cursor_field: "created" + path: "/v3/suppression/unsubscribes" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "blocks" + primary_key: "email" + stream_cursor_field: "created" + path: "/v3/suppression/blocks" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "suppression_groups" + primary_key: "id" + path: "/v3/asm/groups" + field_pointer: [] + - $ref: "*ref(definitions.base_stream)" + $options: + name: "suppression_group_members" + primary_key: "group_id" + path: "/v3/asm/suppressions" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "invalid_emails" + primary_key: "email" + stream_cursor_field: "created" + path: "/v3/suppression/invalid_emails" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "spam_reports" + primary_key: "email" + stream_cursor_field: "created" + path: "/v3/suppression/spam_reports" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + - $ref: "*ref(definitions.base_stream)" + $options: + name: "messages" + primary_key: "msg_id" + stream_cursor_field: "last_event_time" + path: "/v3/messages" + field_pointer: [] + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + requester: + $ref: "*ref(definitions.requester)" + request_options_provider: + request_parameters: + limit: 1000 + query: 'last_event_time BETWEEN TIMESTAMP "{{stream_slice.start_time}}" AND TIMESTAMP "{{stream_slice.end_time}}"' + stream_slicer: + $ref: "*ref(definitions.messages_stream_slicer)" +check: + type: CheckStream + stream_names: ["lists"] diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py index 973c7d45ad0b..925955ee4e18 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py @@ -2,63 +2,17 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -from .streams import ( - Blocks, - Bounces, - Campaigns, - Contacts, - GlobalSuppressions, - InvalidEmails, - Lists, - Messages, - Scopes, - Segments, - SingleSends, - SpamReports, - StatsAutomations, - SuppressionGroupMembers, - SuppressionGroups, - Templates, -) - -class SourceSendgrid(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - authenticator = TokenAuthenticator(config["apikey"]) - scopes_gen = Scopes(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh) - next(scopes_gen) - return True, None - except Exception as error: - return False, f"Unable to connect to Sendgrid API with the provided credentials - {error}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = TokenAuthenticator(config["apikey"]) - - streams = [ - Lists(authenticator=authenticator), - Campaigns(authenticator=authenticator), - Contacts(authenticator=authenticator), - StatsAutomations(authenticator=authenticator), - Segments(authenticator=authenticator), - SingleSends(authenticator=authenticator), - Templates(authenticator=authenticator), - Messages(authenticator=authenticator, start_time=config["start_time"]), - GlobalSuppressions(authenticator=authenticator, start_time=config["start_time"]), - SuppressionGroups(authenticator=authenticator), - SuppressionGroupMembers(authenticator=authenticator), - Blocks(authenticator=authenticator, start_time=config["start_time"]), - Bounces(authenticator=authenticator, start_time=config["start_time"]), - InvalidEmails(authenticator=authenticator, start_time=config["start_time"]), - SpamReports(authenticator=authenticator, start_time=config["start_time"]), - ] - - return streams +# Declarative Source +class SourceSendgrid(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "./source_sendgrid/sendgrid.yaml"}) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json index dfd5c3634e7e..acaceb6b30bb 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json @@ -5,7 +5,7 @@ "title": "Sendgrid Spec", "type": "object", "required": ["apikey"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "apikey": { "title": "Sendgrid API key", diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py deleted file mode 100644 index 542c0dafd21c..000000000000 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ /dev/null @@ -1,273 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import datetime -import urllib -from abc import ABC, abstractmethod -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import pendulum -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class SendgridStream(HttpStream, ABC): - url_base = "https://api.sendgrid.com/v3/" - primary_key = "id" - limit = 50 - data_field = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - pass - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - json_response = response.json() - records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - - if records is not None: - for record in records: - yield record - else: - # TODO sendgrid's API is sending null responses at times. This seems like a bug on the API side, so we're adding - # log statements to help reproduce and prevent the connector from failing. - err_msg = ( - f"Response contained no valid JSON data. Response body: {response.text}\n" - f"Response status: {response.status_code}\n" - f"Response body: {response.text}\n" - f"Response headers: {response.headers}\n" - f"Request URL: {response.request.url}\n" - f"Request body: {response.request.body}\n" - ) - # do NOT print request headers as it contains auth token - self.logger.info(err_msg) - - -class SendgridStreamOffsetPagination(SendgridStream): - offset = 0 - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(next_page_token=next_page_token, **kwargs) - params["limit"] = self.limit - if next_page_token: - params.update(**next_page_token) - return params - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = response.json() - if self.data_field: - stream_data = stream_data[self.data_field] - if len(stream_data) < self.limit: - return - self.offset += self.limit - return {"offset": self.offset} - - -class SendgridStreamIncrementalMixin(HttpStream, ABC): - cursor_field = "created" - - def __init__(self, start_time: int, **kwargs): - super().__init__(**kwargs) - self._start_time = start_time - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} - - def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state) - start_time = self._start_time - if stream_state.get(self.cursor_field): - start_time = stream_state[self.cursor_field] - params.update({"start_time": start_time, "end_time": pendulum.now().int_timestamp}) - return params - - -class SendgridStreamMetadataPagination(SendgridStream): - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = {} - if not next_page_token: - params = {"page_size": self.limit} - return params - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page_url = response.json()["_metadata"].get("next", False) - if next_page_url: - return {"next_page_url": next_page_url.replace(self.url_base, "")} - - @staticmethod - @abstractmethod - def initial_path() -> str: - """ - :return: initial path for the API endpoint if no next metadata url found - """ - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - if next_page_token: - return next_page_token["next_page_url"] - return self.initial_path() - - -class Scopes(SendgridStream): - def path(self, **kwargs) -> str: - return "scopes" - - -class Lists(SendgridStreamMetadataPagination): - data_field = "result" - - @staticmethod - def initial_path() -> str: - return "marketing/lists" - - -class Campaigns(SendgridStreamMetadataPagination): - data_field = "result" - - @staticmethod - def initial_path() -> str: - return "marketing/campaigns" - - -class Contacts(SendgridStream): - data_field = "result" - - def path(self, **kwargs) -> str: - return "marketing/contacts" - - -class StatsAutomations(SendgridStreamMetadataPagination): - data_field = "results" - - @staticmethod - def initial_path() -> str: - return "marketing/stats/automations" - - -class Segments(SendgridStream): - data_field = "results" - - def path(self, **kwargs) -> str: - return "marketing/segments" - - -class SingleSends(SendgridStreamMetadataPagination): - """ - https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-single-sends-stats - """ - - data_field = "results" - - @staticmethod - def initial_path() -> str: - return "marketing/stats/singlesends" - - -class Templates(SendgridStreamMetadataPagination): - data_field = "result" - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(next_page_token=next_page_token, **kwargs) - params["generations"] = "legacy,dynamic" - return params - - @staticmethod - def initial_path() -> str: - return "templates" - - -class Messages(SendgridStream, SendgridStreamIncrementalMixin): - """ - https://docs.sendgrid.com/api-reference/e-mail-activity/filter-all-messages - """ - - data_field = "messages" - cursor_field = "last_event_time" - primary_key = "msg_id" - limit = 1000 - - def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - time_filter_template = "%Y-%m-%dT%H:%M:%SZ" - params = super().request_params(stream_state=stream_state, **kwargs) - if isinstance(params["start_time"], int): - date_start = datetime.datetime.fromtimestamp(params["start_time"]).strftime(time_filter_template) - else: - date_start = params["start_time"] - date_end = datetime.datetime.fromtimestamp(int(params["end_time"])).strftime(time_filter_template) - queryapi = f'last_event_time BETWEEN TIMESTAMP "{date_start}" AND TIMESTAMP "{date_end}"' - params["query"] = urllib.parse.quote(queryapi) - params["limit"] = self.limit - payload_str = "&".join("%s=%s" % (k, v) for k, v in params.items() if k not in ["start_time", "end_time"]) - return payload_str - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return "messages" - - -class GlobalSuppressions(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): - primary_key = "email" - - def path(self, **kwargs) -> str: - return "suppression/unsubscribes" - - -class SuppressionGroups(SendgridStream): - def path(self, **kwargs) -> str: - return "asm/groups" - - -class SuppressionGroupMembers(SendgridStreamOffsetPagination): - primary_key = "group_id" - - def path(self, **kwargs) -> str: - return "asm/suppressions" - - -class Blocks(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): - primary_key = "email" - - def path(self, **kwargs) -> str: - return "suppression/blocks" - - -class Bounces(SendgridStream, SendgridStreamIncrementalMixin): - primary_key = "email" - - def path(self, **kwargs) -> str: - return "suppression/bounces" - - -class InvalidEmails(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): - primary_key = "email" - - def path(self, **kwargs) -> str: - return "suppression/invalid_emails" - - -class SpamReports(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): - primary_key = "email" - - def path(self, **kwargs) -> str: - return "suppression/spam_reports" diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index 8f50befe1e23..3173b3aea5e8 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -2,31 +2,59 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +import json +import unittest +import pendulum import pytest import requests from airbyte_cdk.logger import AirbyteLogger from source_sendgrid.source import SourceSendgrid -from source_sendgrid.streams import SendgridStream +FAKE_NOW = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("utc")) -@pytest.fixture(name="sendgrid_stream") -def sendgrid_stream_fixture(mocker) -> SendgridStream: - # Wipe the internal list of abstract methods to allow instantiating the abstract class without implementing its abstract methods - mocker.patch("source_sendgrid.streams.SendgridStream.__abstractmethods__", set()) - # Mypy yells at us because we're init'ing an abstract class - return SendgridStream() # type: ignore +@pytest.fixture() +def mock_pendulum_now(monkeypatch): + pendulum_mock = unittest.mock.MagicMock(wraps=pendulum.now) + pendulum_mock.return_value = FAKE_NOW + monkeypatch.setattr(pendulum, "now", pendulum_mock) -def test_parse_response_gracefully_handles_nulls(mocker, sendgrid_stream: SendgridStream): + +def get_stream(stream_name): + source = SourceSendgrid() + streams = source.streams({}) + + return [s for s in streams if s.name == stream_name][0] + + +def create_response(): response = requests.Response() - mocker.patch.object(response, "json", return_value=None) - mocker.patch.object(response, "request", return_value=MagicMock()) - assert [] == list(sendgrid_stream.parse_response(response)) + response_body = {} + response.status_code = 200 + response._content = json.dumps(response_body).encode("utf-8") + return response + + +def test_parse_response_gracefully_handles_nulls(): + response = create_response() + assert [] == list(get_stream("contacts").retriever.parse_response(response, stream_slice={}, stream_state={})) def test_source_wrong_credentials(): source = SourceSendgrid() status, error = source.check_connection(logger=AirbyteLogger(), config={"apikey": "wrong.api.key123"}) assert not status + + +def test_messages_stream_request_params(mock_pendulum_now): + stream = get_stream("messages") + state = {"last_event_time": 1558359000} + expected_params = { + "query": 'last_event_time BETWEEN TIMESTAMP "2019-05-20T06:30:00Z" AND TIMESTAMP "2021-12-31T16:00:00Z"', + "limit": 1000, + } + request_params = stream.retriever.request_params( + stream_state=state, stream_slice={"start_time": "2019-05-20T06:30:00Z", "end_time": "2021-12-31T16:00:00Z"} + ) + assert request_params == expected_params diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index 12c54ceaa2ad..6fc810db4ee9 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py index 8b20427c76b9..2c6a8538a5f0 100644 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ b/airbyte-integrations/connectors/source-sentry/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1.74", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml new file mode 100644 index 000000000000..7a69b16a37b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -0,0 +1,109 @@ +definitions: + page_size: 50 + schema_loader: + type: JsonSchema + file_path: "./source_sentry/schemas/{{ options.name }}.json" + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_pointer: [] + requester: + type: HttpRequester + name: "{{ options['name'] }}" + url_base: "https://{{ config.hostname }}/api/0/" + http_method: "GET" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.auth_token }}" + paginator: + type: LimitPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size: "*ref(definitions.page_size)" + limit_option: + inject_into: "request_parameter" + field_name: "" + page_token_option: + inject_into: "request_parameter" + field_name: "cursor" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ headers.link.next.cursor }}" + stop_condition: "{{ headers.link.next.results != 'true' }}" + retriever: + type: SimpleRetriever + name: "{{ options['name'] }}" + primary_key: "{{ options['primary_key'] }}" + +streams: + - type: DeclarativeStream + $options: + # https://docs.sentry.io/api/events/list-a-projects-events/ + name: "events" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/{{config.organization}}/{{config.project}}/events/" + request_options_provider: + request_parameters: + full: "true" + paginator: + $ref: "*ref(definitions.paginator)" + - type: DeclarativeStream + $options: + name: "issues" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/{{config.organization}}/{{config.project}}/issues/" + request_options_provider: + request_parameters: + statsPeriod: "" + query: "" + paginator: + $ref: "*ref(definitions.paginator)" + - type: DeclarativeStream + $options: + name: "projects" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/" + paginator: + $ref: "*ref(definitions.paginator)" + - type: DeclarativeStream + $options: + name: "project_detail" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/{{config.organization}}/{{config.project}}/" + paginator: + type: NoPagination +check: + type: CheckStream + stream_names: ["project_detail"] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/source.py b/airbyte-integrations/connectors/source-sentry/source_sentry/source.py index 398ec1274de4..5d68e9a6172d 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/source.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/source.py @@ -2,43 +2,16 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from .streams import Events, Issues, ProjectDetail, Projects - - -# Source -class SourceSentry(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, Any]: - try: - projects_stream = Projects( - authenticator=TokenAuthenticator(token=config["auth_token"]), - hostname=config.get("hostname"), - ) - next(projects_stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - stream_args = { - "authenticator": TokenAuthenticator(token=config["auth_token"]), - "hostname": config.get("hostname"), - } - project_stream_args = { - **stream_args, - "organization": config["organization"], - "project": config["project"], - } - return [ - Events(**project_stream_args), - Issues(**project_stream_args), - ProjectDetail(**project_stream_args), - Projects(**stream_args), - ] +# Declarative Source +class SourceSentry(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "./source_sentry/sentry.yaml"}) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json b/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json index a1d9e35c0a35..7820a6e6bcb3 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json @@ -5,7 +5,7 @@ "title": "Sentry Spec", "type": "object", "required": ["auth_token", "organization", "project"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "auth_token": { "type": "string", diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py deleted file mode 100644 index 4e0cb131cae5..000000000000 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ /dev/null @@ -1,158 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class SentryStream(HttpStream, ABC): - API_VERSION = "0" - URL_TEMPLATE = "https://{hostname}/api/{api_version}/" - primary_key = "id" - - def __init__(self, hostname: str, **kwargs): - super().__init__(**kwargs) - self._url_base = self.URL_TEMPLATE.format(hostname=hostname, api_version=self.API_VERSION) - - @property - def url_base(self) -> str: - return self._url_base - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - return {} - - -class SentryStreamPagination(SentryStream): - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Expect the link header field to always contain the values ​​for `rel`, `results`, and `cursor`. - If there is actually the next page, rel="next"; results="true"; cursor="". - """ - if response.links["next"]["results"] == "true": - return {"cursor": response.links["next"]["cursor"]} - else: - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - -class Events(SentryStreamPagination): - """ - Docs: https://docs.sentry.io/api/events/list-a-projects-events/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/events/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"full": "true"}) - - return params - - -class Issues(SentryStreamPagination): - """ - Docs: https://docs.sentry.io/api/events/list-a-projects-issues/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/issues/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"statsPeriod": "", "query": ""}) - - return params - - -class Projects(SentryStreamPagination): - """ - Docs: https://docs.sentry.io/api/projects/list-your-projects/ - """ - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return "projects/" - - -class ProjectDetail(SentryStream): - """ - Docs: https://docs.sentry.io/api/projects/retrieve-a-project/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield response.json() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py deleted file mode 100644 index 2d81d29cea0d..000000000000 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_sentry.source import SourceSentry -from source_sentry.streams import Projects - - -def test_check_connection(mocker): - source = SourceSentry() - logger_mock, config_mock = MagicMock(), MagicMock() - mocker.patch.object(Projects, "read_records", return_value=iter([{"id": "1", "name": "test"}])) - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceSentry() - config_mock = MagicMock() - config_mock["auth_token"] = "test-token" - config_mock["organization"] = "test-organization" - config_mock["project"] = "test-project" - streams = source.streams(config_mock) - expected_streams_number = 4 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py deleted file mode 100644 index cfd5ef76a10c..000000000000 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ /dev/null @@ -1,110 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pytest -from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryStreamPagination - -INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(SentryStreamPagination, "path", "test_endpoint") - mocker.patch.object(SentryStreamPagination, "__abstractmethods__", set()) - - -def test_next_page_token(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - resp = MagicMock() - cursor = "next_page_num" - resp.links = {"next": {"results": "true", "cursor": cursor}} - inputs = {"response": resp} - expected_token = {"cursor": cursor} - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_is_none(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - resp = MagicMock() - resp.links = {"next": {"results": "false", "cursor": "no_next"}} - inputs = {"response": resp} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def next_page_token_inputs(): - links_headers = [ - {}, - {"next": {}}, - ] - responses = [MagicMock() for _ in links_headers] - for mock, header in zip(responses, links_headers): - mock.links = header - - return responses - - -@pytest.mark.parametrize("response", next_page_token_inputs()) -def test_next_page_token_raises(patch_base_class, response): - stream = SentryStreamPagination(hostname="sentry.io") - inputs = {"response": response} - with pytest.raises(KeyError): - stream.next_page_token(**inputs) - - -def test_events_path(): - stream = Events(**INIT_ARGS) - expected = "projects/test-org/test-project/events/" - assert stream.path() == expected - - -def test_issues_path(): - stream = Issues(**INIT_ARGS) - expected = "projects/test-org/test-project/issues/" - assert stream.path() == expected - - -def test_projects_path(): - stream = Projects(hostname="sentry.io") - expected = "projects/" - assert stream.path() == expected - - -def test_project_detail_path(): - stream = ProjectDetail(**INIT_ARGS) - expected = "projects/test-org/test-project/" - assert stream.path() == expected - - -def test_sentry_stream_pagination_request_params(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - expected = {"cursor": "next-page"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_events_request_params(): - stream = Events(**INIT_ARGS) - expected = {"cursor": "next-page", "full": "true"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_issues_request_params(): - stream = Issues(**INIT_ARGS) - expected = {"cursor": "next-page", "statsPeriod": "", "query": ""} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_projects_request_params(): - stream = Projects(hostname="sentry.io") - expected = {"cursor": "next-page"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_project_detail_request_params(): - stream = ProjectDetail(**INIT_ARGS) - expected = {} - assert stream.request_params(stream_state=None, next_page_token=None) == expected diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java index 7a555cc091d0..f91d0a2a6dea 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java @@ -111,11 +111,11 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 0dcc64e056e4..11d615ac8c77 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -4,13 +4,13 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* WORKDIR /airbyte/integration_code -COPY source_stripe ./source_stripe -COPY main.py ./ COPY setup.py ./ RUN pip install . +COPY source_stripe ./source_stripe +COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.35 +LABEL io.airbyte.version=0.1.36 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json index 168cd531425f..aac5fd711ab0 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json @@ -1,20 +1,20 @@ { - "charges": { "created": 161703040300 }, - "coupons": { "created": 161703040300 }, - "events": { "created": 161749384700 }, - "customers": { "created": 160083796900 }, - "plans": { "created": 159484835000 }, - "invoices": { "created": 161749017500 }, - "invoice_items": { "date": 159494698100 }, - "transfers": { "created": 161099582400 }, - "subscriptions": { "created": 159968687300 }, - "balance_transactions": { "created": 161706755600 }, - "payouts": { "created": 161706755600 }, - "disputes": { "created": 161099630500 }, - "products": { "created": 158551134100 }, - "refunds": { "created": 161959562900 }, - "payment_intents": { "created": 161959562900 }, - "promotion_codes": { "created": 163534157100 }, + "charges": { "created": 10000000000 }, + "coupons": { "created": 10000000000 }, + "events": { "created": 10000000000 }, + "customers": { "created": 10000000000 }, + "plans": { "created": 10000000000 }, + "invoices": { "created": 10000000000 }, + "invoice_items": { "date": 10000000000 }, + "transfers": { "created": 10000000000 }, + "subscriptions": { "created": 10000000000 }, + "balance_transactions": { "created": 10000000000 }, + "payouts": { "created": 10000000000 }, + "disputes": { "created": 10000000000 }, + "products": { "created": 10000000000 }, + "refunds": { "created": 10000000000 }, + "payment_intents": { "created": 10000000000 }, + "promotion_codes": { "created": 10000000000 }, "checkout_sessions": { "expires_at": 10000000000 }, "checkout_sessions_line_items": { "checkout_session_expires_at": 10000000000 } } diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/test_dummy.py b/airbyte-integrations/connectors/source-stripe/integration_tests/test_dummy.py deleted file mode 100644 index f1f977513d63..000000000000 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/test_dummy.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -def test_dummy(): - """ - Dummy test to prevent gradle from failing test for this connector - """ - assert True diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index 0b8de104dc9c..492dfc4e5c92 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "stripe==2.56.0", "pendulum==1.2.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "stripe==2.56.0", "pendulum==2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index a3a8835627a4..1d5dd91396b8 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -51,7 +51,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["client_secret"]) start_date = pendulum.parse(config["start_date"]).int_timestamp - args = {"authenticator": authenticator, "account_id": config["account_id"], "start_date": start_date} + args = { + "authenticator": authenticator, + "account_id": config["account_id"], + "start_date": start_date, + "slice_range": config.get("slice_range"), + } incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} return [ BalanceTransactions(**incremental_args), diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml index 20c01267898d..1baa105033d2 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -44,3 +44,14 @@ connectionSpecification: after creation. More info here order: 3 + slice_range: + type: integer + title: Data request time increment in days (Optional) + default: 365 + minimum: 1 + examples: [1, 3, 10, 30, 180, 360] + description: >- + The time increment used by the connector when requesting data from the Stripe API. The bigger the value is, + the less requests will be made and faster the sync will be. On the other hand, the more seldom + the state is persisted. + order: 4 diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 5fc5e1efd2c3..a6023fb37dac 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -5,7 +5,7 @@ import math from abc import ABC, abstractmethod from itertools import chain -from typing import Any, Iterable, Mapping, MutableMapping, Optional +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import pendulum import requests @@ -16,11 +16,13 @@ class StripeStream(HttpStream, ABC): url_base = "https://api.stripe.com/v1/" primary_key = "id" + DEFAULT_SLICE_RANGE = 365 - def __init__(self, start_date: int, account_id: str, **kwargs): + def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_SLICE_RANGE, **kwargs): super().__init__(**kwargs) self.account_id = account_id self.start_date = start_date + self.slice_range = slice_range or self.DEFAULT_SLICE_RANGE def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() @@ -37,6 +39,9 @@ def request_params( # Stripe default pagination is 10, max is 100 params = {"limit": 100} + for key in ("created[gte]", "created[lte]"): + if key in stream_slice: + params[key] = stream_slice[key] # Handle pagination by inserting the next page's token in the request parameters if next_page_token: @@ -54,6 +59,28 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp response_json = response.json() yield from response_json.get("data", []) # Stripe puts records in a container array "data" + def chunk_dates(self, start_date_ts: int) -> Iterable[Tuple[int, int]]: + now = pendulum.now().int_timestamp + step = int(pendulum.duration(days=self.slice_range).total_seconds()) + after_ts = start_date_ts + while after_ts < now: + before_ts = min(now, after_ts + step) + yield after_ts, before_ts + after_ts = before_ts + 1 + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for start, end in self.chunk_dates(self.start_date): + yield {"created[gte]": start, "created[lte]": end} + + +class SingleEmptySliceMixin(object): + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + return [{}] + class IncrementalStripeStream(StripeStream, ABC): # Stripe returns most recently created objects first, so we don't want to persist state until the entire stream has been read @@ -79,14 +106,27 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ return {self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(self.cursor_field, 0))} - def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): - stream_state = stream_state or {} - params = super().request_params(stream_state=stream_state, **kwargs) - - start_timestamp = self.get_start_timestamp(stream_state) - if start_timestamp: - params["created[gte]"] = start_timestamp - return params + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + if stream_slice is None: + return [] + yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + start_ts = self.get_start_timestamp(stream_state) + if start_ts >= pendulum.now().int_timestamp: + # if the state is in the future - this will produce a state message but not make an API request + yield None + else: + for start, end in self.chunk_dates(start_ts): + yield {"created[gte]": start, "created[lte]": end} def get_start_timestamp(self, stream_state) -> int: start_point = self.start_date @@ -134,7 +174,7 @@ def path(self, **kwargs) -> str: return "charges" -class CustomerBalanceTransactions(StripeStream): +class CustomerBalanceTransactions(SingleEmptySliceMixin, StripeStream): """ API docs: https://stripe.com/docs/api/customer_balance_transactions/list """ @@ -147,8 +187,10 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: customers_stream = Customers(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) - for customer in customers_stream.read_records(sync_mode=SyncMode.full_refresh): - yield from super().read_records(stream_slice={"customer_id": customer["id"]}, **kwargs) + slices = customers_stream.stream_slices(sync_mode=SyncMode.full_refresh) + for _slice in slices: + for customer in customers_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): + yield from super().read_records(stream_slice={"customer_id": customer["id"]}, **kwargs) class Coupons(IncrementalStripeStream): @@ -184,7 +226,7 @@ def path(self, **kwargs): return "events" -class StripeSubStream(StripeStream, ABC): +class StripeSubStream(SingleEmptySliceMixin, StripeStream, ABC): """ Research shows that records related to SubStream can be extracted from Parent streams which already contain 1st page of needed items. Thus, it significantly decreases a number of requests needed to get @@ -261,31 +303,32 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): return params def read_records(self, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - parent_stream = self.parent(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) - for record in parent_stream.read_records(sync_mode=SyncMode.full_refresh): + slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) + for _slice in slices: + for record in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): - items_obj = record.get(self.sub_items_attr, {}) - if not items_obj: - continue + items_obj = record.get(self.sub_items_attr, {}) + if not items_obj: + continue - items = items_obj.get("data", []) + items = items_obj.get("data", []) - # non-generic filter, mainly for BankAccounts stream only - if self.filter: - items = [i for i in items if i.get(self.filter["attr"]) == self.filter["value"]] + # non-generic filter, mainly for BankAccounts stream only + if self.filter: + items = [i for i in items if i.get(self.filter["attr"]) == self.filter["value"]] - # get next pages - items_next_pages = [] - if items_obj.get("has_more") and items: - stream_slice = {self.parent_id: record["id"], "starting_after": items[-1]["id"]} - items_next_pages = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, **kwargs) + # get next pages + items_next_pages = [] + if items_obj.get("has_more") and items: + stream_slice = {self.parent_id: record["id"], "starting_after": items[-1]["id"]} + items_next_pages = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, **kwargs) - for item in chain(items, items_next_pages): - if self.add_parent_id: - # add reference to parent object when item doesn't have it already - item[self.parent_id] = record["id"] - yield item + for item in chain(items, items_next_pages): + if self.add_parent_id: + # add reference to parent object when item doesn't have it already + item[self.parent_id] = record["id"] + yield item class Invoices(IncrementalStripeStream): @@ -447,12 +490,12 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): return f"customers/{stream_slice[self.parent_id]}/sources" def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) + params = super().request_params(stream_slice=stream_slice, **kwargs) params["object"] = "bank_account" return params -class CheckoutSessions(IncrementalStripeStream): +class CheckoutSessions(SingleEmptySliceMixin, IncrementalStripeStream): """ API docs: https://stripe.com/docs/api/checkout/sessions/list """ @@ -472,12 +515,6 @@ def __init__(self, **kwargs): def path(self, **kwargs): return "checkout/sessions" - def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_state=stream_state, **kwargs) - # remove odd param, not supported by checkout_sessions api - params.pop("created[gte]", None) - return params - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: since_date = self.get_start_timestamp(stream_state) for item in super().parse_response(response, **kwargs): @@ -487,7 +524,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield item -class CheckoutSessionsLineItems(IncrementalStripeStream): +class CheckoutSessionsLineItems(SingleEmptySliceMixin, IncrementalStripeStream): """ API docs: https://stripe.com/docs/api/checkout/sessions/line_items """ @@ -516,7 +553,9 @@ def read_records( if stream_state: checkout_session_state = {"expires_at": stream_state["checkout_session_expires_at"]} - for checkout_session in checkout_session_stream.read_records(sync_mode=SyncMode.full_refresh, stream_state=checkout_session_state): + for checkout_session in checkout_session_stream.read_records( + sync_mode=SyncMode.full_refresh, stream_state=checkout_session_state, stream_slice={} + ): stream_slice = { "checkout_session_id": checkout_session["id"], "expires_at": checkout_session["expires_at"], @@ -525,10 +564,6 @@ def read_records( def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): params = super().request_params(stream_slice=stream_slice, **kwargs) - - # remove odd param, not supported by checkout_sessions api - params.pop("created[gte]", None) - params["expand[]"] = ["data.discounts", "data.taxes"] return params @@ -545,7 +580,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, response_json = response.json() data = response_json.get("data", []) if data and stream_slice: - print(f"stream_slice: {stream_slice}") + self.logger.info(f"stream_slice: {stream_slice}") cs_id = stream_slice.get("checkout_session_id", None) cs_expires_at = stream_slice.get("expires_at", None) for e in data: @@ -565,7 +600,7 @@ def path(self, **kwargs): return "promotion_codes" -class ExternalAccount(StripeStream, ABC): +class ExternalAccount(SingleEmptySliceMixin, StripeStream, ABC): """ Bank Accounts and Cards are separate streams because they have different schemas """ diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index dfc6bacd71b8..b365b66c4f3f 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import pendulum import pytest from airbyte_cdk.models import SyncMode from source_stripe.streams import ( @@ -128,8 +129,9 @@ def test_sub_stream(requests_mock): "url": "/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", }, ) - - stream = InvoiceLineItems(start_date=1641008947, account_id="None") + # make start date a recent date so there's just one slice in a parent stream + start_date = pendulum.today().subtract(days=3).int_timestamp + stream = InvoiceLineItems(start_date=start_date, account_id="None") records = stream.read_records(sync_mode=SyncMode.full_refresh) assert list(records) == [ {"id": "il_1", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, @@ -140,7 +142,7 @@ def test_sub_stream(requests_mock): @pytest.fixture(name="config") def config_fixture(): - config = {"authenticator": "authenticator", "account_id": "", "start_date": 1652783086} + config = {"authenticator": "authenticator", "account_id": "", "start_date": 1596466368} return config @@ -184,16 +186,32 @@ def test_path( @pytest.mark.parametrize( "stream, kwargs, expected", [ - (CustomerBalanceTransactions, {"stream_state": {}}, {"limit": 100}), - (Customers, {}, {"created[gte]": 1652783086, "limit": 100}), + ( + CustomerBalanceTransactions, + {"stream_state": {}, "stream_slice": {"created[gte]": 1596466368, "created[lte]": 1596552768}}, + {"limit": 100, "created[gte]": 1596466368, "created[lte]": 1596552768}, + ), + ( + Customers, + {"stream_state": {}, "stream_slice": {"created[gte]": 1596466368, "created[lte]": 1596552768}}, + {"created[gte]": 1596466368, "created[lte]": 1596552768, "limit": 100}, + ), (InvoiceLineItems, {"stream_state": {}, "stream_slice": {"starting_after": "2030"}}, {"limit": 100, "starting_after": "2030"}), - (Subscriptions, {}, {"created[gte]": 1652783086, "limit": 100, "status": "all"}), + ( + Subscriptions, + {"stream_slice": {"created[gte]": 1596466368, "created[lte]": 1596552768}}, + {"created[gte]": 1596466368, "limit": 100, "status": "all", "created[lte]": 1596552768}, + ), (SubscriptionItems, {"stream_state": {}, "stream_slice": {"subscription_id": "SI"}}, {"limit": 100, "subscription": "SI"}), (BankAccounts, {"stream_state": {}, "stream_slice": {"subscription_id": "SI"}}, {"limit": 100, "object": "bank_account"}), - (CheckoutSessions, {"stream_state": None}, {"limit": 100}), - (CheckoutSessionsLineItems, {"stream_state": None}, {"limit": 100, "expand[]": ["data.discounts", "data.taxes"]}), - (ExternalAccountBankAccounts, {"stream_state": None}, {"limit": 100, "object": "bank_account"}), - (ExternalAccountCards, {"stream_state": None}, {"limit": 100, "object": "card"}), + (CheckoutSessions, {"stream_state": None, "stream_slice": {}}, {"limit": 100}), + ( + CheckoutSessionsLineItems, + {"stream_state": None, "stream_slice": {}}, + {"limit": 100, "expand[]": ["data.discounts", "data.taxes"]}, + ), + (ExternalAccountBankAccounts, {"stream_state": None, "stream_slice": {}}, {"limit": 100, "object": "bank_account"}), + (ExternalAccountCards, {"stream_state": None, "stream_slice": {}}, {"limit": 100, "object": "card"}), ], ) def test_request_params( diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json index 5b3c31bd6697..d280b4f63f82 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json @@ -10,4 +10,4 @@ "submitted_at": 9999999999 } } -} \ No newline at end of file +} diff --git a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java index 1411eb29bc81..67c75cce479a 100644 --- a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java +++ b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java @@ -52,7 +52,7 @@ public enum ToEmit { NUM_ABNORMAL_SCHEDULED_SYNCS(countMetricEmission(() -> { final var count = ReporterApp.configDatabase.query(MetricQueries::numOfJobsNotRunningOnSchedule); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.NUM_ABNORMAL_SCHEDULED_SYNCS, count); - })), + }), 1, TimeUnit.HOURS), OVERALL_JOB_RUNTIME_IN_LAST_HOUR_BY_TERMINAL_STATE_SECS(countMetricEmission(() -> { final var times = ReporterApp.configDatabase.query(MetricQueries::overallJobRuntimeForTerminalJobsInLastHour); for (Pair pair : times) { diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_tracker/TrackingMetadata.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_tracker/TrackingMetadata.java index 7bbe227b19c0..42db33fca844 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_tracker/TrackingMetadata.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_tracker/TrackingMetadata.java @@ -109,6 +109,8 @@ public static ImmutableMap generateJobAttemptMetadata(final Job metadata.put("duration", Math.round((syncSummary.getEndTime() - syncSummary.getStartTime()) / 1000.0)); metadata.put("volume_mb", syncSummary.getBytesSynced()); metadata.put("volume_rows", syncSummary.getRecordsSynced()); + metadata.put("count_state_messages_from_source", syncSummary.getTotalStats().getSourceStateMessagesEmitted()); + metadata.put("count_state_messages_from_destination", syncSummary.getTotalStats().getDestinationStateMessagesEmitted()); } } diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java index 970efef1c300..56f5347677af 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java @@ -37,6 +37,7 @@ import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SyncStats; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.CatalogHelpers; @@ -111,6 +112,8 @@ class JobTrackerTest { .put("duration", SYNC_DURATION) .put("volume_rows", SYNC_RECORDS_SYNC) .put("volume_mb", SYNC_BYTES_SYNC) + .put("count_state_messages_from_source", 3L) + .put("count_state_messages_from_destination", 1L) .build(); private static final ImmutableMap SYNC_CONFIG_METADATA = ImmutableMap.builder() .put(JobTracker.CONFIG + ".source.key", JobTracker.SET) @@ -481,14 +484,18 @@ private Attempt getAttemptMock() { final JobOutput jobOutput = mock(JobOutput.class); final StandardSyncOutput syncOutput = mock(StandardSyncOutput.class); final StandardSyncSummary syncSummary = mock(StandardSyncSummary.class); + final SyncStats syncStats = mock(SyncStats.class); when(syncSummary.getStartTime()).thenReturn(SYNC_START_TIME); when(syncSummary.getEndTime()).thenReturn(SYNC_END_TIME); when(syncSummary.getBytesSynced()).thenReturn(SYNC_BYTES_SYNC); when(syncSummary.getRecordsSynced()).thenReturn(SYNC_RECORDS_SYNC); when(syncOutput.getStandardSyncSummary()).thenReturn(syncSummary); + when(syncSummary.getTotalStats()).thenReturn(syncStats); when(jobOutput.getSync()).thenReturn(syncOutput); when(attempt.getOutput()).thenReturn(java.util.Optional.of(jobOutput)); + when(syncStats.getSourceStateMessagesEmitted()).thenReturn(3L); + when(syncStats.getDestinationStateMessagesEmitted()).thenReturn(1L); return attempt; } diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 55ade6948821..cf47ced29cfe 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -32,6 +32,7 @@ dependencies { implementation 'org.glassfish.jersey.inject:jersey-hk2' implementation 'org.glassfish.jersey.media:jersey-media-json-jackson' implementation 'org.glassfish.jersey.ext:jersey-bean-validation' + implementation 'org.quartz-scheduler:quartz:2.3.2' testImplementation project(':airbyte-test-utils') diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java index 0ce5af388239..989f11d40420 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java @@ -4,9 +4,13 @@ package io.airbyte.server.converters; +import io.airbyte.api.client.model.generated.ConnectionScheduleType; import io.airbyte.api.model.generated.ActorDefinitionResourceRequirements; import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionSchedule; +import io.airbyte.api.model.generated.ConnectionScheduleData; +import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule; +import io.airbyte.api.model.generated.ConnectionScheduleDataCron; import io.airbyte.api.model.generated.ConnectionStatus; import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.JobType; @@ -17,7 +21,10 @@ import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; import io.airbyte.config.Schedule; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.ScheduleType; import io.airbyte.server.handlers.helpers.CatalogConverter; +import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper; +import io.airbyte.validation.json.JsonValidationException; import java.util.stream.Collectors; public class ApiPojoConverters { @@ -78,7 +85,7 @@ public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.co .memoryLimit(resourceReqs.getMemoryLimit()); } - public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) { + public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) throws JsonValidationException { final StandardSync newConnection = new StandardSync() .withNamespaceDefinition(Enums.convertTo(update.getNamespaceDefinition(), NamespaceDefinitionType.class)) @@ -99,7 +106,9 @@ public static io.airbyte.config.StandardSync connectionUpdateToInternal(final Co } // update sync schedule - if (update.getSchedule() != null) { + if (update.getScheduleType() != null) { + ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(newConnection, update.getScheduleType(), update.getScheduleData()); + } else if (update.getSchedule() != null) { final Schedule newSchedule = new Schedule() .withTimeUnit(toPersistenceTimeUnit(update.getSchedule().getTimeUnit())) .withUnits(update.getSchedule().getUnits()); @@ -112,21 +121,12 @@ public static io.airbyte.config.StandardSync connectionUpdateToInternal(final Co } public static ConnectionRead internalToConnectionRead(final StandardSync standardSync) { - ConnectionSchedule apiSchedule = null; - - if (!standardSync.getManual()) { - apiSchedule = new ConnectionSchedule() - .timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit())) - .units(standardSync.getSchedule().getUnits()); - } - final ConnectionRead connectionRead = new ConnectionRead() .connectionId(standardSync.getConnectionId()) .sourceId(standardSync.getSourceId()) .destinationId(standardSync.getDestinationId()) .operationIds(standardSync.getOperationIds()) .status(toApiStatus(standardSync.getStatus())) - .schedule(apiSchedule) .name(standardSync.getName()) .namespaceDefinition(Enums.convertTo(standardSync.getNamespaceDefinition(), io.airbyte.api.model.generated.NamespaceDefinitionType.class)) .namespaceFormat(standardSync.getNamespaceFormat()) @@ -138,6 +138,8 @@ public static ConnectionRead internalToConnectionRead(final StandardSync standar connectionRead.resourceRequirements(resourceRequirementsToApi(standardSync.getResourceRequirements())); } + populateConnectionReadSchedule(standardSync, connectionRead); + return connectionRead; } @@ -149,10 +151,15 @@ public static io.airbyte.config.JobTypeResourceLimit.JobType toInternalJobType(f return Enums.convertTo(jobType, io.airbyte.config.JobTypeResourceLimit.JobType.class); } + // TODO(https://github.com/airbytehq/airbyte/issues/11432): remove these helpers. public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final Schedule.TimeUnit apiTimeUnit) { return Enums.convertTo(apiTimeUnit, ConnectionSchedule.TimeUnitEnum.class); } + public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final BasicSchedule.TimeUnit timeUnit) { + return Enums.convertTo(timeUnit, ConnectionSchedule.TimeUnitEnum.class); + } + public static ConnectionStatus toApiStatus(final StandardSync.Status status) { return Enums.convertTo(status, ConnectionStatus.class); } @@ -169,4 +176,75 @@ public static BasicSchedule.TimeUnit toBasicScheduleTimeUnit(final ConnectionSch return Enums.convertTo(apiTimeUnit, BasicSchedule.TimeUnit.class); } + public static BasicSchedule.TimeUnit toBasicScheduleTimeUnit(final ConnectionScheduleDataBasicSchedule.TimeUnitEnum apiTimeUnit) { + return Enums.convertTo(apiTimeUnit, BasicSchedule.TimeUnit.class); + } + + public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicScheduleTimeUnit(final BasicSchedule.TimeUnit timeUnit) { + return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class); + } + + public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicScheduleTimeUnit(final Schedule.TimeUnit timeUnit) { + return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class); + } + + public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) { + // TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is + // ready. + if (standardSync.getScheduleType() != null) { + // Populate everything based on the new schema. + switch (standardSync.getScheduleType()) { + case MANUAL -> { + connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL); + } + case BASIC_SCHEDULE -> { + connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC); + connectionRead.scheduleData(new ConnectionScheduleData() + .basicSchedule(new ConnectionScheduleDataBasicSchedule() + .timeUnit(toApiBasicScheduleTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit())) + .units(standardSync.getScheduleData().getBasicSchedule().getUnits()))); + connectionRead.schedule(new ConnectionSchedule() + .timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit())) + .units(standardSync.getScheduleData().getBasicSchedule().getUnits())); + } + case CRON -> { + // We don't populate any legacy data here. + connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.CRON); + connectionRead.scheduleData(new ConnectionScheduleData() + .cron(new ConnectionScheduleDataCron() + .cronExpression(standardSync.getScheduleData().getCron().getCronExpression()) + .cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone()))); + } + } + } else if (standardSync.getManual()) { + // Legacy schema, manual sync. + connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL); + } else { + // Legacy schema, basic schedule. + connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC); + connectionRead.schedule(new ConnectionSchedule() + .timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit())) + .units(standardSync.getSchedule().getUnits())); + connectionRead.scheduleData(new ConnectionScheduleData() + .basicSchedule(new ConnectionScheduleDataBasicSchedule() + .timeUnit(toApiBasicScheduleTimeUnit(standardSync.getSchedule().getTimeUnit())) + .units(standardSync.getSchedule().getUnits()))); + } + } + + public static ConnectionScheduleType toApiScheduleType(final ScheduleType scheduleType) { + switch (scheduleType) { + case MANUAL -> { + return ConnectionScheduleType.MANUAL; + } + case BASIC_SCHEDULE -> { + return ConnectionScheduleType.BASIC; + } + case CRON -> { + return ConnectionScheduleType.CRON; + } + } + throw new RuntimeException("Unexpected schedule type"); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index 4df3a51331b2..b431dea76cd1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -158,7 +158,7 @@ private static AttemptStats getTotalAttemptStats(final Attempt attempt) { return new AttemptStats() .bytesEmitted(totalStats.getBytesEmitted()) .recordsEmitted(totalStats.getRecordsEmitted()) - .stateMessagesEmitted(totalStats.getStateMessagesEmitted()) + .stateMessagesEmitted(totalStats.getSourceStateMessagesEmitted()) .recordsCommitted(totalStats.getRecordsCommitted()); } @@ -175,7 +175,7 @@ private static List getAttemptStreamStats(final Attempt atte .stats(new AttemptStats() .bytesEmitted(streamStat.getStats().getBytesEmitted()) .recordsEmitted(streamStat.getStats().getRecordsEmitted()) - .stateMessagesEmitted(streamStat.getStats().getStateMessagesEmitted()) + .stateMessagesEmitted(streamStat.getStats().getSourceStateMessagesEmitted()) .recordsCommitted(streamStat.getStats().getRecordsCommitted()))) .collect(Collectors.toList()); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index 691a3546a976..4e965afb534a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -47,6 +47,7 @@ import io.airbyte.server.converters.CatalogDiffConverters; import io.airbyte.server.handlers.helpers.CatalogConverter; import io.airbyte.server.handlers.helpers.ConnectionMatcher; +import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper; import io.airbyte.server.handlers.helpers.DestinationMatcher; import io.airbyte.server.handlers.helpers.SourceMatcher; import io.airbyte.validation.json.JsonValidationException; @@ -140,6 +141,34 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) standardSync.withCatalog(new ConfiguredAirbyteCatalog().withStreams(Collections.emptyList())); } + if (connectionCreate.getSchedule() != null && connectionCreate.getScheduleType() != null) { + throw new JsonValidationException("supply old or new schedule schema but not both"); + } + + if (connectionCreate.getScheduleType() != null) { + ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(standardSync, connectionCreate.getScheduleType(), + connectionCreate.getScheduleData()); + } else { + populateSyncFromLegacySchedule(standardSync, connectionCreate); + } + + configRepository.writeStandardSync(standardSync); + + trackNewConnection(standardSync); + + try { + LOGGER.info("Starting a connection manager workflow"); + eventRunner.createConnectionManagerWorkflow(connectionId); + } catch (final Exception e) { + LOGGER.error("Start of the connection manager workflow failed", e); + configRepository.deleteStandardSyncDefinition(standardSync.getConnectionId()); + throw e; + } + + return buildConnectionRead(connectionId); + } + + private void populateSyncFromLegacySchedule(final StandardSync standardSync, final ConnectionCreate connectionCreate) { if (connectionCreate.getSchedule() != null) { final Schedule schedule = new Schedule() .withTimeUnit(ApiPojoConverters.toPersistenceTimeUnit(connectionCreate.getSchedule().getTimeUnit())) @@ -159,21 +188,6 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) standardSync.withManual(true); standardSync.withScheduleType(ScheduleType.MANUAL); } - - configRepository.writeStandardSync(standardSync); - - trackNewConnection(standardSync); - - try { - LOGGER.info("Starting a connection manager workflow"); - eventRunner.createConnectionManagerWorkflow(connectionId); - } catch (final Exception e) { - LOGGER.error("Start of the connection manager workflow failed", e); - configRepository.deleteStandardSyncDefinition(standardSync.getConnectionId()); - throw e; - } - - return buildConnectionRead(connectionId); } private void trackNewConnection(final StandardSync standardSync) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index cad89aaf331c..57632ca9eb9e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -177,6 +177,8 @@ private static WebBackendConnectionRead getWebBackendConnectionRead(final Connec .syncCatalog(connectionRead.getSyncCatalog()) .status(connectionRead.getStatus()) .schedule(connectionRead.getSchedule()) + .scheduleType(connectionRead.getScheduleType()) + .scheduleData(connectionRead.getScheduleData()) .source(source) .destination(destination) .operations(operations.getOperations()) @@ -495,6 +497,8 @@ protected static ConnectionCreate toConnectionCreate(final WebBackendConnectionC connectionCreate.operationIds(operationIds); connectionCreate.syncCatalog(webBackendConnectionCreate.getSyncCatalog()); connectionCreate.schedule(webBackendConnectionCreate.getSchedule()); + connectionCreate.scheduleType(webBackendConnectionCreate.getScheduleType()); + connectionCreate.scheduleData(webBackendConnectionCreate.getScheduleData()); connectionCreate.status(webBackendConnectionCreate.getStatus()); connectionCreate.resourceRequirements(webBackendConnectionCreate.getResourceRequirements()); connectionCreate.sourceCatalogId(webBackendConnectionCreate.getSourceCatalogId()); @@ -514,6 +518,8 @@ protected static ConnectionUpdate toConnectionUpdate(final WebBackendConnectionU connectionUpdate.operationIds(operationIds); connectionUpdate.syncCatalog(webBackendConnectionUpdate.getSyncCatalog()); connectionUpdate.schedule(webBackendConnectionUpdate.getSchedule()); + connectionUpdate.scheduleType(webBackendConnectionUpdate.getScheduleType()); + connectionUpdate.scheduleData(webBackendConnectionUpdate.getScheduleData()); connectionUpdate.status(webBackendConnectionUpdate.getStatus()); connectionUpdate.resourceRequirements(webBackendConnectionUpdate.getResourceRequirements()); connectionUpdate.sourceCatalogId(webBackendConnectionUpdate.getSourceCatalogId()); @@ -534,6 +540,8 @@ protected static ConnectionSearch toConnectionSearch(final WebBackendConnectionS .namespaceFormat(webBackendConnectionSearch.getNamespaceFormat()) .prefix(webBackendConnectionSearch.getPrefix()) .schedule(webBackendConnectionSearch.getSchedule()) + .scheduleType(webBackendConnectionSearch.getScheduleType()) + .scheduleData(webBackendConnectionSearch.getScheduleData()) .status(webBackendConnectionSearch.getStatus()); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionMatcher.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionMatcher.java index 5890f665b5af..163132502cda 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionMatcher.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionMatcher.java @@ -33,6 +33,8 @@ public ConnectionRead match(final ConnectionRead query) { search.getNamespaceDefinition() == null ? query.getNamespaceDefinition() : search.getNamespaceDefinition()); fromSearch.prefix(Strings.isBlank(search.getPrefix()) ? query.getPrefix() : search.getPrefix()); fromSearch.schedule(search.getSchedule() == null ? query.getSchedule() : search.getSchedule()); + fromSearch.scheduleType(search.getScheduleType() == null ? query.getScheduleType() : search.getScheduleType()); + fromSearch.scheduleData(search.getScheduleData() == null ? query.getScheduleData() : search.getScheduleData()); fromSearch.sourceId(search.getSourceId() == null ? query.getSourceId() : search.getSourceId()); fromSearch.status(search.getStatus() == null ? query.getStatus() : search.getStatus()); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionScheduleHelper.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionScheduleHelper.java new file mode 100644 index 000000000000..62fdb540d9fd --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionScheduleHelper.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.handlers.helpers; + +import io.airbyte.api.model.generated.ConnectionScheduleData; +import io.airbyte.api.model.generated.ConnectionScheduleType; +import io.airbyte.config.BasicSchedule; +import io.airbyte.config.Cron; +import io.airbyte.config.ScheduleData; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.ScheduleType; +import io.airbyte.server.converters.ApiPojoConverters; +import io.airbyte.validation.json.JsonValidationException; +import java.text.ParseException; +import java.util.TimeZone; +import org.joda.time.DateTimeZone; +import org.quartz.CronExpression; + +/** + * Helper class to handle connection schedules, including validation and translating between API and + * config. + */ +public class ConnectionScheduleHelper { + + public static void populateSyncFromScheduleTypeAndData(final StandardSync standardSync, + final ConnectionScheduleType scheduleType, + final ConnectionScheduleData scheduleData) + throws JsonValidationException { + if (scheduleType != ConnectionScheduleType.MANUAL && scheduleData == null) { + throw new JsonValidationException("schedule data must be populated if schedule type is populated"); + } + switch (scheduleType) { + // NOTE: the `manual` column is marked required, so we populate it until it's removed. + case MANUAL -> standardSync.withScheduleType(ScheduleType.MANUAL).withManual(true); + case BASIC -> { + if (scheduleData.getBasicSchedule() == null) { + throw new JsonValidationException("if schedule type is basic, then scheduleData.basic must be populated"); + } + standardSync + .withScheduleType(ScheduleType.BASIC_SCHEDULE) + .withScheduleData(new ScheduleData().withBasicSchedule( + new BasicSchedule().withTimeUnit(ApiPojoConverters.toBasicScheduleTimeUnit(scheduleData.getBasicSchedule().getTimeUnit())) + .withUnits(scheduleData.getBasicSchedule().getUnits()))) + .withManual(false); + } + case CRON -> { + if (scheduleData.getCron() == null) { + throw new JsonValidationException("if schedule type is cron, then scheduleData.cron must be populated"); + } + // Validate that this is a valid cron expression and timezone. + final String cronExpression = scheduleData.getCron().getCronExpression(); + final String cronTimeZone = scheduleData.getCron().getCronTimeZone(); + if (cronExpression == null || cronTimeZone == null) { + throw new JsonValidationException("Cron expression and timezone are required"); + } + if (cronTimeZone.toLowerCase().startsWith("etc")) { + throw new JsonValidationException("Etc/ timezones are unsupported"); + } + try { + final TimeZone timeZone = DateTimeZone.forID(cronTimeZone).toTimeZone(); + final CronExpression parsedCronExpression = new CronExpression(cronExpression); + parsedCronExpression.setTimeZone(timeZone); + } catch (ParseException e) { + throw (JsonValidationException) new JsonValidationException("invalid cron expression").initCause(e); + } catch (IllegalArgumentException e) { + throw (JsonValidationException) new JsonValidationException("invalid cron timezone").initCause(e); + } + standardSync + .withScheduleType(ScheduleType.CRON) + .withScheduleData(new ScheduleData().withCron(new Cron() + .withCronExpression(cronExpression) + .withCronTimeZone(cronTimeZone))) + .withManual(false); + } + } + } + +} diff --git a/airbyte-server/src/test/java/io/airbyte/server/converters/JobConverterTest.java b/airbyte-server/src/test/java/io/airbyte/server/converters/JobConverterTest.java index 5e37872494f5..047e5e65cdee 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/converters/JobConverterTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/converters/JobConverterTest.java @@ -99,14 +99,14 @@ class JobConverterTest { .withTotalStats(new SyncStats() .withRecordsEmitted(RECORDS_EMITTED) .withBytesEmitted(BYTES_EMITTED) - .withStateMessagesEmitted(STATE_MESSAGES_EMITTED) + .withSourceStateMessagesEmitted(STATE_MESSAGES_EMITTED) .withRecordsCommitted(RECORDS_COMMITTED)) .withStreamStats(Lists.newArrayList(new StreamSyncStats() .withStreamName(STREAM_NAME) .withStats(new SyncStats() .withRecordsEmitted(RECORDS_EMITTED) .withBytesEmitted(BYTES_EMITTED) - .withStateMessagesEmitted(STATE_MESSAGES_EMITTED) + .withSourceStateMessagesEmitted(STATE_MESSAGES_EMITTED) .withRecordsCommitted(RECORDS_COMMITTED)))))); private JobConverter jobConverter; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java new file mode 100644 index 000000000000..f1f4291eaaff --- /dev/null +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.handlers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.api.model.generated.ConnectionScheduleData; +import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule; +import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule.TimeUnitEnum; +import io.airbyte.api.model.generated.ConnectionScheduleDataCron; +import io.airbyte.api.model.generated.ConnectionScheduleType; +import io.airbyte.config.BasicSchedule.TimeUnit; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.ScheduleType; +import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper; +import io.airbyte.validation.json.JsonValidationException; +import org.junit.jupiter.api.Test; + +class ConnectionSchedulerHelperTest { + + private final static String EXPECTED_CRON_TIMEZONE = "UTC"; + private final static String EXPECTED_CRON_EXPRESSION = "* */2 * * * ?"; + + @Test + void testPopulateSyncScheduleFromManualType() throws JsonValidationException { + final StandardSync actual = new StandardSync(); + ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, + ConnectionScheduleType.MANUAL, null); + assertTrue(actual.getManual()); + assertEquals(ScheduleType.MANUAL, actual.getScheduleType()); + assertNull(actual.getSchedule()); + assertNull(actual.getScheduleData()); + } + + @Test + void testPopulateSyncScheduleFromBasicType() throws JsonValidationException { + final StandardSync actual = new StandardSync(); + ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, + ConnectionScheduleType.BASIC, new ConnectionScheduleData() + .basicSchedule(new ConnectionScheduleDataBasicSchedule() + .timeUnit(TimeUnitEnum.HOURS) + .units(1L))); + assertFalse(actual.getManual()); + assertEquals(ScheduleType.BASIC_SCHEDULE, actual.getScheduleType()); + assertEquals(TimeUnit.HOURS, actual.getScheduleData().getBasicSchedule().getTimeUnit()); + assertEquals(1L, actual.getScheduleData().getBasicSchedule().getUnits()); + assertNull(actual.getSchedule()); + } + + @Test + void testPopulateSyncScheduleFromCron() throws JsonValidationException { + final StandardSync actual = new StandardSync(); + ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, + ConnectionScheduleType.CRON, new ConnectionScheduleData() + .cron(new ConnectionScheduleDataCron() + .cronTimeZone(EXPECTED_CRON_TIMEZONE) + .cronExpression(EXPECTED_CRON_EXPRESSION))); + assertEquals(ScheduleType.CRON, actual.getScheduleType()); + assertEquals(EXPECTED_CRON_TIMEZONE, actual.getScheduleData().getCron().getCronTimeZone()); + assertEquals(EXPECTED_CRON_EXPRESSION, actual.getScheduleData().getCron().getCronExpression()); + assertNull(actual.getSchedule()); + } + + @Test + void testScheduleValidation() throws JsonValidationException { + final StandardSync actual = new StandardSync(); + assertThrows(JsonValidationException.class, () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, + ConnectionScheduleType.CRON, null)); + assertThrows(JsonValidationException.class, + () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.BASIC, new ConnectionScheduleData())); + assertThrows(JsonValidationException.class, + () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData())); + assertThrows(JsonValidationException.class, + () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData() + .cron(new ConnectionScheduleDataCron()))); + assertThrows(JsonValidationException.class, + () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData() + .cron(new ConnectionScheduleDataCron().cronExpression(EXPECTED_CRON_EXPRESSION).cronTimeZone("Etc/foo")))); + assertThrows(JsonValidationException.class, + () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData() + .cron(new ConnectionScheduleDataCron().cronExpression("bad cron").cronTimeZone(EXPECTED_CRON_TIMEZONE)))); + } + +} diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java index 65b5374793df..0bd17ea9d480 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java @@ -22,6 +22,7 @@ import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionReadList; import io.airbyte.api.model.generated.ConnectionSchedule; +import io.airbyte.api.model.generated.ConnectionScheduleType; import io.airbyte.api.model.generated.ConnectionSearch; import io.airbyte.api.model.generated.ConnectionStatus; import io.airbyte.api.model.generated.ConnectionUpdate; @@ -216,6 +217,13 @@ void testCreateConnection() throws JsonValidationException, ConfigNotFoundExcept assertEquals(expectedConnectionRead, actualConnectionRead); verify(configRepository).writeStandardSync(standardSync); + + // Use new schedule schema, verify that we get the same results. + connectionCreate + .schedule(null) + .scheduleType(ConnectionScheduleType.BASIC) + .scheduleData(ConnectionHelpers.generateBasicConnectionScheduleData()); + assertEquals(expectedConnectionRead, connectionsHandler.createConnection(connectionCreate)); } @Test @@ -360,6 +368,8 @@ void testUpdateConnection() throws JsonValidationException, ConfigNotFoundExcept standardSync.getOperationIds(), newSourceCatalogId) .schedule(null) + .scheduleType(ConnectionScheduleType.MANUAL) + .scheduleData(null) .syncCatalog(catalog) .status(ConnectionStatus.INACTIVE); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index 2fab0ddfea4a..102486f1d773 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -205,6 +205,8 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio .syncCatalog(connectionRead.getSyncCatalog()) .status(connectionRead.getStatus()) .schedule(connectionRead.getSchedule()) + .scheduleType(connectionRead.getScheduleType()) + .scheduleData(connectionRead.getScheduleData()) .source(sourceRead) .destination(destinationRead) .operations(operationReadList.getOperations()) @@ -239,6 +241,8 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio .syncCatalog(modifiedCatalog) .status(expected.getStatus()) .schedule(expected.getSchedule()) + .scheduleType(expected.getScheduleType()) + .scheduleData(expected.getScheduleData()) .source(expected.getSource()) .destination(expected.getDestination()) .operations(expected.getOperations()) @@ -481,7 +485,7 @@ void testToConnectionUpdate() throws IOException { void testForConnectionCreateCompleteness() { final Set handledMethods = Set.of("name", "namespaceDefinition", "namespaceFormat", "prefix", "sourceId", "destinationId", "operationIds", "syncCatalog", "schedule", - "status", "resourceRequirements", "sourceCatalogId"); + "scheduleType", "scheduleData", "status", "resourceRequirements", "sourceCatalogId"); final Set methods = Arrays.stream(ConnectionCreate.class.getMethods()) .filter(method -> method.getReturnType() == ConnectionCreate.class) @@ -502,7 +506,7 @@ void testForConnectionCreateCompleteness() { void testForConnectionUpdateCompleteness() { final Set handledMethods = Set.of("schedule", "connectionId", "syncCatalog", "namespaceDefinition", "namespaceFormat", "prefix", "status", "operationIds", - "resourceRequirements", "name", "sourceCatalogId"); + "resourceRequirements", "name", "sourceCatalogId", "scheduleType", "scheduleData"); final Set methods = Arrays.stream(ConnectionUpdate.class.getMethods()) .filter(method -> method.getReturnType() == ConnectionUpdate.class) diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index 53deb0add230..cbd076b72618 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -13,6 +13,9 @@ import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionSchedule; import io.airbyte.api.model.generated.ConnectionSchedule.TimeUnitEnum; +import io.airbyte.api.model.generated.ConnectionScheduleData; +import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule; +import io.airbyte.api.model.generated.ConnectionScheduleType; import io.airbyte.api.model.generated.ConnectionStatus; import io.airbyte.api.model.generated.ResourceRequirements; import io.airbyte.api.model.generated.SyncMode; @@ -30,6 +33,7 @@ import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.handlers.helpers.CatalogConverter; import java.util.ArrayList; import java.util.Collections; @@ -105,6 +109,11 @@ public static Schedule generateBasicSchedule() { .withUnits(BASIC_SCHEDULE_UNITS); } + public static ConnectionScheduleData generateBasicConnectionScheduleData() { + return new ConnectionScheduleData().basicSchedule( + new ConnectionScheduleDataBasicSchedule().timeUnit(ConnectionScheduleDataBasicSchedule.TimeUnitEnum.DAYS).units(BASIC_SCHEDULE_UNITS)); + } + public static ScheduleData generateBasicScheduleData() { return new ScheduleData().withBasicSchedule(new BasicSchedule() .withTimeUnit(BasicSchedule.TimeUnit.fromValue((BASIC_SCHEDULE_DATA_TIME_UNITS))) @@ -128,6 +137,8 @@ public static ConnectionRead generateExpectedConnectionRead(final UUID connectio .prefix("presto_to_hudi") .status(ConnectionStatus.ACTIVE) .schedule(generateBasicConnectionSchedule()) + .scheduleType(ConnectionScheduleType.BASIC) + .scheduleData(generateBasicConnectionScheduleData()) .syncCatalog(ConnectionHelpers.generateBasicApiCatalog()) .resourceRequirements(new ResourceRequirements() .cpuRequest(TESTING_RESOURCE_REQUIREMENTS.getCpuRequest()) @@ -175,11 +186,8 @@ public static ConnectionRead connectionReadFromStandardSync(final StandardSync s if (standardSync.getStatus() != null) { connectionRead.status(io.airbyte.api.model.generated.ConnectionStatus.fromValue(standardSync.getStatus().value())); } - if (standardSync.getSchedule() != null) { - connectionRead.schedule(new io.airbyte.api.model.generated.ConnectionSchedule() - .timeUnit(TimeUnitEnum.fromValue(standardSync.getSchedule().getTimeUnit().value())) - .units(standardSync.getSchedule().getUnits())); - } + ApiPojoConverters.populateConnectionReadSchedule(standardSync, connectionRead); + if (standardSync.getCatalog() != null) { connectionRead.syncCatalog(CatalogConverter.toApi(standardSync.getCatalog())); } diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java index 333b264dfe38..c85f034e1a11 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java @@ -293,6 +293,7 @@ public void cleanup() { for (final UUID destinationId : destinationIds) { deleteDestination(destinationId); } + destinationPsql.stop(); } catch (final Exception e) { LOGGER.error("Error tearing down test fixtures:", e); } diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java index 35c0db1de9c5..87a9f67a0e09 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java @@ -199,7 +199,9 @@ void testCheckpointing() throws Exception { // now cancel it so that we freeze state! try { apiClient.getJobsApi().cancelJob(new JobIdRequestBody().id(connectionSyncRead1.getJob().getId())); - } catch (final Exception e) {} + } catch (final Exception e) { + LOGGER.error("error:", e); + } final ConnectionState connectionState = waitForConnectionState(apiClient, connectionId); diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java index 922b9411d364..a3057cc74450 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java @@ -5,58 +5,39 @@ package io.airbyte.test.automaticMigrationAcceptance; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import com.google.common.io.Resources; -import io.airbyte.api.client.generated.ConnectionApi; import io.airbyte.api.client.generated.DestinationDefinitionApi; import io.airbyte.api.client.generated.HealthApi; import io.airbyte.api.client.generated.SourceDefinitionApi; import io.airbyte.api.client.generated.WorkspaceApi; import io.airbyte.api.client.invoker.generated.ApiClient; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.api.client.model.generated.ConnectionRead; -import io.airbyte.api.client.model.generated.ConnectionStatus; import io.airbyte.api.client.model.generated.DestinationDefinitionRead; -import io.airbyte.api.client.model.generated.ImportRead; -import io.airbyte.api.client.model.generated.ImportRead.StatusEnum; import io.airbyte.api.client.model.generated.SourceDefinitionRead; import io.airbyte.api.client.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.client.model.generated.WorkspaceRead; import io.airbyte.commons.concurrency.VoidCallable; -import io.airbyte.commons.concurrency.WaitingUtils; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreProperties; -import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.test.airbyte_test_container.AirbyteTestContainer; import java.io.File; -import java.net.URISyntaxException; import java.nio.file.Path; -import java.time.Duration; import java.util.List; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.utility.ComparableVersion; /** * This class contains an e2e test simulating what a user encounter when trying to upgrade Airybte. - *

- * Three invariants are tested: - *

- * - upgrading pass 0.32.0 without first upgrading to 0.32.0 should error. - *

- * - upgrading pass 0.32.0 without first upgrading to 0.32.0 should not put the db in a bad state. - *

- * - upgrading from 0.32.0 to the latest version should work. + * - upgrading from 0.32.0 to the latest version should work. - This test previously tested + * upgrading from even older versions, which has since been removed *

* This test runs on the current code version and expects local images with the `dev` tag to be * available. To do so, run SUB_BUILD=PLATFORM ./gradlew build. @@ -78,36 +59,22 @@ class MigrationAcceptanceTest { private static final String TEST_LOCAL_ROOT = "/tmp/airbyte_local_migration_test"; private static final String TEST_LOCAL_DOCKER_MOUNT = "/tmp/airbyte_local_migration_test"; + private static WorkspaceIdRequestBody workspaceIdRequestBody = null; + @Test + @Disabled void testAutomaticMigration() throws Exception { - // run version 17 (the oldest version of airbyte that supports auto migration) - final File version17DockerComposeFile = MoreResources.readResourceAsFile("docker-compose-migration-test-0-17-0-alpha.yaml"); - final Properties version17EnvVariables = MoreProperties - .envFileToProperties(MoreResources.readResourceAsFile("env-file-migration-test-0-17-0.env")); - runAirbyte(version17DockerComposeFile, version17EnvVariables, () -> { - populateDataForFirstRun(); - healthCheck(getApiClient()); - }); - - LOGGER.info("Finish initial 0.17.0-alpha start.."); - - // attempt to run from pre-version bump version to post-version bump version. expect failure. - final File currentDockerComposeFile = MoreResources.readResourceAsFile("docker-compose.yaml"); - // piggybacks off of whatever the existing .env file is, so override default filesystem values in to - // point at test paths. - final Properties envFileProperties = overrideDirectoriesForTest(MoreProperties.envFileToProperties(ENV_FILE)); - // use the dev version so the test is run on the current code version. - envFileProperties.setProperty("VERSION", "dev"); - runAirbyteAndWaitForUpgradeException(currentDockerComposeFile, envFileProperties); - LOGGER.info("Finished testing upgrade exception.."); - - // run "faux" major version bump version + // start at "faux" major version bump version. This was the last version that required db data + // migrations. final File version32DockerComposeFile = MoreResources.readResourceAsFile("docker-compose-migration-test-0-32-0-alpha.yaml"); - final Properties version32EnvFileProperties = MoreProperties .envFileToProperties(MoreResources.readResourceAsFile("env-file-migration-test-0-32-0.env")); runAirbyte(version32DockerComposeFile, version32EnvFileProperties, MigrationAcceptanceTest::assertHealthy); + final File currentDockerComposeFile = MoreResources.readResourceAsFile("docker-compose.yaml"); + // piggybacks off of whatever the existing .env file is, so override default filesystem values in to + // point at test paths. + final Properties envFileProperties = overrideDirectoriesForTest(MoreProperties.envFileToProperties(ENV_FILE)); // run from last major version bump to current version. runAirbyte(currentDockerComposeFile, envFileProperties, MigrationAcceptanceTest::assertHealthy, false); } @@ -145,22 +112,6 @@ private void runAirbyte(final File dockerComposeFile, } } - private void runAirbyteAndWaitForUpgradeException(final File dockerComposeFile, final Properties env) throws Exception { - final WaitForLogLine waitForLogLine = new WaitForLogLine(); - LOGGER.info("Start up Airbyte at version {}", env.get("VERSION")); - final AirbyteTestContainer airbyteTestContainer = new AirbyteTestContainer.Builder(dockerComposeFile) - .setEnv(env) - .setLogListener("bootloader", waitForLogLine.getListener("After that upgrade is complete, you may upgrade to version")) - .build(); - - airbyteTestContainer.startAsync(); - - final Supplier condition = waitForLogLine.hasSeenLine(); - final boolean loggedUpgradeException = WaitingUtils.waitForCondition(Duration.ofSeconds(5), Duration.ofMinutes(1), condition); - airbyteTestContainer.stopRetainVolumes(); - assertTrue(loggedUpgradeException, "Airbyte failed to throw upgrade exception."); - } - /** * Allows the test to listen for a specific log line so that the test can end as soon as that log * line has been encountered. @@ -189,11 +140,16 @@ private static void assertHealthy() throws ApiException { assertDataFromApi(apiClient); } + @SuppressWarnings("PMD.NonThreadSafeSingleton") private static void assertDataFromApi(final ApiClient apiClient) throws ApiException { - final WorkspaceIdRequestBody workspaceIdRequestBody = assertWorkspaceInformation(apiClient); + if (workspaceIdRequestBody != null) { + assertEquals(assertWorkspaceInformation(apiClient).getWorkspaceId(), workspaceIdRequestBody.getWorkspaceId()); + } else { + workspaceIdRequestBody = assertWorkspaceInformation(apiClient); + } + assertSourceDefinitionInformation(apiClient); assertDestinationDefinitionInformation(apiClient); - assertConnectionInformation(apiClient, workspaceIdRequestBody); } private static void assertSourceDefinitionInformation(final ApiClient apiClient) throws ApiException { @@ -205,18 +161,9 @@ private static void assertSourceDefinitionInformation(final ApiClient apiClient) for (final SourceDefinitionRead sourceDefinitionRead : sourceDefinitions) { if ("435bb9a5-7887-4809-aa58-28c27df0d7ad".equals(sourceDefinitionRead.getSourceDefinitionId().toString())) { assertEquals(sourceDefinitionRead.getName(), "MySQL"); - assertEquals(sourceDefinitionRead.getDockerImageTag(), "0.2.0"); foundMysqlSourceDefinition = true; } else if ("decd338e-5647-4c0b-adf4-da0e75f5a750".equals(sourceDefinitionRead.getSourceDefinitionId().toString())) { - final String[] tagBrokenAsArray = sourceDefinitionRead.getDockerImageTag().replace(".", ",").split(","); - assertEquals(3, tagBrokenAsArray.length); - // todo (cgardens) - this is very brittle. depending on when this connector gets updated in - // source_definitions.yaml this test can start to break. for now just doing quick fix, but we should - // be able to do an actual version comparison like we do with AirbyteVersion. - assertTrue(Integer.parseInt(tagBrokenAsArray[0]) >= 0, "actual tag: " + sourceDefinitionRead.getDockerImageTag()); - assertTrue(Integer.parseInt(tagBrokenAsArray[1]) >= 3, "actual tag: " + sourceDefinitionRead.getDockerImageTag()); - assertTrue(Integer.parseInt(tagBrokenAsArray[2]) >= 0, "actual tag: " + sourceDefinitionRead.getDockerImageTag()); - assertTrue(sourceDefinitionRead.getName().contains("Postgres")); + assertEquals(sourceDefinitionRead.getName(), "Postgres"); foundPostgresSourceDefinition = true; } } @@ -237,21 +184,11 @@ private static void assertDestinationDefinitionInformation(final ApiClient apiCl destinationId = destinationDefinitionRead.getDestinationDefinitionId().toString(); if ("25c5221d-dce2-4163-ade9-739ef790f503".equals(destinationId)) { assertEquals("Postgres", destinationDefinitionRead.getName()); - assertEquals("0.2.0", destinationDefinitionRead.getDockerImageTag()); foundPostgresDestinationDefinition = true; } else if ("8be1cf83-fde1-477f-a4ad-318d23c9f3c6".equals(destinationId)) { - final String tag = destinationDefinitionRead.getDockerImageTag(); - final AirbyteVersion currentVersion = new AirbyteVersion(tag); - final AirbyteVersion previousVersion = new AirbyteVersion("0.2.0"); - final AirbyteVersion finalVersion = - (currentVersion.checkOnlyPatchVersionIsUpdatedComparedTo(previousVersion) ? currentVersion : previousVersion); - assertEquals(finalVersion.toString(), currentVersion.toString()); assertTrue(destinationDefinitionRead.getName().contains("Local CSV")); foundLocalCSVDestinationDefinition = true; } else if ("424892c4-daac-4491-b35d-c6688ba547ba".equals(destinationId)) { - final String tag = destinationDefinitionRead.getDockerImageTag(); - final ComparableVersion version = new ComparableVersion(tag); - assertTrue(version.compareTo(new ComparableVersion("0.3.9")) >= 0); assertTrue(destinationDefinitionRead.getName().contains("Snowflake")); foundSnowflakeDestinationDefinition = true; } @@ -262,60 +199,17 @@ private static void assertDestinationDefinitionInformation(final ApiClient apiCl assertTrue(foundSnowflakeDestinationDefinition); } - private static void assertConnectionInformation(final ApiClient apiClient, final WorkspaceIdRequestBody workspaceIdRequestBody) - throws ApiException { - final ConnectionApi connectionApi = new ConnectionApi(apiClient); - final List connections = connectionApi.listConnectionsForWorkspace(workspaceIdRequestBody).getConnections(); - assertEquals(connections.size(), 2); - for (final ConnectionRead connection : connections) { - if ("a294256f-1abe-4837-925f-91602c7207b4".equals(connection.getConnectionId().toString())) { - assertEquals("", connection.getPrefix()); - assertEquals("28ffee2b-372a-4f72-9b95-8ed56a8b99c5", connection.getSourceId().toString()); - assertEquals("4e00862d-5484-4f50-9860-f3bbb4317397", connection.getDestinationId().toString()); - assertEquals(ConnectionStatus.ACTIVE, connection.getStatus()); - assertNull(connection.getSchedule()); - } else if ("49dae3f0-158b-4737-b6e4-0eed77d4b74e".equals(connection.getConnectionId().toString())) { - assertEquals("", connection.getPrefix()); - assertEquals("28ffee2b-372a-4f72-9b95-8ed56a8b99c5", connection.getSourceId().toString()); - assertEquals("5434615d-a3b7-4351-bc6b-a9a695555a30", connection.getDestinationId().toString()); - assertEquals(ConnectionStatus.ACTIVE, connection.getStatus()); - assertNull(connection.getSchedule()); - } else { - fail("Unknown sync " + connection.getConnectionId().toString()); - } - } - } - private static WorkspaceIdRequestBody assertWorkspaceInformation(final ApiClient apiClient) throws ApiException { final WorkspaceApi workspaceApi = new WorkspaceApi(apiClient); final WorkspaceRead workspace = workspaceApi.listWorkspaces().getWorkspaces().get(0); - // originally the default workspace started with a hardcoded id. the migration in version 0.29.0 - // took that id and randomized it. we want to check that the id is now NOT that hardcoded id and - // that all related resources use the updated workspaceId as well. assertNotNull(workspace.getWorkspaceId().toString()); - assertNotEquals("5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", workspace.getWorkspaceId().toString()); - assertEquals("17f90b72-5ae4-40b7-bc49-d6c2943aea57", workspace.getCustomerId().toString()); - assertEquals("default", workspace.getName()); - assertEquals("default", workspace.getSlug()); - assertEquals(true, workspace.getInitialSetupComplete()); - assertEquals(false, workspace.getAnonymousDataCollection()); - assertEquals(false, workspace.getNews()); - assertEquals(false, workspace.getSecurityUpdates()); - assertEquals(false, workspace.getDisplaySetupWizard()); + assertNotNull(workspace.getName()); + assertNotNull(workspace.getSlug()); + assertEquals(false, workspace.getInitialSetupComplete()); return new WorkspaceIdRequestBody().workspaceId(workspace.getWorkspaceId()); } - @SuppressWarnings("UnstableApiUsage") - private static void populateDataForFirstRun() throws ApiException, URISyntaxException { - final ImportApi deploymentApi = new ImportApi(getApiClient()); - final File file = Path - .of(Resources.getResource("03a4c904-c91d-447f-ab59-27a43b52c2fd.gz").toURI()) - .toFile(); - final ImportRead importRead = deploymentApi.importArchive(file); - assertEquals(importRead.getStatus(), StatusEnum.SUCCEEDED); - } - private static void healthCheck(final ApiClient apiClient) { final HealthApi healthApi = new HealthApi(apiClient); try { diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-17-0-alpha.yaml b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-17-0-alpha.yaml deleted file mode 100644 index 116796652765..000000000000 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-17-0-alpha.yaml +++ /dev/null @@ -1,60 +0,0 @@ -#The file is used for testing and is from version 0.17.0-alpha -version: "3.7" -#https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments -x-logging: &default-logging - options: - max-size: "1m" - max-file: "1" - driver: json-file -services: - init: - image: airbyte/init:${VERSION} - logging: *default-logging - command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}" - environment: - - LOCAL_ROOT=${LOCAL_ROOT} - - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT} - volumes: - - ${HACK_LOCAL_ROOT_PARENT}:/local_parent - db: - image: airbyte/db:${VERSION} - logging: *default-logging - restart: unless-stopped - environment: - - POSTGRES_USER=${DATABASE_USER} - - POSTGRES_PASSWORD=${DATABASE_PASSWORD} - volumes: - - db:/var/lib/postgresql/data - seed: - image: airbyte/seed:${VERSION} - # Pre-populate the volume if it is empty. - # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container - volumes: - - data:/app/seed - server: - image: airbyte/server:${VERSION} - logging: *default-logging - restart: unless-stopped - environment: - - WEBAPP_URL=${WEBAPP_URL} - - DATABASE_USER=${DATABASE_USER} - - DATABASE_PASSWORD=${DATABASE_PASSWORD} - - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB} - - WORKSPACE_ROOT=${WORKSPACE_ROOT} - - CONFIG_ROOT=${CONFIG_ROOT} - - TRACKING_STRATEGY=${TRACKING_STRATEGY} - - AIRBYTE_VERSION=${VERSION} - - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - - TEMPORAL_HOST=${TEMPORAL_HOST} - ports: - - 8001:8001 - volumes: - - workspace:${WORKSPACE_ROOT} - - data:${CONFIG_ROOT} -volumes: - workspace: - name: ${WORKSPACE_DOCKER_MOUNT} - data: - name: ${DATA_DOCKER_MOUNT} - db: - name: ${DB_DOCKER_MOUNT} diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-17-0.env b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-17-0.env deleted file mode 100644 index cdcebcec9b9d..000000000000 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-17-0.env +++ /dev/null @@ -1,22 +0,0 @@ -VERSION=0.17.0-alpha-db-patch -DATABASE_USER=docker -DATABASE_PASSWORD=docker -DATABASE_DB=airbyte -CONFIG_ROOT=/data -WORKSPACE_ROOT=/tmp/workspace -DATA_DOCKER_MOUNT=airbyte_data_migration_test -DB_DOCKER_MOUNT=airbyte_db_migration_test -WORKSPACE_DOCKER_MOUNT=airbyte_workspace_migration_test -LOCAL_ROOT=/tmp/airbyte_local_migration_test -LOCAL_DOCKER_MOUNT=/tmp/airbyte_local_migration_test -TRACKING_STRATEGY=logging -HACK_LOCAL_ROOT_PARENT=/tmp -WEBAPP_URL=http://localhost:8000/ -API_URL=http://localhost:8001/api/v1/ -TEMPORAL_HOST=airbyte-temporal:7233 -INTERNAL_API_HOST=airbyte-server:8001 -S3_LOG_BUCKET= -S3_LOG_BUCKET_REGION= -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -GCP_STORAGE_BUCKET= diff --git a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.module.scss b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.module.scss new file mode 100644 index 000000000000..00bd408cd9da --- /dev/null +++ b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.module.scss @@ -0,0 +1,11 @@ +@use "../../scss/colors"; + +.container { + min-height: 100vh; + padding-bottom: 40px; + background-color: colors.$white; +} + +.content { + padding: 0 35px; +} diff --git a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx index ff10ce5b5f1e..5bbb9e1ae8e7 100644 --- a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx +++ b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx @@ -1,40 +1,28 @@ import type { Url } from "url"; import { useMemo } from "react"; -import { FormattedMessage } from "react-intl"; -import { useIntl } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; import { PluggableList } from "react-markdown/lib/react-markdown"; -import { ReflexElement } from "react-reflex"; import { useLocation } from "react-router-dom"; import { useUpdateEffect } from "react-use"; import rehypeSlug from "rehype-slug"; import urls from "rehype-urls"; -import styled from "styled-components"; import { LoadingPage, PageTitle } from "components"; -import Markdown from "components/Markdown/Markdown"; +import { Markdown } from "components/Markdown"; import { useConfig } from "config"; import { useDocumentation } from "hooks/services/useDocumentation"; import { useDocumentationPanelContext } from "views/Connector/ConnectorDocumentationLayout/DocumentationPanelContext"; -export const DocumentationContainer = styled.div` - padding: 0px 0px 20px; - background-color: #ffffff; - min-height: 100vh; -`; - -export const DocumentationContent = styled(Markdown)` - padding: 0px 35px 20px; -`; +import styles from "./DocumentationPanel.module.scss"; export const DocumentationPanel: React.FC = () => { + const { formatMessage } = useIntl(); const config = useConfig(); - const { setDocumentationPanelOpen, documentationUrl } = useDocumentationPanelContext(); - const { data: docs, isLoading } = useDocumentation(documentationUrl); - const { formatMessage } = useIntl(); + // @ts-expect-error rehype-slug currently has type conflicts due to duplicate vfile dependencies const urlReplacerPlugin: PluggableList = useMemo(() => { const sanitizeLinks = (url: Url, element: Element) => { @@ -62,18 +50,14 @@ export const DocumentationPanel: React.FC = () => { return isLoading || documentationUrl === "" ? ( - ) : docs ? ( - - } /> - {!docs.includes("") ? ( - - ) : ( - - )} - ) : ( - - - +

+ } /> + ") ? docs : formatMessage({ id: "connector.setupGuide.notFound" })} + rehypePlugins={urlReplacerPlugin} + /> +
); }; diff --git a/airbyte-webapp/src/components/GroupControls/GroupControls.tsx b/airbyte-webapp/src/components/GroupControls/GroupControls.tsx index ccf6d7714c14..e2007158cea8 100644 --- a/airbyte-webapp/src/components/GroupControls/GroupControls.tsx +++ b/airbyte-webapp/src/components/GroupControls/GroupControls.tsx @@ -3,17 +3,13 @@ import styled from "styled-components"; import { Label, TextWithHTML } from "components"; -const GroupTitle = styled.div` +const GroupTitle = styled.div<{ $fullWidthTitle: boolean }>` margin-top: -23px; background: ${({ theme }) => theme.whiteColor}; padding: 0 5px; display: inline-block; vertical-align: middle; - - & > div { - min-width: 180px; - display: inline-block; - } + width: ${({ $fullWidthTitle }) => ($fullWidthTitle ? "100%" : "auto")}; `; const FormGroup = styled.div` @@ -28,12 +24,19 @@ interface GroupControlsProps { title: React.ReactNode; description?: string; name?: string; + fullWidthTitle?: boolean; } -const GroupControls: React.FC = ({ title, description, children, name }) => { +const GroupControls: React.FC = ({ + title, + description, + children, + name, + fullWidthTitle = false, +}) => { return ( - {title} + {title} {description && diff --git a/airbyte-webapp/src/components/Markdown/Markdown.tsx b/airbyte-webapp/src/components/Markdown/Markdown.tsx index 492e4e26bece..fd423b76ab49 100644 --- a/airbyte-webapp/src/components/Markdown/Markdown.tsx +++ b/airbyte-webapp/src/components/Markdown/Markdown.tsx @@ -1,10 +1,12 @@ import type { PluggableList } from "react-markdown/lib/react-markdown"; +import classNames from "classnames"; import React from "react"; import ReactMarkdown from "react-markdown"; import remarkFrontmatter from "remark-frontmatter"; import remarkGfm from "remark-gfm"; -import styled from "styled-components"; + +import "./styles.scss"; interface Props { content?: string; @@ -12,12 +14,12 @@ interface Props { rehypePlugins?: PluggableList; } -const Markdown: React.FC = ({ content, className, rehypePlugins }) => { +export const Markdown: React.FC = ({ content, className, rehypePlugins }) => { return ( (href.startsWith("#") ? undefined : "_blank")} - className={className} + className={classNames("airbyte-markdown", className)} skipHtml // @ts-expect-error remarkFrontmatter currently has type conflicts due to duplicate vfile dependencies // This is not actually causing any issues, but requires to disable TS on this for now. @@ -27,85 +29,3 @@ const Markdown: React.FC = ({ content, className, rehypePlugins }) => { /> ); }; - -const StyledMarkdown = styled(Markdown)` - * { - color: ${({ theme }) => theme.textColor}; - line-height: 24px; - font-size: 16px; - font-weight: 400; - } - - h1 { - font-size: 48px; - line-height: 56px; - font-weight: bold; - } - - h2 { - font-size: 24px; - line-height: 36px; - font-weight: bold; - } - - h3 { - font-size: 18px; - font-weight: bold; - } - - h4 { - font-weight: bold; - } - - a { - color: ${({ theme }) => theme.primaryColor}; - text-decoration: none; - line-height: 24px; - - &:hover { - text-decoration: underline; - } - } - - table { - border-collapse: collapse; - } - - th, - td { - border: 1px solid ${({ theme }) => theme.borderTableColor}; - margin: 0; - padding: 8px 16px; - } - - th { - background: ${({ theme }) => theme.lightTableColor}; - } - - blockquote { - border-left: 4px solid ${({ theme }) => theme.borderTableColor}; - padding-left: 16px; - margin-left: 25px; - } - - strong { - font-weight: bold; - } - - code { - background: ${({ theme }) => theme.lightTableColor}; - - &.language-sql, - &.language-text { - display: block; - padding: 15px; - overflow: auto; - } - } - - img { - max-width: 100%; - } -`; - -export default StyledMarkdown; diff --git a/airbyte-webapp/src/components/Markdown/index.ts b/airbyte-webapp/src/components/Markdown/index.ts index f14dafbcd5b6..642b3cdeeec1 100644 --- a/airbyte-webapp/src/components/Markdown/index.ts +++ b/airbyte-webapp/src/components/Markdown/index.ts @@ -1 +1 @@ -export { default as Markdown } from "./Markdown"; +export { Markdown } from "./Markdown"; diff --git a/airbyte-webapp/src/components/Markdown/styles.scss b/airbyte-webapp/src/components/Markdown/styles.scss new file mode 100644 index 000000000000..86ebc4cc9bec --- /dev/null +++ b/airbyte-webapp/src/components/Markdown/styles.scss @@ -0,0 +1,82 @@ +@use "../../scss/colors"; + +.airbyte-markdown { + * { + color: colors.$dark-blue; + line-height: 24px; + font-size: 16px; + font-weight: 400; + } + + h1 { + font-size: 48px; + line-height: 56px; + font-weight: bold; + } + + h2 { + font-size: 24px; + line-height: 36px; + font-weight: bold; + } + + h3 { + font-size: 18px; + font-weight: bold; + } + + h4 { + font-weight: bold; + } + + a { + color: colors.$blue; + text-decoration: none; + line-height: 24px; + + &:hover { + text-decoration: underline; + } + } + + table { + border-collapse: collapse; + } + + th, + td { + border: 1px solid colors.$grey-100; + margin: 0; + padding: 8px 16px; + } + + th { + background: colors.$grey-50; + } + + blockquote { + border-left: 4px solid colors.$grey-100; + padding-left: 16px; + margin-left: 25px; + } + + strong { + font-weight: bold; + } + + code { + white-space: break-spaces; + background: colors.$grey-50; + + &.language-sql, + &.language-text { + display: block; + padding: 15px; + overflow: auto; + } + } + + img { + max-width: 100%; + } +} diff --git a/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.module.scss b/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.module.scss new file mode 100644 index 000000000000..f7b8ce4a4c4f --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.module.scss @@ -0,0 +1,29 @@ +$banner-height: 30px; + +.mainContainer { + overflow: hidden; + display: flex; + flex-direction: row; + width: 100%; + height: 100%; + min-height: 680px; + + .content { + overflow-y: auto; + width: 100%; + height: 100%; + + .dataBlock { + width: 100%; + height: 100%; + } + + &.alertBanner { + margin-top: $banner-height; + + .dataBlock { + height: calc(100% - #{$banner-height}); + } + } + } +} diff --git a/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx b/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx index 0383a09a6ac4..9e01e6b0cd2d 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx @@ -1,7 +1,7 @@ +import classNames from "classnames"; import React, { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Link, Outlet } from "react-router-dom"; -import styled from "styled-components"; import { LoadingPage } from "components"; import { AlertBanner } from "components/base/Banner/AlertBanner"; @@ -15,27 +15,7 @@ import { ResourceNotFoundErrorBoundary } from "views/common/ResorceNotFoundError import { StartOverErrorView } from "views/common/StartOverErrorView"; import { InsufficientPermissionsErrorBoundary } from "./InsufficientPermissionsErrorBoundary"; - -const MainContainer = styled.div` - width: 100%; - height: 100%; - overflow: hidden; - display: flex; - flex-direction: row; - min-height: 680px; -`; - -const Content = styled.div` - overflow-y: auto; - width: 100%; - height: 100%; -`; - -const DataBlock = styled.div<{ hasBanner?: boolean }>` - width: 100%; - height: 100%; - padding-top: ${({ hasBanner }) => (hasBanner ? 30 : 0)}px; -`; +import styles from "./MainView.module.scss"; const MainView: React.FC = (props) => { const { formatMessage } = useIntl(); @@ -79,19 +59,19 @@ const MainView: React.FC = (props) => { }, [alertToShow, cloudWorkspace, formatMessage]); return ( - +
}> - +
{alertToShow && } - +
}> }>{props.children ?? } - - +
+
- +
); }; diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx new file mode 100644 index 000000000000..8f025a281d07 --- /dev/null +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx @@ -0,0 +1,265 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { IntlProvider } from "react-intl"; + +import { + AirbyteCatalog, + CatalogDiff, + DestinationSyncMode, + StreamTransform, + SyncMode, +} from "core/request/AirbyteClient"; + +import messages from "../../../locales/en.json"; +import { CatalogDiffModal } from "./CatalogDiffModal"; + +const mockCatalogDiff: CatalogDiff = { + transforms: [], +}; + +const removedItems: StreamTransform[] = [ + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "dragonfruit" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "eclair" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "fishcake" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "gelatin_mold" }, + }, +]; + +const addedItems: StreamTransform[] = [ + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "banana" }, + }, + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "carrot" }, + }, +]; + +const updatedItems: StreamTransform[] = [ + { + transformType: "update_stream", + streamDescriptor: { namespace: "apple", name: "harissa_paste" }, + updateStream: [ + { transformType: "add_field", fieldName: ["users", "phone"] }, + { transformType: "add_field", fieldName: ["users", "email"] }, + { transformType: "remove_field", fieldName: ["users", "lastName"] }, + + { + transformType: "update_field_schema", + fieldName: ["users", "address"], + updateFieldSchema: { oldSchema: { type: "number" }, newSchema: { type: "string" } }, + }, + ], + }, +]; + +const mockCatalog: AirbyteCatalog = { + streams: [ + { + stream: { + namespace: "apple", + name: "banana", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "carrot", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "dragonfruit", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "eclair", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "fishcake", + }, + config: { + syncMode: SyncMode.incremental, + destinationSyncMode: DestinationSyncMode.append_dedup, + }, + }, + { + stream: { + namespace: "apple", + name: "gelatin_mold", + }, + config: { + syncMode: SyncMode.incremental, + destinationSyncMode: DestinationSyncMode.append_dedup, + }, + }, + { + stream: { + namespace: "apple", + name: "harissa_paste", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + ], +}; + +describe("catalog diff modal", () => { + afterEach(cleanup); + beforeEach(() => { + mockCatalogDiff.transforms = []; + }); + + test("it renders the correct section for each type of transform", () => { + mockCatalogDiff.transforms.push(...addedItems, ...removedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + /** + * tests for: + * - proper sections being created + * - syncmode string is only rendered for removed streams + */ + + const newStreamsTable = screen.getByRole("table", { name: /new streams/ }); + expect(newStreamsTable).toBeInTheDocument(); + + const newStreamRow = screen.getByRole("row", { name: "apple banana" }); + expect(newStreamRow).toBeInTheDocument(); + + const newStreamRowWithSyncMode = screen.queryByRole("row", { name: "apple carrot incremental | append_dedup" }); + expect(newStreamRowWithSyncMode).not.toBeInTheDocument(); + + const removedStreamsTable = screen.getByRole("table", { name: /removed streams/ }); + expect(removedStreamsTable).toBeInTheDocument(); + + const removedStreamRowWithSyncMode = screen.getByRole("row", { + name: "apple dragonfruit full_refresh | overwrite", + }); + expect(removedStreamRowWithSyncMode).toBeInTheDocument(); + + const updatedStreamsSection = screen.getByRole("list", { name: /table with changes/ }); + expect(updatedStreamsSection).toBeInTheDocument(); + + const updatedStreamRowWithSyncMode = screen.queryByRole("row", { + name: "apple harissa_paste full_refresh | overwrite", + }); + expect(updatedStreamRowWithSyncMode).not.toBeInTheDocument(); + }); + + test("added fields are not rendered when not in the diff", () => { + mockCatalogDiff.transforms.push(...removedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + const newStreamsTable = screen.queryByRole("table", { name: /new streams/ }); + expect(newStreamsTable).not.toBeInTheDocument(); + }); + + test("removed fields are not rendered when not in the diff", () => { + mockCatalogDiff.transforms.push(...addedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + const removedStreamsTable = screen.queryByRole("table", { name: /removed streams/ }); + expect(removedStreamsTable).not.toBeInTheDocument(); + }); + + test("changed streams accordion opens/closes on clicking the description row", () => { + mockCatalogDiff.transforms.push(...addedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + const accordionHeader = screen.getByRole("button", { name: /toggle accordion/ }); + + expect(accordionHeader).toBeInTheDocument(); + + const nullAccordionBody = screen.queryByRole("table", { name: /removed fields/ }); + expect(nullAccordionBody).not.toBeInTheDocument(); + + userEvent.click(accordionHeader); + const openAccordionBody = screen.getByRole("table", { name: /removed fields/ }); + expect(openAccordionBody).toBeInTheDocument(); + + userEvent.click(accordionHeader); + const nullAccordionBodyAgain = screen.queryByRole("table", { name: /removed fields/ }); + expect(nullAccordionBodyAgain).not.toBeInTheDocument(); + mockCatalogDiff.transforms = []; + }); +}); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx index d42a051c03b9..112563dba001 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx @@ -23,7 +23,7 @@ export const DiffAccordion: React.FC = ({ streamTransform }) {({ open }) => ( <> - + = ({ fieldTransforms, diffVerb }) => { return ( - +
diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx index 453f23443dcf..a206a18fd5f2 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx @@ -19,13 +19,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={removedCount} color="red" light - ariaLabel={`${removedCount} ${formatMessage( + ariaLabel={`${formatMessage( { id: "connection.updateSchema.removed", }, { value: removedCount, - item: formatMessage({ id: "field" }, { values: { count: removedCount } }), + item: formatMessage({ id: "connection.updateSchema.field" }, { count: removedCount }), } )}`} /> @@ -35,13 +35,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={newCount} color="green" light - ariaLabel={`${newCount} ${formatMessage( + ariaLabel={`${formatMessage( { id: "connection.updateSchema.new", }, { value: newCount, - item: formatMessage({ id: "field" }, { values: { count: newCount } }), + item: formatMessage({ id: "connection.updateSchema.field" }, { count: newCount }), } )}`} /> @@ -51,13 +51,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={changedCount} color="blue" light - ariaLabel={`${changedCount} ${formatMessage( + ariaLabel={`${formatMessage( { id: "connection.updateSchema.changed", }, { value: changedCount, - item: formatMessage({ id: "field" }, { values: { count: changedCount } }), + item: formatMessage({ id: "connection.updateSchema.field" }, { count: changedCount }), } )}`} /> diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx index 47981779d2d9..cd82648fdee9 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx @@ -31,7 +31,7 @@ export const DiffSection: React.FC = ({ streams, catalog, diff
- +
- + ); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.module.scss b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.module.scss index 1c608338fea6..2a15129809ef 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.module.scss +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.module.scss @@ -1,16 +1,6 @@ +@use "../../../../scss/colors"; @use "../../../../scss/variables"; @use "./DiffSection.module.scss"; -@use "../../../../scss/colors"; - -ul, -li { - list-style-type: none; - list-style-position: inside; - margin: 0px; - padding: 0px; - height: auto; - font-weight: 400; -} .fieldHeader { @extend .sectionHeader; @@ -38,4 +28,14 @@ li { .fieldRowsContainer { padding-left: variables.$spacing-lg; + + ul, + li { + list-style-type: none; + list-style-position: inside; + margin: 0px; + padding: 0px; + height: auto; + font-weight: 400; + } } diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx index 4c03fcf8a4e7..f6d150fb80b1 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx @@ -32,7 +32,15 @@ export const FieldSection: React.FC = ({ streams, diffVerb })
{streams.length > 0 && ( -
    +
      {streams.map((stream) => { return (
    • diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx index 2931ed5ed644..de077b279de8 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx @@ -45,7 +45,7 @@ export const StreamRow: React.FC = ({ streamTransform, syncMode, )}
- {" "} + ); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx new file mode 100644 index 000000000000..ca2181e4a465 --- /dev/null +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx @@ -0,0 +1,74 @@ +import { ComponentStory, ComponentMeta } from "@storybook/react"; +import { FormattedMessage } from "react-intl"; + +import Modal from "components/Modal"; + +import { CatalogDiffModal } from "./CatalogDiffModal"; + +export default { + title: "Ui/CatalogDiffModal", + component: CatalogDiffModal, +} as ComponentMeta; + +const Template: ComponentStory = (args) => { + return ( + }> + { + return null; + }} + /> + + ); +}; + +export const Primary = Template.bind({}); + +Primary.args = { + catalogDiff: { + transforms: [ + { + transformType: "update_stream", + streamDescriptor: { namespace: "apple", name: "harissa_paste" }, + updateStream: [ + { transformType: "add_field", fieldName: ["users", "phone"] }, + { transformType: "add_field", fieldName: ["users", "email"] }, + { transformType: "remove_field", fieldName: ["users", "lastName"] }, + + { + transformType: "update_field_schema", + fieldName: ["users", "address"], + updateFieldSchema: { oldSchema: { type: "number" }, newSchema: { type: "string" } }, + }, + ], + }, + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "banana" }, + }, + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "carrot" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "dragonfruit" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "eclair" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "fishcake" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "gelatin_mold" }, + }, + ], + }, + catalog: { streams: [] }, +}; diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx index 3595bc354543..a4058e0dc0ba 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx @@ -58,7 +58,7 @@ export const ConnectorDocumentationLayout: React.FC = ({ children }) => { const screenWidth = useWindowSize().width; return ( - + {children} diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.module.scss b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.module.scss new file mode 100644 index 000000000000..e8b85e49c909 --- /dev/null +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.module.scss @@ -0,0 +1,7 @@ +.sectionTitle { + display: flex; +} + +.sectionTitleDropdown { + flex: 1 1 auto; +} diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx index fed6d6e17b46..cc8ed2460fba 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/ConditionSection.tsx @@ -11,6 +11,7 @@ import { isDefined } from "utils/common"; import { useServiceForm } from "../../serviceFormContext"; import { ServiceFormValues } from "../../types"; +import styles from "./ConditionSection.module.scss"; import { FormSection } from "./FormSection"; const GroupLabel = styled(Label)` @@ -78,17 +79,19 @@ export const ConditionSection: React.FC = ({ formField, p +
{label ? {label}: : null} - +
} > diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 946f7e43c92f..c6e785527097 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation libs.micrometer.statsd implementation project(':airbyte-analytics') + implementation project(':airbyte-api') implementation project(':airbyte-commons-docker') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index c82d3785a7ef..c4c75b3346e4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -223,9 +223,10 @@ private void registerConnectionManager(final WorkerFactory factory) { private void registerSync(final WorkerFactory factory) { final ReplicationActivityImpl replicationActivity = getReplicationActivityImpl(replicationWorkerConfigs, replicationProcessFactory); - final NormalizationActivityImpl normalizationActivity = getNormalizationActivityImpl( - defaultWorkerConfigs, - defaultProcessFactory); + // Note that the configuration injected here is for the normalization orchestrator, and not the + // normalization pod itself. + // Configuration for the normalization pod is injected via the SyncWorkflowImpl. + final NormalizationActivityImpl normalizationActivity = getNormalizationActivityImpl(defaultWorkerConfigs, defaultProcessFactory); final DbtTransformationActivityImpl dbtTransformationActivity = getDbtActivityImpl( defaultWorkerConfigs, @@ -314,6 +315,15 @@ private DbtTransformationActivityImpl getDbtActivityImpl(final WorkerConfigs wor airbyteVersion); } + /** + * Return either a docker or kubernetes process factory depending on the environment in + * {@link WorkerConfigs} + * + * @param configs used to determine which process factory to create. + * @param workerConfigs used to create the process factory. + * @return either a {@link DockerProcessFactory} or a {@link KubeProcessFactory}. + * @throws IOException + */ private static ProcessFactory getJobProcessFactory(final Configs configs, final WorkerConfigs workerConfigs) throws IOException { if (configs.getWorkerEnvironment() == Configs.WorkerEnvironment.KUBERNETES) { final KubernetesClient fabricClient = new DefaultKubernetesClient(); @@ -341,14 +351,14 @@ private static WorkerOptions getWorkerOptions(final int max) { .build(); } - public static record ContainerOrchestratorConfig( - String namespace, - DocumentStoreClient documentStoreClient, - KubernetesClient kubernetesClient, - String secretName, - String secretMountPath, - String containerOrchestratorImage, - String googleApplicationCredentials) {} + public record ContainerOrchestratorConfig( + String namespace, + DocumentStoreClient documentStoreClient, + KubernetesClient kubernetesClient, + String secretName, + String secretMountPath, + String containerOrchestratorImage, + String googleApplicationCredentials) {} static Optional getContainerOrchestratorConfig(final Configs configs) { if (configs.getContainerOrchestratorEnabled()) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index 4b5e26425a1a..f1b708549389 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -201,7 +201,8 @@ else if (hasFailed.get()) { final SyncStats totalSyncStats = new SyncStats() .withRecordsEmitted(messageTracker.getTotalRecordsEmitted()) .withBytesEmitted(messageTracker.getTotalBytesEmitted()) - .withStateMessagesEmitted(messageTracker.getTotalStateMessagesEmitted()); + .withSourceStateMessagesEmitted(messageTracker.getTotalSourceStateMessagesEmitted()) + .withDestinationStateMessagesEmitted(messageTracker.getTotalDestinationStateMessagesEmitted()); if (outputStatus == ReplicationStatus.COMPLETED) { totalSyncStats.setRecordsCommitted(totalSyncStats.getRecordsEmitted()); @@ -217,7 +218,8 @@ else if (hasFailed.get()) { final SyncStats syncStats = new SyncStats() .withRecordsEmitted(messageTracker.getStreamToEmittedRecords().get(stream)) .withBytesEmitted(messageTracker.getStreamToEmittedBytes().get(stream)) - .withStateMessagesEmitted(null); // TODO (parker) populate per-stream state messages emitted once supported in V2 + .withSourceStateMessagesEmitted(null) + .withDestinationStateMessagesEmitted(null); if (outputStatus == ReplicationStatus.COMPLETED) { syncStats.setRecordsCommitted(messageTracker.getStreamToEmittedRecords().get(stream)); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java b/airbyte-workers/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java index ad3eecea35eb..e2280ab8edf1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java @@ -86,8 +86,13 @@ public static StandardSync updateConnectionObject(final WorkspaceHelper workspac newConnection.withResourceRequirements(original.getResourceRequirements()); } - // update sync schedule - if (update.getSchedule() != null) { + if (update.getScheduleType() != null) { + newConnection.withScheduleType(update.getScheduleType()); + newConnection.withManual(update.getManual()); + if (update.getScheduleData() != null) { + newConnection.withScheduleData(Jsons.clone(update.getScheduleData())); + } + } else if (update.getSchedule() != null) { final Schedule newSchedule = new Schedule() .withTimeUnit(update.getSchedule().getTimeUnit()) .withUnits(update.getSchedule().getUnits()); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java index 1f603d72ce9f..81d25e1ed091 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java @@ -38,7 +38,8 @@ public class AirbyteMessageTracker implements MessageTracker { private final AtomicReference sourceOutputState; private final AtomicReference destinationOutputState; - private final AtomicLong totalEmittedStateMessages; + private final AtomicLong totalSourceEmittedStateMessages; + private final AtomicLong totalDestinationEmittedStateMessages; private final Map streamToRunningCount; private final HashFunction hashFunction; private final BiMap streamNameToIndex; @@ -71,7 +72,8 @@ public AirbyteMessageTracker() { protected AirbyteMessageTracker(final StateDeltaTracker stateDeltaTracker, final StateAggregator stateAggregator) { this.sourceOutputState = new AtomicReference<>(); this.destinationOutputState = new AtomicReference<>(); - this.totalEmittedStateMessages = new AtomicLong(0L); + this.totalSourceEmittedStateMessages = new AtomicLong(0L); + this.totalDestinationEmittedStateMessages = new AtomicLong(0L); this.streamToRunningCount = new HashMap<>(); this.streamNameToIndex = HashBiMap.create(); this.hashFunction = Hashing.murmur3_32_fixed(); @@ -130,7 +132,7 @@ private void handleSourceEmittedRecord(final AirbyteRecordMessage recordMessage) */ private void handleSourceEmittedState(final AirbyteStateMessage stateMessage) { sourceOutputState.set(new State().withState(stateMessage.getData())); - totalEmittedStateMessages.incrementAndGet(); + totalSourceEmittedStateMessages.incrementAndGet(); final int stateHash = getStateHashCode(stateMessage); try { if (!unreliableCommittedCounts) { @@ -150,6 +152,7 @@ private void handleSourceEmittedState(final AirbyteStateMessage stateMessage) { * committed in the {@link StateDeltaTracker}. Also record this state as the last committed state. */ private void handleDestinationEmittedState(final AirbyteStateMessage stateMessage) { + totalDestinationEmittedStateMessages.incrementAndGet(); stateAggregator.ingest(stateMessage); destinationOutputState.set(stateAggregator.getAggregated()); try { @@ -315,8 +318,13 @@ public Optional getTotalRecordsCommitted() { } @Override - public Long getTotalStateMessagesEmitted() { - return totalEmittedStateMessages.get(); + public Long getTotalSourceStateMessagesEmitted() { + return totalSourceEmittedStateMessages.get(); + } + + @Override + public Long getTotalDestinationStateMessagesEmitted() { + return totalDestinationEmittedStateMessages.get(); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/MessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/MessageTracker.java index 4012bece6266..94266bf0f34c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/MessageTracker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/MessageTracker.java @@ -98,11 +98,13 @@ public interface MessageTracker { Optional getTotalRecordsCommitted(); /** - * Get the overall emitted state message count. + * Get the count of state messages emitted from the source connector. * - * @return returns the total count of emitted state messages. + * @return returns the total count of state messages emitted from the source. */ - Long getTotalStateMessagesEmitted(); + Long getTotalSourceStateMessagesEmitted(); + + Long getTotalDestinationStateMessagesEmitted(); AirbyteTraceMessage getFirstDestinationErrorTraceMessage(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index fa86ddf27a9d..dc1240cdbe05 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.17"; + public static final String NORMALIZATION_VERSION = "0.2.18"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index 72c52a97c180..99227c82cc8f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -97,7 +97,7 @@ public Process create( try { // used to differentiate source and destination processes with the same id and attempt final String podName = ProcessFactory.createProcessName(imageName, jobType, jobId, attempt, KUBE_NAME_LEN_LIMIT); - LOGGER.info("Attempting to start pod = {} for {}", podName, imageName); + LOGGER.info("Attempting to start pod = {} for {} with resources {}", podName, imageName, resourceRequirements); final int stdoutLocalPort = KubePortManagerSingleton.getInstance().take(); LOGGER.info("{} stdoutLocalPort = {}", podName, stdoutLocalPort); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java index e3a82a9649cf..a6cdb57e1d08 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java @@ -30,6 +30,14 @@ @Slf4j public class ConnectionManagerUtils { + private static final ConnectionManagerUtils instance = new ConnectionManagerUtils(); + + private ConnectionManagerUtils() {} + + public static ConnectionManagerUtils getInstance() { + return instance; + } + /** * Attempts to send a signal to the existing ConnectionManagerWorkflow for the provided connection. * @@ -44,9 +52,9 @@ public class ConnectionManagerUtils { * @return the healthy connection manager workflow that was signaled * @throws DeletedWorkflowException if the connection manager workflow was deleted */ - static ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, - final Function signalMethod) + ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, + final UUID connectionId, + final Function signalMethod) throws DeletedWorkflowException { return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty()); } @@ -66,10 +74,10 @@ static ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl * @return the healthy connection manager workflow that was signaled * @throws DeletedWorkflowException if the connection manager workflow was deleted */ - static ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, - final Function> signalMethod, - final T signalArgument) + ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, + final UUID connectionId, + final Function> signalMethod, + final T signalArgument) throws DeletedWorkflowException { return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument)); } @@ -79,10 +87,10 @@ static ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Wo // Keeping this private and only exposing the above methods outside this class provides a strict // type enforcement for external calls, and means this method can assume consistent type // implementations for both cases. - private static ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, - final Function signalMethod, - final Optional signalArgument) + private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, + final UUID connectionId, + final Function signalMethod, + final Optional signalArgument) throws DeletedWorkflowException { try { final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); @@ -129,10 +137,10 @@ private static ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary( } } - static void safeTerminateWorkflow(final WorkflowClient client, final UUID connectionId, final String reason) { - log.info("Attempting to terminate existing workflow for connection {}.", connectionId); + void safeTerminateWorkflow(final WorkflowClient client, final String workflowId, final String reason) { + log.info("Attempting to terminate existing workflow for workflowId {}.", workflowId); try { - client.newUntypedWorkflowStub(getConnectionManagerName(connectionId)).terminate(reason); + client.newUntypedWorkflowStub(workflowId).terminate(reason); } catch (final Exception e) { log.warn( "Could not terminate temporal workflow due to the following error; " @@ -141,7 +149,11 @@ static void safeTerminateWorkflow(final WorkflowClient client, final UUID connec } } - static ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { + void safeTerminateWorkflow(final WorkflowClient client, final UUID connectionId, final String reason) { + safeTerminateWorkflow(client, getConnectionManagerName(connectionId), reason); + } + + ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); final ConnectionUpdaterInput input = buildStartWorkflowInput(connectionId); WorkflowClient.start(connectionManagerWorkflow::run, input); @@ -157,7 +169,7 @@ static ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowCl * @throws DeletedWorkflowException if the workflow was deleted, according to the workflow state * @throws UnreachableWorkflowException if the workflow is in an unreachable state */ - static ConnectionManagerWorkflow getConnectionManagerWorkflow(final WorkflowClient client, final UUID connectionId) + ConnectionManagerWorkflow getConnectionManagerWorkflow(final WorkflowClient client, final UUID connectionId) throws DeletedWorkflowException, UnreachableWorkflowException { final ConnectionManagerWorkflow connectionManagerWorkflow; @@ -193,7 +205,7 @@ static ConnectionManagerWorkflow getConnectionManagerWorkflow(final WorkflowClie return connectionManagerWorkflow; } - static boolean isWorkflowStateRunning(final WorkflowClient client, final UUID connectionId) { + boolean isWorkflowStateRunning(final WorkflowClient client, final UUID connectionId) { try { final ConnectionManagerWorkflow connectionManagerWorkflow = client.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); @@ -203,7 +215,7 @@ static boolean isWorkflowStateRunning(final WorkflowClient client, final UUID co } } - static WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final WorkflowClient workflowClient, final UUID connectionId) { + WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final WorkflowClient workflowClient, final UUID connectionId) { final DescribeWorkflowExecutionRequest describeWorkflowExecutionRequest = DescribeWorkflowExecutionRequest.newBuilder() .setExecution(WorkflowExecution.newBuilder() .setWorkflowId(getConnectionManagerName(connectionId)) @@ -216,25 +228,25 @@ static WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final Workflow return describeWorkflowExecutionResponse.getWorkflowExecutionInfo().getStatus(); } - static long getCurrentJobId(final WorkflowClient client, final UUID connectionId) { + long getCurrentJobId(final WorkflowClient client, final UUID connectionId) { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = ConnectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); return connectionManagerWorkflow.getJobInformation().getJobId(); } catch (final Exception e) { return ConnectionManagerWorkflowImpl.NON_RUNNING_JOB_ID; } } - static ConnectionManagerWorkflow newConnectionManagerWorkflowStub(final WorkflowClient client, final UUID connectionId) { + ConnectionManagerWorkflow newConnectionManagerWorkflowStub(final WorkflowClient client, final UUID connectionId) { return client.newWorkflowStub(ConnectionManagerWorkflow.class, TemporalUtils.getWorkflowOptionsWithWorkflowId(TemporalJobType.CONNECTION_UPDATER, getConnectionManagerName(connectionId))); } - static String getConnectionManagerName(final UUID connectionId) { + String getConnectionManagerName(final UUID connectionId) { return "connection_manager_" + connectionId; } - public static ConnectionUpdaterInput buildStartWorkflowInput(final UUID connectionId) { + public ConnectionUpdaterInput buildStartWorkflowInput(final UUID connectionId) { return ConnectionUpdaterInput.builder() .connectionId(connectionId) .jobId(null) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java index 2689c281e09d..595ee5480433 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java @@ -29,8 +29,12 @@ import io.airbyte.workers.temporal.scheduling.ConnectionManagerWorkflow; import io.airbyte.workers.temporal.spec.SpecWorkflow; import io.airbyte.workers.temporal.sync.SyncWorkflow; +import io.temporal.api.common.v1.WorkflowType; +import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsRequest; import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsResponse; +import io.temporal.api.workflowservice.v1.ListWorkflowExecutionsRequest; +import io.temporal.api.workflowservice.v1.ListWorkflowExecutionsResponse; import io.temporal.client.WorkflowClient; import io.temporal.serviceclient.WorkflowServiceStubs; import java.io.IOException; @@ -51,6 +55,7 @@ import lombok.Builder; import lombok.Value; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; @Slf4j @@ -60,6 +65,7 @@ public class TemporalClient { private final WorkflowClient client; private final WorkflowServiceStubs service; private final StreamResetPersistence streamResetPersistence; + private final ConnectionManagerUtils connectionManagerUtils; /** * This is use to sleep between 2 temporal queries. The query are needed to ensure that the cancel @@ -72,10 +78,20 @@ public TemporalClient(final WorkflowClient client, final Path workspaceRoot, final WorkflowServiceStubs workflowServiceStubs, final StreamResetPersistence streamResetPersistence) { + this(client, workspaceRoot, workflowServiceStubs, streamResetPersistence, ConnectionManagerUtils.getInstance()); + } + + @VisibleForTesting + TemporalClient(final WorkflowClient client, + final Path workspaceRoot, + final WorkflowServiceStubs workflowServiceStubs, + final StreamResetPersistence streamResetPersistence, + final ConnectionManagerUtils connectionManagerUtils) { this.client = client; this.workspaceRoot = workspaceRoot; this.service = workflowServiceStubs; this.streamResetPersistence = streamResetPersistence; + this.connectionManagerUtils = connectionManagerUtils; } /** @@ -170,7 +186,7 @@ public void migrateSyncIfNeeded(final Set connectionIds) { connectionIds.forEach((connectionId) -> { final StopWatch singleSyncMigrationWatch = new StopWatch(); singleSyncMigrationWatch.start(); - if (!isInRunningWorkflowCache(ConnectionManagerUtils.getConnectionManagerName(connectionId))) { + if (!isInRunningWorkflowCache(connectionManagerUtils.getConnectionManagerName(connectionId))) { log.info("Migrating: " + connectionId); try { submitConnectionUpdaterAsync(connectionId); @@ -236,7 +252,8 @@ public Set getAllRunningWorkflows() { public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connectionId) { log.info("Starting the scheduler temporal wf"); - final ConnectionManagerWorkflow connectionManagerWorkflow = ConnectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = + connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); try { CompletableFuture.supplyAsync(() -> { try { @@ -257,7 +274,7 @@ public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connect public void deleteConnection(final UUID connectionId) { try { - ConnectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, connectionManagerWorkflow -> connectionManagerWorkflow::deleteConnection); } catch (final DeletedWorkflowException e) { log.info("Connection {} has already been deleted.", connectionId); @@ -267,7 +284,7 @@ public void deleteConnection(final UUID connectionId) { public void update(final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow; try { - connectionManagerWorkflow = ConnectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + connectionManagerWorkflow = connectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); } catch (final DeletedWorkflowException e) { log.info("Connection {} is deleted, and therefore cannot be updated.", connectionId); return; @@ -275,7 +292,7 @@ public void update(final UUID connectionId) { log.error( String.format("Failed to retrieve ConnectionManagerWorkflow for connection %s. Repairing state by creating new workflow.", connectionId), e); - ConnectionManagerUtils.safeTerminateWorkflow(client, connectionId, + connectionManagerUtils.safeTerminateWorkflow(client, connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); submitConnectionUpdaterAsync(connectionId); return; @@ -297,7 +314,7 @@ public static class ManualOperationResult { public ManualOperationResult startNewManualSync(final UUID connectionId) { log.info("Manual sync request"); - if (ConnectionManagerUtils.isWorkflowStateRunning(client, connectionId)) { + if (connectionManagerUtils.isWorkflowStateRunning(client, connectionId)) { // TODO Bmoric: Error is running return new ManualOperationResult( Optional.of("A sync is already running for: " + connectionId), @@ -305,7 +322,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { } try { - ConnectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync); } catch (final DeletedWorkflowException e) { log.error("Can't sync a deleted connection.", e); return new ManualOperationResult( @@ -321,11 +338,11 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { Optional.of("Didn't managed to start a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (!ConnectionManagerUtils.isWorkflowStateRunning(client, connectionId)); + } while (!connectionManagerUtils.isWorkflowStateRunning(client, connectionId)); log.info("end of manual schedule"); - final long jobId = ConnectionManagerUtils.getCurrentJobId(client, connectionId); + final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); return new ManualOperationResult( Optional.empty(), @@ -335,10 +352,10 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { public ManualOperationResult startNewCancellation(final UUID connectionId) { log.info("Manual cancellation request"); - final long jobId = ConnectionManagerUtils.getCurrentJobId(client, connectionId); + final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); try { - ConnectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); } catch (final DeletedWorkflowException e) { log.error("Can't cancel a deleted workflow", e); return new ManualOperationResult( @@ -354,7 +371,7 @@ public ManualOperationResult startNewCancellation(final UUID connectionId) { Optional.of("Didn't manage to cancel a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (ConnectionManagerUtils.isWorkflowStateRunning(client, connectionId)); + } while (connectionManagerUtils.isWorkflowStateRunning(client, connectionId)); log.info("end of manual cancellation"); @@ -376,10 +393,10 @@ public ManualOperationResult resetConnection(final UUID connectionId, final List } // get the job ID before the reset, defaulting to NON_RUNNING_JOB_ID if workflow is unreachable - final long oldJobId = ConnectionManagerUtils.getCurrentJobId(client, connectionId); + final long oldJobId = connectionManagerUtils.getCurrentJobId(client, connectionId); try { - ConnectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection); } catch (final DeletedWorkflowException e) { log.error("Can't reset a deleted workflow", e); return new ManualOperationResult( @@ -408,7 +425,7 @@ public ManualOperationResult resetConnection(final UUID connectionId, final List } private Optional getNewJobId(final UUID connectionId, final long oldJobId) { - final long currentJobId = ConnectionManagerUtils.getCurrentJobId(client, connectionId); + final long currentJobId = connectionManagerUtils.getCurrentJobId(client, connectionId); if (currentJobId == NON_RUNNING_JOB_ID || currentJobId == oldJobId) { return Optional.empty(); } else { @@ -437,7 +454,7 @@ public ManualOperationResult synchronousResetConnection(final UUID connectionId, Optional.of("Didn't manage to reset a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (ConnectionManagerUtils.getCurrentJobId(client, connectionId) == resetJobId); + } while (connectionManagerUtils.getCurrentJobId(client, connectionId) == resetJobId); log.info("End of reset"); @@ -446,6 +463,73 @@ public ManualOperationResult synchronousResetConnection(final UUID connectionId, Optional.of(resetJobId), Optional.empty()); } + public void restartWorkflowByStatus(final WorkflowExecutionStatus executionStatus) { + final Set workflowExecutionInfos = fetchWorkflowsByStatus(executionStatus); + + final Set nonRunningWorkflow = filterOutRunningWorkspaceId(workflowExecutionInfos); + + nonRunningWorkflow.forEach(connectionId -> { + connectionManagerUtils.safeTerminateWorkflow(client, connectionId, "Terminating workflow in " + + "unreachable state before starting a new workflow for this connection"); + connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + }); + } + + /** + * This should be in the class {@li} + * + * @param workflowId + * @return + */ + Optional extractConnectionIdFromWorkflowId(final String workflowId) { + if (!workflowId.startsWith("connection_manager_")) { + return Optional.empty(); + } + return Optional.ofNullable(StringUtils.removeStart(workflowId, "connection_manager_")) + .map( + stringUUID -> UUID.fromString(stringUUID)); + } + + Set fetchWorkflowsByStatus(final WorkflowExecutionStatus executionStatus) { + ByteString token; + ListWorkflowExecutionsRequest workflowExecutionsRequest = + ListWorkflowExecutionsRequest.newBuilder() + .setNamespace(client.getOptions().getNamespace()) + .build(); + + final Set workflowExecutionInfos = new HashSet<>(); + do { + final ListWorkflowExecutionsResponse listOpenWorkflowExecutionsRequest = + service.blockingStub().listWorkflowExecutions(workflowExecutionsRequest); + final WorkflowType connectionManagerWorkflowType = WorkflowType.newBuilder().setName(ConnectionManagerWorkflow.class.getSimpleName()).build(); + workflowExecutionInfos.addAll(listOpenWorkflowExecutionsRequest.getExecutionsList().stream() + .filter(workflowExecutionInfo -> workflowExecutionInfo.getType() == connectionManagerWorkflowType || + workflowExecutionInfo.getStatus() == executionStatus) + .flatMap((workflowExecutionInfo -> extractConnectionIdFromWorkflowId(workflowExecutionInfo.getExecution().getWorkflowId()).stream())) + .collect(Collectors.toSet())); + token = listOpenWorkflowExecutionsRequest.getNextPageToken(); + + workflowExecutionsRequest = + ListWorkflowExecutionsRequest.newBuilder() + .setNamespace(client.getOptions().getNamespace()) + .setNextPageToken(token) + .build(); + + } while (token != null && token.size() > 0); + + return workflowExecutionInfos; + } + + @VisibleForTesting + Set filterOutRunningWorkspaceId(final Set workflowIds) { + refreshRunningWorkflow(); + + final Set runningWorkflowByUUID = + workflowNames.stream().flatMap(name -> extractConnectionIdFromWorkflowId(name).stream()).collect(Collectors.toSet()); + + return workflowIds.stream().filter(workflowId -> !runningWorkflowByUUID.contains(workflowId)).collect(Collectors.toSet()); + } + private T getWorkflowStub(final Class workflowClass, final TemporalJobType jobType) { return client.newWorkflowStub(workflowClass, TemporalUtils.getWorkflowOptions(jobType)); } @@ -484,7 +568,7 @@ TemporalResponse execute(final JobRunConfig jobRunConfig, final Supplier< @VisibleForTesting boolean isWorkflowReachable(final UUID connectionId) { try { - ConnectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + connectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); return true; } catch (final Exception e) { return false; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index 0e6f1c54f4bf..23caab44ba55 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -560,7 +560,7 @@ private OUTPUT runMandatoryActivityWithOutput(final Function, Except throws IOException { final var jobScope = jobPersistence.getJob(Long.parseLong(jobRunConfig.getJobId())).getScope(); final var connectionId = UUID.fromString(jobScope); - return () -> new NormalizationLauncherWorker( connectionId, destinationLauncherConfig, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index e60ff71409bf..0ca129ae780a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -4,9 +4,12 @@ package io.airbyte.workers.temporal.sync; +import io.airbyte.config.Configs; +import io.airbyte.config.EnvConfigs; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; import io.airbyte.config.OperatorDbtInput; +import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncOperation.OperatorType; @@ -55,11 +58,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, if (syncInput.getOperationSequence() != null && !syncInput.getOperationSequence().isEmpty()) { for (final StandardSyncOperation standardSyncOperation : syncInput.getOperationSequence()) { if (standardSyncOperation.getOperatorType() == OperatorType.NORMALIZATION) { - final NormalizationInput normalizationInput = new NormalizationInput() - .withDestinationConfiguration(syncInput.getDestinationConfiguration()) - .withCatalog(syncOutput.getOutputCatalog()) - .withResourceRequirements(syncInput.getDestinationResourceRequirements()); - + final Configs configs = new EnvConfigs(); + final NormalizationInput normalizationInput = generateNormalizationInput(syncInput, syncOutput, configs); final NormalizationSummary normalizationSummary = normalizationActivity.normalize(jobRunConfig, destinationLauncherConfig, normalizationInput); syncOutput = syncOutput.withNormalizationSummary(normalizationSummary); @@ -80,4 +80,19 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, return syncOutput; } + private NormalizationInput generateNormalizationInput(final StandardSyncInput syncInput, + final StandardSyncOutput syncOutput, + final Configs configs) { + final ResourceRequirements resourceReqs = new ResourceRequirements() + .withCpuRequest(configs.getNormalizationJobMainContainerCpuRequest()) + .withCpuLimit(configs.getNormalizationJobMainContainerCpuLimit()) + .withMemoryRequest(configs.getNormalizationJobMainContainerMemoryRequest()) + .withMemoryLimit(configs.getNormalizationJobMainContainerMemoryLimit()); + + return new NormalizationInput() + .withDestinationConfiguration(syncInput.getDestinationConfiguration()) + .withCatalog(syncOutput.getOutputCatalog()) + .withResourceRequirements(resourceReqs); + } + } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index 6d8a5d890c30..b51e9bad82b6 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -441,7 +441,8 @@ void testPopulatesOutputOnSuccess() throws WorkerException { when(messageTracker.getDestinationOutputState()).thenReturn(Optional.of(new State().withState(expectedState))); when(messageTracker.getTotalRecordsEmitted()).thenReturn(12L); when(messageTracker.getTotalBytesEmitted()).thenReturn(100L); - when(messageTracker.getTotalStateMessagesEmitted()).thenReturn(3L); + when(messageTracker.getTotalSourceStateMessagesEmitted()).thenReturn(3L); + when(messageTracker.getTotalDestinationStateMessagesEmitted()).thenReturn(1L); when(messageTracker.getStreamToEmittedBytes()).thenReturn(Collections.singletonMap(STREAM1, 100L)); when(messageTracker.getStreamToEmittedRecords()).thenReturn(Collections.singletonMap(STREAM1, 12L)); @@ -464,7 +465,8 @@ void testPopulatesOutputOnSuccess() throws WorkerException { .withTotalStats(new SyncStats() .withRecordsEmitted(12L) .withBytesEmitted(100L) - .withStateMessagesEmitted(3L) + .withSourceStateMessagesEmitted(3L) + .withDestinationStateMessagesEmitted(1L) .withRecordsCommitted(12L)) // since success, should use emitted count .withStreamStats(Collections.singletonList( new StreamSyncStats() @@ -473,7 +475,8 @@ void testPopulatesOutputOnSuccess() throws WorkerException { .withBytesEmitted(100L) .withRecordsEmitted(12L) .withRecordsCommitted(12L) // since success, should use emitted count - .withStateMessagesEmitted(null))))) + .withSourceStateMessagesEmitted(null) + .withDestinationStateMessagesEmitted(null))))) .withOutputCatalog(syncInput.getCatalog()) .withState(new State().withState(expectedState)); @@ -540,7 +543,8 @@ void testPopulatesStatsOnFailureIfAvailable() throws Exception { when(messageTracker.getTotalRecordsEmitted()).thenReturn(12L); when(messageTracker.getTotalBytesEmitted()).thenReturn(100L); when(messageTracker.getTotalRecordsCommitted()).thenReturn(Optional.of(6L)); - when(messageTracker.getTotalStateMessagesEmitted()).thenReturn(3L); + when(messageTracker.getTotalSourceStateMessagesEmitted()).thenReturn(3L); + when(messageTracker.getTotalDestinationStateMessagesEmitted()).thenReturn(2L); when(messageTracker.getStreamToEmittedBytes()).thenReturn(Collections.singletonMap(STREAM1, 100L)); when(messageTracker.getStreamToEmittedRecords()).thenReturn(Collections.singletonMap(STREAM1, 12L)); when(messageTracker.getStreamToCommittedRecords()).thenReturn(Optional.of(Collections.singletonMap(STREAM1, 6L))); @@ -559,7 +563,8 @@ void testPopulatesStatsOnFailureIfAvailable() throws Exception { final SyncStats expectedTotalStats = new SyncStats() .withRecordsEmitted(12L) .withBytesEmitted(100L) - .withStateMessagesEmitted(3L) + .withSourceStateMessagesEmitted(3L) + .withDestinationStateMessagesEmitted(2L) .withRecordsCommitted(6L); final List expectedStreamStats = Collections.singletonList( new StreamSyncStats() @@ -568,7 +573,8 @@ void testPopulatesStatsOnFailureIfAvailable() throws Exception { .withBytesEmitted(100L) .withRecordsEmitted(12L) .withRecordsCommitted(6L) - .withStateMessagesEmitted(null))); + .withSourceStateMessagesEmitted(null) + .withDestinationStateMessagesEmitted(null))); assertNotNull(actual); assertEquals(expectedTotalStats, actual.getReplicationAttemptSummary().getTotalStats()); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java index 68e3d304cdb6..df7f1f988b7c 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java @@ -58,7 +58,7 @@ void testGetTotalRecordsStatesAndBytesEmitted() { assertEquals(3, messageTracker.getTotalRecordsEmitted()); assertEquals(3L * Jsons.getEstimatedByteSize(r1.getRecord().getData()), messageTracker.getTotalBytesEmitted()); - assertEquals(2, messageTracker.getTotalStateMessagesEmitted()); + assertEquals(2, messageTracker.getTotalSourceStateMessagesEmitted()); } @Test diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java index 2258e03ecc99..dc979922d58a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java @@ -100,7 +100,8 @@ public void setUp() { normalizationInput = new NormalizationInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) - .withCatalog(syncInput.getCatalog()); + .withCatalog(syncInput.getCatalog()) + .withResourceRequirements(new ResourceRequirements()); operatorDbtInput = new OperatorDbtInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java index c39044fedc8f..c89a82925697 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java @@ -11,6 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -60,6 +61,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; @@ -99,6 +101,7 @@ class TemporalClientTest { private WorkflowServiceStubs workflowServiceStubs; private WorkflowServiceBlockingStub workflowServiceBlockingStub; private StreamResetPersistence streamResetPersistence; + private ConnectionManagerUtils connectionManagerUtils; @BeforeEach void setup() throws IOException { @@ -112,7 +115,8 @@ void setup() throws IOException { when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - temporalClient = spy(new TemporalClient(workflowClient, workspaceRoot, workflowServiceStubs, streamResetPersistence)); + connectionManagerUtils = spy(ConnectionManagerUtils.class); + temporalClient = spy(new TemporalClient(workflowClient, workspaceRoot, workflowServiceStubs, streamResetPersistence, connectionManagerUtils)); } @Nested @@ -288,9 +292,9 @@ void migrateCalled() { final UUID migratedId = UUID.randomUUID(); doReturn(false) - .when(temporalClient).isInRunningWorkflowCache(ConnectionManagerUtils.getConnectionManagerName(nonMigratedId)); + .when(temporalClient).isInRunningWorkflowCache(ConnectionManagerUtils.getInstance().getConnectionManagerName(nonMigratedId)); doReturn(true) - .when(temporalClient).isInRunningWorkflowCache(ConnectionManagerUtils.getConnectionManagerName(migratedId)); + .when(temporalClient).isInRunningWorkflowCache(ConnectionManagerUtils.getInstance().getConnectionManagerName(migratedId)); doNothing() .when(temporalClient).refreshRunningWorkflow(); @@ -730,6 +734,40 @@ void testResetConnectionDeletedWorkflow() throws IOException { } + @Nested + class RestartPerStatus { + + private ConnectionManagerUtils mConnectionManagerUtils; + + @BeforeEach + public void init() throws IOException { + mConnectionManagerUtils = mock(ConnectionManagerUtils.class); + + final Path workspaceRoot = Files.createTempDirectory(Path.of("/tmp"), "temporal_client_test"); + temporalClient = spy(new TemporalClient(workflowClient, workspaceRoot, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils)); + } + + @Test + void testRestartFailed() { + final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); + + when(workflowClient.newWorkflowStub(any(), anyString())).thenReturn(mConnectionManagerWorkflow); + final UUID connectionId = UUID.fromString("ebbfdc4c-295b-48a0-844f-88551dfad3db"); + final Set workflowIds = Set.of(connectionId); + + doReturn(workflowIds) + .when(temporalClient).fetchWorkflowsByStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); + doReturn(workflowIds) + .when(temporalClient).filterOutRunningWorkspaceId(workflowIds); + mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); + temporalClient.restartWorkflowByStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); + verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(workflowClient), eq(connectionId), + anyString()); + verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId)); + } + + } + @Test @DisplayName("Test manual operation on quarantined workflow causes a restart") void testManualOperationOnQuarantinedWorkflow() { diff --git a/build.gradle b/build.gradle index 0615fa7671c6..1fedfe594eab 100644 --- a/build.gradle +++ b/build.gradle @@ -239,6 +239,9 @@ subprojects { maven { url 'https://jitpack.io' } + maven { + url 'https://maven.twttr.com' + } } pmd { diff --git a/charts/airbyte-bootloader/values.yaml b/charts/airbyte-bootloader/values.yaml index b521f0cf5254..838cd4a8e740 100644 --- a/charts/airbyte-bootloader/values.yaml +++ b/charts/airbyte-bootloader/values.yaml @@ -60,4 +60,5 @@ resources: ## @param bootloader.affinity [object] Affinity and anti-affinity for bootloader pod assignment. ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity ## -affinity: {} \ No newline at end of file +affinity: {} +extraEnv: [] diff --git a/charts/airbyte-temporal/templates/deployment.yaml b/charts/airbyte-temporal/templates/deployment.yaml index ff2120146a8e..f74699092d87 100644 --- a/charts/airbyte-temporal/templates/deployment.yaml +++ b/charts/airbyte-temporal/templates/deployment.yaml @@ -61,7 +61,10 @@ spec: name: {{ .Values.global.database.secretName | default (printf "%s-postgresql" .Release.Name ) }} key: {{ .Values.global.database.secretValue | default "postgresql-password" }} - name: POSTGRES_SEEDS - value: {{ .Release.Name }}-postgresql + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: DATABASE_HOST - name: DYNAMIC_CONFIG_FILE_PATH value: "config/dynamicconfig/development.yaml" {{- end }} diff --git a/charts/airbyte-webapp/templates/ingress.yaml b/charts/airbyte-webapp/templates/ingress.yaml index 2c0f29ecdb04..785333a8a77c 100644 --- a/charts/airbyte-webapp/templates/ingress.yaml +++ b/charts/airbyte-webapp/templates/ingress.yaml @@ -54,11 +54,11 @@ spec: backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ $fullName }}-webapp + name: {{ $fullName }} port: number: {{ $svcPort }} {{- else }} - serviceName: {{ $fullName }}-webapp + serviceName: {{ $fullName }} servicePort: {{ $svcPort }} {{- end }} {{- end }} diff --git a/charts/airbyte/templates/gcs-log-creds-secret.yaml b/charts/airbyte/templates/gcs-log-creds-secret.yaml index fd494d52acbe..4848f4c20f5e 100644 --- a/charts/airbyte/templates/gcs-log-creds-secret.yaml +++ b/charts/airbyte/templates/gcs-log-creds-secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ include "common.names.fullname" . }}-gcs-log-creds + name: {{ .Release.Name }}-gcs-log-creds type: Opaque data: gcp.json: "{{ .Values.global.logs.gcs.credentialsJson }}" diff --git a/deps.toml b/deps.toml index a47ebcbc3a57..9c4bc88b157f 100644 --- a/deps.toml +++ b/deps.toml @@ -9,6 +9,8 @@ slf4j = "1.7.30" lombok = "1.18.22" jooq = "3.13.4" junit-jupiter = "5.8.2" +micronaut = "3.6.0" +micronaut-test = "3.5.0" postgresql = "42.3.5" connectors-testcontainers = "1.15.3" connectors-testcontainers-cassandra = "1.16.0" @@ -40,6 +42,7 @@ log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4 jul-to-slf4j = { module = "org.slf4j:jul-to-slf4j", version.ref = "slf4j" } jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } +javax-databind = { module = "javax.xml.bind:jaxb-api", version = "2.4.0-b180830.0359" } jooq = { module = "org.jooq:jooq", version.ref = "jooq" } jooq-codegen = { module = "org.jooq:jooq-codegen", version.ref = "jooq" } jooq-meta = { module = "org.jooq:jooq-meta", version.ref = "jooq" } @@ -90,9 +93,36 @@ micrometer-statsd = {module = "io.micrometer:micrometer-registry-statsd", versio quartz-scheduler = {module="org.quartz-scheduler:quartz", version = "2.3.2"} +# Micronaut-related dependencies +h2-database = { module = "com.h2database:h2", version = "2.1.214" } +hibernate-types = { module = "com.vladmihalcea:hibernate-types-52", version = "2.16.3" } +javax-inject = { module = "javax.inject:javax.inject", version = "1" } +javax-transaction = { module = "javax.transaction:javax.transaction-api", version = "1.3" } +micronaut-bom = { module = "io.micronaut:micronaut-bom", version.ref = "micronaut" } +micronaut-data-processor = { module = "io.micronaut.data:micronaut-data-processor", version = "3.7.2" } +micronaut-flyway = { module = "io.micronaut.flyway:micronaut-flyway", version = "5.4.0" } +micronaut-inject = { module = "io.micronaut:micronaut-inject" } +micronaut-http-client = { module = "io.micronaut:micronaut-http-client" } +micronaut-http-server-netty = { module = "io.micronaut:micronaut-http-server-netty" } +micronaut-inject-java = { module = "io.micronaut:micronaut-inject-java" } +micronaut-jaxrs-processor = { module = "io.micronaut.jaxrs:micronaut-jaxrs-processor", version = "3.4.0" } +micronaut-jaxrs-server = { module = "io.micronaut.jaxrs:micronaut-jaxrs-server", version = "3.4.0" } +micronaut-jdbc-hikari = { module = "io.micronaut.sql:micronaut-jdbc-hikari" } +micronaut-jooq = { module = "io.micronaut.sql:micronaut-jooq" } +micronaut-management = { module = "io.micronaut:micronaut-management" } +micronaut-runtime = { module = "io.micronaut:micronaut-runtime" } +micronaut-security = { module = "io.micronaut.security:micronaut-security", version = "3.6.3" } +micronaut-test-core = { module = "io.micronaut.test:micronaut-test-core", version.ref = "micronaut-test" } +micronaut-test-junit5 = { module = "io.micronaut.test:micronaut-test-junit5", version.ref = "micronaut-test" } +micronaut-validation = { module = "io.micronaut:micronaut-validation" } + [bundles] jackson = ["jackson-databind", "jackson-annotations", "jackson-dataformat", "jackson-datatype"] apache = ["apache-commons", "apache-commons-lang"] log4j = ["log4j-api", "log4j-core", "log4j-impl", "log4j-web"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] junit = ["junit-jupiter-api", "junit-jupiter-params", "mockito-junit-jupiter"] +micronaut = ["javax-inject", "javax-transaction", "micronaut-http-server-netty", "micronaut-http-client", "micronaut-inject", "micronaut-validation", "micronaut-runtime", "micronaut-management", "micronaut-security", "micronaut-jaxrs-server", "micronaut-flyway", "micronaut-jdbc-hikari", "micronaut-jooq"] +micronaut-annotation-processor = ["micronaut-inject-java", "micronaut-management", "micronaut-validation", "micronaut-data-processor", "micronaut-jaxrs-processor"] +micronaut-test = ["micronaut-test-core", "micronaut-test-junit5", "h2-database"] +micronaut-test-annotation-processor = ["micronaut-inject-java"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 13566de5e479..61502aa9a1c6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -77,6 +77,10 @@ services: - MAX_DISCOVER_WORKERS=${MAX_DISCOVER_WORKERS} - MAX_SPEC_WORKERS=${MAX_SPEC_WORKERS} - MAX_SYNC_WORKERS=${MAX_SYNC_WORKERS} + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT} + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST} + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT} + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST} - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS} - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS} diff --git a/docs/assets/docs/save_actions_settings.png b/docs/assets/docs/save_actions_settings.png new file mode 100644 index 000000000000..c7022c12b754 Binary files /dev/null and b/docs/assets/docs/save_actions_settings.png differ diff --git a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md index b3b99d1310b3..c80a4d968d0c 100644 --- a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md @@ -121,6 +121,8 @@ Verifies when a discover operation is run on the connector using the given confi | `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | | `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | | `timeout_seconds` | int | 30 | Test execution timeout in seconds | +| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests (expects a version following semantic versioning). | +| `backward_compatibility_tests_config.disable_for_version` | string | None | Disable the backward compatibility test for a specific version (expects a version following semantic versioning). | ## Test Basic Read diff --git a/docs/contributing-to-airbyte/code-style.md b/docs/contributing-to-airbyte/code-style.md index 735bd39d7f97..e26305bb0cfe 100644 --- a/docs/contributing-to-airbyte/code-style.md +++ b/docs/contributing-to-airbyte/code-style.md @@ -15,20 +15,20 @@ Install it in IntelliJ: 1. `Import Scheme > IntelliJ IDEA code style XML` 2. Select the file we just downloaded 3. Select `GoogleStyle` in the dropdown -4. Change default `Hard wrap at` in `Wrapping and Braces` tab to **150**. -5. We prefer `import foo.bar.ClassName` over `import foo.bar.*`. Even in cases where we import multiple classes from the same package. This can be set by going to `Preferences > Code Style > Java > Imports` and changing `Class count to use import with '*'` to 9999 and \`Names count to use static import with '\*' to 9999. -6. We add the `final` keyword wherever possible. It's a drag to have to do it manually, however, so we set up the IDE to do it for us. You can either set this as the default for your IDE or you can set it just for the Airbyte project(s) that you are using. - 1. Turn on the inspection. Go into IntelliJ Preferences... - 1. Editor > Inspections > Search (with the quotation marks included) "Field may be 'final'" > check the box - 2. Editor > Inspections > Search "local variable or parameter can be final" > check the box - 3. Apply the changes. - 2. Turn on the auto add final. Go into IntelliJ Preferences... - 1. Plugins - install Save Actions if not already installed. - 2. Go to Save Actions in the preferences left nav (NOT Tools > Actions on Save -- that is a different tool) - 1. Activate save actions on save > check the box - 2. Active save actions on shortcut > check the box - 3. Activate save actions on batch > check the box - 4. Add final modifier to field > check the box - 5. Add final modifier to local variable or parameter > check the box - 6. Apply the changes. +4. Change default `Hard wrap at` in `Wrapping and Braces` tab to **150** +5. Use explicit imports (example: `import foo.bar.ClassName` over `import foo.bar.*`) even when importing multiple classes from the same package. This can be set by going to `Preferences > Code Style > Java > Imports` and changing `Class count to use import with '*'` to `9999` and `Names count to use static import with '\*'` to `9999` +6. Add the `final` keyword wherever possible. You can either set this as the default for your IDE or you can set it just for the Airbyte project(s) that you are using + 1. Turn on the inspection. Go into `Preferences > Editor > Inspections` + 1. Search `"Field may be 'final'"` > check the box + 2. Search `"local variable or parameter can be 'final'"` > check the box + 3. Apply the changes + 2. Turn on the auto add final. Go into IntelliJ Preferences + 1. Plugins - install Save Actions if not already installed + 2. Go to Save Actions in the preferences [left navigation column](../assets/docs/save_actions_settings.png) (NOT Tools > Actions on Save -- that is a different tool) + 1. `Activate save actions on save` > check the box + 2. `Active save actions on shortcut` > check the box + 3. `Activate save actions on batch` > check the box + 4. `Add final modifier to field` > check the box + 5. `Add final modifier to local variable or parameter` > check the box + 6. Apply the changes 7. You're done! diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 27a819813d78..9aee5c9f310e 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -88,6 +88,7 @@ For more information about the grading system, see [Product Release Stages](http | [Lemlist](sources/lemlist.md) | Alpha | Yes | | [Lever](sources/lever-hiring.md) | Alpha | No | | [LinkedIn Ads](sources/linkedin-ads.md) | Generally Available | Yes | +| [LinkedIn Pages](sources/linkedin-pages.md) | Alpha | Yes | | [Linnworks](sources/linnworks.md) | Alpha | Yes | | [Looker](sources/looker.md) | Alpha | Yes | | [Magento](sources/magento.md) | Alpha | No | diff --git a/docs/integrations/destinations/azureblobstorage.md b/docs/integrations/destinations/azureblobstorage.md index 52ad7dd8d735..d36470b1d79d 100644 --- a/docs/integrations/destinations/azureblobstorage.md +++ b/docs/integrations/destinations/azureblobstorage.md @@ -6,6 +6,9 @@ This destination writes data to Azure Blob Storage. The Airbyte Azure Blob Storage destination allows you to sync data to Azure Blob Storage. Each stream is written to its own blob under the container. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your AzureBlobStorage connector to version `0.1.6` or newer + ## Sync Mode | Feature | Support | Notes | diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 3dfbe059f1d7..a5d99d6d0df3 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -6,6 +6,7 @@ This page guides you through setting up the BigQuery destination connector. ## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your BigQuery connector to version `1.1.14` or newer - [A Google Cloud project with BigQuery enabled](https://cloud.google.com/bigquery/docs/quickstarts/query-public-dataset-console) - [A BigQuery dataset](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui#create_a_dataset) to sync data to. diff --git a/docs/integrations/destinations/cassandra.md b/docs/integrations/destinations/cassandra.md index 2280daf9da0a..f92fd58df912 100644 --- a/docs/integrations/destinations/cassandra.md +++ b/docs/integrations/destinations/cassandra.md @@ -1,5 +1,9 @@ # Cassandra +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Cassandra connector to version `0.1.3` or newer + + ## Sync overview ### Output schema diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md index b02293b2bd30..7ca194666b56 100644 --- a/docs/integrations/destinations/dynamodb.md +++ b/docs/integrations/destinations/dynamodb.md @@ -4,6 +4,9 @@ This destination writes data to AWS DynamoDB. The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each stream is written to its own table under the DynamoDB. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your DynamoDB connector to version `0.1.5` or newer + ## Sync overview ### Output schema diff --git a/docs/integrations/destinations/kafka.md b/docs/integrations/destinations/kafka.md index 4e70da2a3d43..cf58ca50a5d2 100644 --- a/docs/integrations/destinations/kafka.md +++ b/docs/integrations/destinations/kafka.md @@ -4,6 +4,10 @@ The Airbyte Kafka destination allows you to sync data to Kafka. Each stream is written to the corresponding Kafka topic. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Kafka connector to version `0.1.10` or newer + + ### Sync overview #### Output schema diff --git a/docs/integrations/destinations/keen.md b/docs/integrations/destinations/keen.md index 55bc286c2459..55aeede379e9 100644 --- a/docs/integrations/destinations/keen.md +++ b/docs/integrations/destinations/keen.md @@ -9,6 +9,10 @@ description: >- The Airbyte Keen destination allows you to stream data from any Airbyte Source into [Keen](https://keen.io?utm_campaign=Airbyte%20Destination%20Connector&utm_source=Airbyte%20Hosted%20Docs&utm_medium=Airbyte%20Hosted%20Docs&utm_term=Airbyte%20Hosted%20Docs&utm_content=Airbyte%20Hosted%20Docs) for storage, analysis, and visualization. Keen is a flexible, fully managed event streaming and analytics platform that empowers anyone to ship custom, embeddable dashboards in minutes, not months. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Keen connector to version `0.2.4` or newer + + ### Sync overview #### Output schema diff --git a/docs/integrations/destinations/kinesis.md b/docs/integrations/destinations/kinesis.md index 6a3ca281b379..c6788dc8f7f8 100644 --- a/docs/integrations/destinations/kinesis.md +++ b/docs/integrations/destinations/kinesis.md @@ -1,5 +1,8 @@ # Kinesis +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Kinesis connector to version `0.1.4` or newer + ## Sync overview diff --git a/docs/integrations/destinations/mongodb.md b/docs/integrations/destinations/mongodb.md index 7b10c1c55f9d..fb806e2b38bc 100644 --- a/docs/integrations/destinations/mongodb.md +++ b/docs/integrations/destinations/mongodb.md @@ -9,6 +9,10 @@ | Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. | | Namespaces | Yes | | +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your MongoDB connector to version `0.1.6` or newer + + ## Output Schema for `destination-mongodb` Each stream will be output into its own collection in MongoDB. Each collection will contain 3 fields: diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md index 61a0f24bac29..b9611b016741 100644 --- a/docs/integrations/destinations/mqtt.md +++ b/docs/integrations/destinations/mqtt.md @@ -4,6 +4,9 @@ The Airbyte MQTT destination allows you to sync data to any MQTT system compliance with version 3.1.X. Each stream is written to the corresponding MQTT topic. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your MQTT connector to the latest version + ### Sync overview #### Output schema diff --git a/docs/integrations/destinations/pubsub.md b/docs/integrations/destinations/pubsub.md index 804668afc2d6..8ae20f6df7cb 100644 --- a/docs/integrations/destinations/pubsub.md +++ b/docs/integrations/destinations/pubsub.md @@ -10,6 +10,9 @@ description: >- The Airbyte Google PubSub destination allows you to send/stream data into PubSub. Pub/Sub is an asynchronous messaging service provided by Google Cloud Provider. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your PubSub connector to version `0.1.6` or newer + ### Sync overview #### Output schema diff --git a/docs/integrations/destinations/pulsar.md b/docs/integrations/destinations/pulsar.md index c6279745c7ec..3921c3b2cae9 100644 --- a/docs/integrations/destinations/pulsar.md +++ b/docs/integrations/destinations/pulsar.md @@ -4,6 +4,9 @@ The Airbyte Pulsar destination allows you to sync data to Pulsar. Each stream is written to the corresponding Pulsar topic. +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Pulsar connector to version `0.1.3` or newer + ### Sync overview #### Output schema diff --git a/docs/integrations/destinations/rockset.md b/docs/integrations/destinations/rockset.md index 28f1ce653fc5..635ee2f7d332 100644 --- a/docs/integrations/destinations/rockset.md +++ b/docs/integrations/destinations/rockset.md @@ -1,5 +1,7 @@ # Rockset +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Rockset connector to version `0.1.4` or newer ## Features | Feature | Support | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index b021c4427067..22e9d9c4c94d 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -320,6 +320,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.13 | 2022-08-09 | [\#15394](https://github.com/airbytehq/airbyte/pull/15394) | Added LZO compression support to Parquet format | | 0.3.12 | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.3.11 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | | 0.3.10 | 2022-06-30 | [\#14332](https://github.com/airbytehq/airbyte/pull/14332) | Change INSTANCE_PROFILE to use `AWSDefaultProfileCredential`, which supports more authentications on AWS | diff --git a/docs/integrations/destinations/scylla.md b/docs/integrations/destinations/scylla.md index 386a1fd418aa..dfd9a621b702 100644 --- a/docs/integrations/destinations/scylla.md +++ b/docs/integrations/destinations/scylla.md @@ -1,7 +1,9 @@ # Scylla -## Sync overview +## Prerequisites +- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Scylla connector to version `0.1.3` or newer +## Sync overview ### Output schema The incoming airbyte data is structured in keyspaces and tables and is partitioned and replicated across different nodes diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index 95fdc78185f0..b896e0117512 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -126,8 +126,11 @@ In order to read large files from a remote location, this connector uses the [sm ## Changelog | Version | Date | Pull Request | Subject | + |---------|------------|----------------------------------------------------------| ------------------------------------------------- | -| 0.2.16 | 2022-08-11 | [00000](https://github.com/airbytehq/airbyte/pull/0000) | Specify `pyxlsb` library as engine for Excel file parsing | +| 0.2.18 | 2022-08-11 | [00000](https://github.com/airbytehq/airbyte/pull/0000) | Specify `pyxlsb` library as engine for Excel file parsing | +| 0.2.17 | 2022-08-11 | [15501](https://github.com/airbytehq/airbyte/pull/15501) | Cache binary stream to file | +| 0.2.16 | 2022-08-10 | [15293](https://github.com/airbytehq/airbyte/pull/15293) | added support for encoding reader option | | 0.2.15 | 2022-08-05 | [15269](https://github.com/airbytehq/airbyte/pull/15269) | Bump `smart-open` version to 6.0.0 | | 0.2.12 | 2022-07-12 | [14535](https://github.com/airbytehq/airbyte/pull/14535) | Fix invalid schema generation for JSON files | | 0.2.11 | 2022-07-12 | [9974](https://github.com/airbytehq/airbyte/pull/14588) | Add support to YAML format | diff --git a/docs/integrations/sources/flexport.md b/docs/integrations/sources/flexport.md index a15ee0c247f3..db8b15a7ec53 100644 --- a/docs/integrations/sources/flexport.md +++ b/docs/integrations/sources/flexport.md @@ -46,4 +46,5 @@ Authentication uses a pre-created API token which can be [created in the UI](htt | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.1 | 2022-07-26 | [15033](https://github.com/airbytehq/airbyte/pull/15033) | Source Flexport: Update schemas | | 0.1.0 | 2021-12-14 | [8777](https://github.com/airbytehq/airbyte/pull/8777) | New Source: Flexport | diff --git a/docs/integrations/sources/freshcaller.md b/docs/integrations/sources/freshcaller.md new file mode 100644 index 000000000000..ff79efce6e6a --- /dev/null +++ b/docs/integrations/sources/freshcaller.md @@ -0,0 +1,43 @@ +# Freshcaller + +## Overview + +The Freshcaller source supports full refresh and incremental sync. Depending on your needs, one could choose appropriate sync mode - `full refresh` replicates all records every time a sync happens where as `incremental` replicates net-new records since the last successful sync. + +### Output schema + +The following endpoints are supported from this source: + +* [Users](https://developers.freshcaller.com/api/#users) +* [Teams](https://developers.freshcaller.com/api/#teams) +* [Calls](https://developers.freshcaller.com/api/#calls) +* [Call Metrics](https://developers.freshcaller.com/api/#call-metrics) + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| SSL connection | Yes | +| Namespaces | No | + +### Performance considerations + +The Freshcaller connector should not run into Freshcaller API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Freshcaller Account +* Freshcaller API Key + +### Setup guide + +Please read [How to find your API key](https://support.freshdesk.com/en/support/solutions/articles/225435-where-can-i-find-my-api-key-). + +## Changelog +| 0.1.0 | 2022-08-11 | [14759](https://github.com/airbytehq/airbyte/pull/14759) | 🎉 New Source: Freshcaller | \ No newline at end of file diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 9b34220f4251..03b96c6a2c18 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -141,6 +141,7 @@ The GitHub connector should not run into GitHub API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------| +| 0.2.45 | 2022-08-11 | [15420](https://github.com/airbytehq/airbyte/pull/15420) | "User" object can be "null" | | 0.2.44 | 2022-08-01 | [14795](https://github.com/airbytehq/airbyte/pull/14795) | Use GraphQL for `pull_request_comment_reactions` stream | | 0.2.43 | 2022-07-26 | [15049](https://github.com/airbytehq/airbyte/pull/15049) | Bugfix schemas for streams `deployments`, `workflow_runs`, `teams` | | 0.2.42 | 2022-07-12 | [14613](https://github.com/airbytehq/airbyte/pull/14613) | Improve schema for stream `pull_request_commits` added "null" | diff --git a/docs/integrations/sources/greenhouse.md b/docs/integrations/sources/greenhouse.md index 735b54d2b428..6013093e32f4 100644 --- a/docs/integrations/sources/greenhouse.md +++ b/docs/integrations/sources/greenhouse.md @@ -55,9 +55,10 @@ Please follow the [Greenhouse documentation for generating an API key](https://d ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | -| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | -| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | -| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | added identification of accessible streams for API keys with limited permissions | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------| +| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | +| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | +| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | +| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | +| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | added identification of accessible streams for API keys with limited permissions | diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index 95a78a3f6d27..5208b0dbcd67 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -84,6 +84,7 @@ The Harvest connector will gracefully handle rate limits. For more information, | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.10 | 2022-08-08 | [15221](https://github.com/airbytehq/airbyte/pull/15221) | Added `parent_id` for all streams which have parent stream | | 0.1.9 | 2022-08-04 | [15312](https://github.com/airbytehq/airbyte/pull/15312) | Fix `started_time` and `ended_time` format schema error and updated report slicing | | 0.1.8 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | | 0.1.6 | 2021-11-14 | [7952](https://github.com/airbytehq/airbyte/pull/7952) | Implement OAuth 2.0 support | diff --git a/docs/integrations/sources/hubplanner.md b/docs/integrations/sources/hubplanner.md new file mode 100644 index 000000000000..e6e1e2648edf --- /dev/null +++ b/docs/integrations/sources/hubplanner.md @@ -0,0 +1,42 @@ +# Hubplanner + +Hubplanner is a tool to plan, schedule, report and manage your entire team. + +## Prerequisites +* Create the API Key to access your data in Hubplanner. + +## Airbyte OSS +* API Key + +## Airbyte Cloud +* Comming Soon. + + +## Setup guide +### For Airbyte OSS: + +1. Access https://your-domain.hubplanner.com/settings#api or access the panel in left side Integrations/Hub Planner API +2. Click in Generate Key + +## Supported sync modes + +The Okta source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + +## Supported Streams + +- [Billing Rates](https://github.com/hubplanner/API/blob/master/Sections/billingrate.md) +- [Bookings](https://github.com/hubplanner/API/blob/master/Sections/bookings.md) +- [Clients](https://github.com/hubplanner/API/blob/master/Sections/clients.md) +- [Events](https://github.com/hubplanner/API/blob/master/Sections/events.md) +- [Holidays](https://github.com/hubplanner/API/blob/master/Sections/holidays.md) +- [Projects](https://github.com/hubplanner/API/blob/master/Sections/project.md) +- [Resources](https://github.com/hubplanner/API/blob/master/Sections/resource.md) + + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| + +| 0.1.0 | 2021-08-10 | [12145](https://github.com/airbytehq/airbyte/pull/12145) | Initial Release | diff --git a/docs/integrations/sources/linkedin-pages.md b/docs/integrations/sources/linkedin-pages.md new file mode 100644 index 000000000000..5801bcfff1f3 --- /dev/null +++ b/docs/integrations/sources/linkedin-pages.md @@ -0,0 +1,113 @@ +# LinkedIn Pages + +## Sync overview + +The LinkedIn Pages source only supports Full Refresh for now. Incremental Sync will be coming soon. + +This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). Airbyte uses [LinkedIn Marketing Developer Platform - API](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/marketing-integrations-overview) to fetch data from LinkedIn Pages. + +### Output schema + +This Source is capable of syncing the following data as streams: + +* [Organization Lookup](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/organization-lookup-api?tabs=http#retrieve-organizations) +* [Follower Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/follower-statistics?tabs=http#retrieve-lifetime-follower-statistics) +* [Page Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/page-statistics?tabs=http#retrieve-lifetime-organization-page-statistics) +* [Share Statistics](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/share-statistics?tabs=http#retrieve-lifetime-share-statistics) +* [Shares (Latest 50)](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/share-api?tabs=http#find-shares-by-owner) +* [Total Follower Count](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/organizations/organization-lookup-api?tabs=http#retrieve-organization-follower-count) +* [UGC Posts](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/ugc-post-api?tabs=http#find-ugc-posts-by-authors) + +### NOTE: + +All streams only sync all-time statistics at this time. A `start_date` field will be added soon to pull data starting at a single point in time. + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--------------- | :----------- | :------------------------- | +| `number` | `number` | float number | +| `integer` | `integer` | whole number | +| `array` | `array` | | +| `boolean` | `boolean` | True/False | +| `string` | `string` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------------------------------- | :------------------- | :---- | +| Full Refresh Overwrite Sync | Yes | | +| Full Refresh Append Sync | No | | +| Incremental - Append Sync | No | | +| Incremental - Append + Deduplication Sync | No | | +| Namespaces | No | | + +### Performance considerations + +There are official Rate Limits for LinkedIn Pages API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. Rate limits specify the maximum number of API calls that can be made in a 24 hour period. These limits reset at midnight UTC every day. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message: + +```text +"Caught retryable error ' or null' after tries. Waiting seconds then retrying..." +``` + +This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Error. If the maximum of available API requests capacity is reached, you will have the following message: + +```text +"Max try rate limit exceded..." +``` + +After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose your app > Analytics. + +## Getting started +The API user account should be assigned the following permissions for the API endpoints: +Endpoints such as: `Organization Lookup API`, `Follower Statistics`, `Page Statistics`, `Share Statistics`, `Shares`, `UGC Posts` require these permissions: +* `r_organization_social`: Retrieve your organization's posts, comments, reactions, and other engagement data. +* `rw_organization_admin`: Manage your organization's pages and retrieve reporting data. + +The API user account should be assigned the `ADMIN` role. + +### Authentication +There are 2 authentication methods: Access Token or OAuth2.0. +OAuth2.0 is recommended since it will continue streaming data for 12 months instead of 2 months with an access token. + +##### Create the `Refresh_Token` or `Access_Token`: +The source LinkedIn Pages can use either the `client_id`, `client_secret` and `refresh_token` for OAuth2.0 authentication or simply use an `access_token` in the UI connector's settings to make API requests. Access tokens expire after `2 months from creation date (60 days)` and require a user to manually authenticate again. Refresh tokens expire after `12 months from creation date (365 days)`. If you receive a `401 invalid token response`, the error logs will state that your token has expired and to re-authenticate your connection to generate a new token. This is described more [here](https://docs.microsoft.com/en-us/linkedin/shared/authentication/authorization-code-flow?context=linkedin/context). + +1. **Log in to LinkedIn as the API user** + +2. **Create an App** [here](https://www.linkedin.com/developers/apps): + * `App Name`: airbyte-source + * `Company`: search and find your LinkedIn Company Page + * `Privacy policy URL`: link to company privacy policy + * `Business email`: developer/admin email address + * `App logo`: Airbyte's \(or Company's\) logo + * Review/agree to legal terms and create app + * Review the **Auth** tab: + * **Save your `client_id` and `client_secret`** \(for later steps\) + * Oauth 2.0 settings: Provide a `redirect_uri` \(for later steps\): `https://airbyte.io` + +3. **Verify App**: + * In the **Settings** tab of your app dashboard, you'll see a **Verify** button. Click that button! + * Generate and provide the verify URL to your Company's LinkedIn Admin to verify the app. + +4. **Request API Access**: + * Navigate to the **Products** tab + * Select the [Marketing Developer Platform](https://docs.microsoft.com/en-us/linkedin/marketing/) and agree to the legal terms + * After a few minutes, refresh the page to see a link to `View access form` in place of the **Select** button + * Fill out the access form and access should be granted **within 72 hours** (usually quicker) + +5. **Create A Refresh Token** (or Access Token): + * Navigate to the LinkedIn Developers' [OAuth Token Tools](https://www.linkedin.com/developers/tools/oauth) and click **Create token** + * Select your newly created app and check the boxes for the following scopes: + * `r_organization_social` + * `rw_organization_admin` + * Click **Request access token** and once generated, **save your Refresh token** + +6. **Use the `client_id`, `client_secret` and `refresh_token`** from Steps 2 and 5 to autorize the LinkedIn Pages connector within the Airbyte UI. + * As mentioned earlier, you can also simply use the Access token auth method for 60-day access. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | +| 0.1.0 | 2022-08-11 | [13098](https://github.com/airbytehq/airbyte/pull/13098) | Initial Release | \ No newline at end of file diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 6c41a9509308..56aa0f4bf636 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -306,6 +306,8 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------------------------------------------------- |:-------------------------------------------------------------------------------------------------------| +| 0.4.15 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | +| 0.4.14 | 2022-08-10 | [15430](https://github.com/airbytehq/airbyte/pull/15430) | fixed a bug on handling special character on database name | 0.4.13 | 2022-08-04 | [15268](https://github.com/airbytehq/airbyte/pull/15268) | Added [] enclosing to escape special character in the database name | | 0.4.12 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.4.11 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 537e0338005d..c54d5eed9ba5 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -185,6 +185,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| +| 0.6.2 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | | 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | | 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 1f66efb72ac3..4019f5c6274a 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -15,6 +15,7 @@ If your dataset is small and you just want a snapshot of your table in the desti ## Prerequisites +- For Airbyte OSS users, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer - Use Postgres v9.3.x or above for non-CDC workflows and Postgres v10 or above for CDC workflows - Allowlist the IP address `34.106.109.131` to enable access to Airbyte - For Airbyte Cloud (and optionally for Airbyte OSS), ensure SSL is enabled in your environment @@ -371,18 +372,21 @@ Possible solutions include: | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------------------| -| 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available | +| 1.0.2 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | +| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | +| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | +| 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available (requires [upgrading](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to `v0.40.0-alpha`) | | 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | | 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | | 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | | 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | -| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently | +| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | | 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | | | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | | 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | | 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | +| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | | 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | | 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | | 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md index 499cc62f3e41..6a5379aee46e 100644 --- a/docs/integrations/sources/posthog.md +++ b/docs/integrations/sources/posthog.md @@ -55,6 +55,7 @@ Please follow these [steps](https://posthog.com/docs/api/overview#how-to-obtain- | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.7 | 2022-07-26 | [14585](https://github.com/airbytehq/airbyte/pull/14585) | Add missing 'properties' field to event attributes | | 0.1.6 | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description | | 0.1.5 | 2021-12-24 | [9082](https://github.com/airbytehq/airbyte/pull/9082) | Remove obsolete session_events and insights streams | | 0.1.4 | 2021-09-14 | [6058](https://github.com/airbytehq/airbyte/pull/6058) | Support self-hosted posthog instances | diff --git a/docs/integrations/sources/recurly.md b/docs/integrations/sources/recurly.md index 94b6e91e6df8..fe5560d9bc5d 100644 --- a/docs/integrations/sources/recurly.md +++ b/docs/integrations/sources/recurly.md @@ -62,9 +62,10 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- | :--- | -| 0.4.0 | 2022-01-28 | [9866](https://github.com/airbytehq/airbyte/pull/9866) | Revamp Recurly Schema and add more resources | -| 0.3.2 | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description | -| 0.3.1 | 2022-01-10 | [9382](https://github.com/airbytehq/airbyte/pull/9382) | Source Recurly: avoid loading all accounts when importing account coupon redemptions | -| 0.3.0 | 2021-12-08 | [8468](https://github.com/airbytehq/airbyte/pull/8468) | Support Incremental Sync Mode | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--------------------------------------------------------| :--------------------------------------------------------------------------------------- | +| 0.4.1 | 2022-06-10 | [13685](https://github.com/airbytehq/airbyte/pull/13685) | Add state_checkpoint_interval to Recurly stream | +| 0.4.0 | 2022-01-28 | [9866](https://github.com/airbytehq/airbyte/pull/9866) | Revamp Recurly Schema and add more resources | +| 0.3.2 | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description | +| 0.3.1 | 2022-01-10 | [9382](https://github.com/airbytehq/airbyte/pull/9382) | Source Recurly: avoid loading all accounts when importing account coupon redemptions | +| 0.3.0 | 2021-12-08 | [8468](https://github.com/airbytehq/airbyte/pull/8468) | Support Incremental Sync Mode | diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index 4095cfdca74a..3113b5a8b757 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -119,6 +119,7 @@ Now that you have set up the Salesforce source connector, check out the followin | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------| +| 1.0.12 | 2022-08-09 | [15444](https://github.com/airbytehq/airbyte/pull/15444) | Fixed bug when `Bulk Job` was timeout by the connector, but remained running on the server | | 1.0.11 | 2022-07-07 | [13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field descriptions | | 1.0.10 | 2022-06-09 | [13658](https://github.com/airbytehq/airbyte/pull/13658) | Correct logic to sync stream larger than page size | | 1.0.9 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md index 77a17ac1e487..4fffdfaedc0e 100644 --- a/docs/integrations/sources/sendgrid.md +++ b/docs/integrations/sources/sendgrid.md @@ -34,8 +34,8 @@ The Sendgrid connector should not run into Sendgrid API limitations under normal * Sendgrid Account * Sendgrid API Key with the following permissions: - * Read-only access to all resources - * Full access to marketing resources + * Read-only access to all resources + * Full access to marketing resources ### Setup guide @@ -45,9 +45,10 @@ We recommend creating a key specifically for Airbyte access. This will allow you To consume Messages resources requires to purchase an extra on Sendgrid. You can read more about this [here](https://docs.sendgrid.com/api-reference/e-mail-activity) -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.2.8 | 2022-06-07 | [13571](https://github.com/airbytehq/airbyte/pull/13571) | Add Message stream | -| 0.2.7 | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910) | Add Single Sends Stats stream | -| 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | +| Version | Date | Pull Request | Subject | +|:--------| :--- |:---------------------------------------------------------|:---------------------------------------------------| +| 0.2.9 | 2022-06-07 | [15257](https://github.com/airbytehq/airbyte/pull/15257) | Migrate to config-based framework | +| 0.2.8 | 2022-06-07 | [13571](https://github.com/airbytehq/airbyte/pull/13571) | Add Message stream | +| 0.2.7 | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910) | Add Single Sends Stats stream | +| 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index 055c1cdcc56e..1e4c696f2b76 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -44,7 +44,8 @@ You can find or create authentication tokens within [Sentry](https://sentry.io/s ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:--------------------------------------------------| +| 0.1.2 | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework | +| 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index c10eab38df34..db355a53458e 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -74,34 +74,35 @@ The Stripe connector should not run into Stripe API limitations under normal usa ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | -| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | added external account streams | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | +| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | +| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | added external account streams - | | 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | added semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | | 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | | 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | | 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | | 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | | 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | -| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | -| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | -| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | -| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | -| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | -| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | -| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | -| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback\_window\_days parameter | -| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | -| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | -| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | -| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | -| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent\_off and discounts data filed types | -| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | -| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | -| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | -| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | -| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | -| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | +| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | +| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | +| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | +| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | +| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | +| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | +| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | +| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback\_window\_days parameter | +| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | +| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | +| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | +| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | +| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent\_off and discounts data filed types | +| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | +| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | +| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | +| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | +| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | +| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index f9d358680948..1096c65e61ef 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -470,7 +470,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -623,7 +633,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -870,7 +890,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", @@ -922,7 +952,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } } ] } @@ -1032,7 +1072,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", @@ -1084,7 +1134,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } } ] } @@ -1369,7 +1429,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", @@ -1421,7 +1491,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } } ] } @@ -1702,7 +1782,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -8151,7 +8241,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -8350,7 +8450,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -8613,7 +8723,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "latestSyncJobCreatedAt" : 0, @@ -8759,7 +8879,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } } ] } @@ -8963,7 +9093,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "latestSyncJobCreatedAt" : 0, @@ -9109,7 +9249,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } } ] } @@ -9313,7 +9463,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "latestSyncJobCreatedAt" : 0, @@ -9459,7 +9619,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } } ] } @@ -9659,7 +9829,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -9858,7 +10038,17 @@

Example data

}, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "namespaceFormat" : "${SOURCE_NAMESPACE}", - "operationIds" : [ null, null ] + "operationIds" : [ null, null ], + "scheduleData" : { + "cron" : { + "cronExpression" : "cronExpression", + "cronTimeZone" : "cronTimeZone" + }, + "basicSchedule" : { + "units" : 6, + "timeUnit" : "minutes" + } + } }

Produces

@@ -10516,6 +10706,10 @@

Table of Contents

  • ConnectionRead -
  • ConnectionReadList -
  • ConnectionSchedule -
    • +
  • ConnectionScheduleData -
    • +
  • ConnectionScheduleData_basicSchedule -
    • +
  • ConnectionScheduleData_cron -
    • +
  • ConnectionScheduleType -
  • ConnectionSearch -
  • ConnectionState -
  • ConnectionStateType -
    • @@ -10861,6 +11055,8 @@

    ConnectionCreate - operationIds (optional)
    array[UUID] format: uuid
    syncCatalog (optional)
    AirbyteCatalog
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status
    ConnectionStatus
    resourceRequirements (optional)
    ResourceRequirements
    sourceCatalogId (optional)
    UUID format: uuid
    @@ -10887,6 +11083,8 @@

    ConnectionRead - operationIds (optional)
    array[UUID] format: uuid
    syncCatalog
    AirbyteCatalog
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status
    ConnectionStatus
    resourceRequirements (optional)
    ResourceRequirements
    sourceCatalogId (optional)
    UUID format: uuid
    @@ -10909,6 +11107,38 @@

    ConnectionSchedule - minutes
    hours
    days
    weeks
    months
    +
    +

    ConnectionScheduleData - Up

    +
    schedule for when the the connection should run, per the schedule type
    +
    +
    basicSchedule (optional)
    ConnectionScheduleData_basicSchedule
    +
    cron (optional)
    ConnectionScheduleData_cron
    +
    +
    +
    +

    ConnectionScheduleData_basicSchedule - Up

    +
    +
    +
    timeUnit
    String
    +
    Enum:
    +
    minutes
    hours
    days
    weeks
    months
    +
    units
    Long format: int64
    +
    +
    +
    +

    ConnectionScheduleData_cron - Up

    +
    +
    +
    cronExpression
    String
    +
    cronTimeZone
    String
    +
    +
    +
    +

    ConnectionScheduleType - Up

    +
    determine how the schedule data should be interpreted
    +
    +
    +

    ConnectionSearch - Up

    @@ -10921,6 +11151,8 @@

    ConnectionSearch - sourceId (optional)

    UUID format: uuid
    destinationId (optional)
    UUID format: uuid
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status (optional)
    ConnectionStatus
    source (optional)
    SourceSearch
    destination (optional)
    DestinationSearch
    @@ -10961,6 +11193,8 @@

    ConnectionUpdate - operationIds (optional)
    array[UUID] format: uuid
    syncCatalog
    AirbyteCatalog
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status
    ConnectionStatus
    resourceRequirements (optional)
    ResourceRequirements
    sourceCatalogId (optional)
    UUID format: uuid
    @@ -11923,6 +12157,8 @@

    WebBackendConnectionCreate
    operationIds (optional)
    array[UUID] format: uuid
    syncCatalog (optional)
    AirbyteCatalog
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status
    ConnectionStatus
    resourceRequirements (optional)
    ResourceRequirements
    operations (optional)
    array[OperationCreate]
    @@ -11942,6 +12178,8 @@

    WebBackendConnectionRead - <
    destinationId
    UUID format: uuid
    syncCatalog
    AirbyteCatalog
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status
    ConnectionStatus
    operationIds (optional)
    array[UUID] format: uuid
    source
    SourceRead
    @@ -11982,6 +12220,8 @@

    WebBackendConnectionSearch
    sourceId (optional)
    UUID format: uuid
    destinationId (optional)
    UUID format: uuid
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status (optional)
    ConnectionStatus
    source (optional)
    SourceSearch
    destination (optional)
    DestinationSearch
    @@ -11999,6 +12239,8 @@

    WebBackendConnectionUpdate
    operationIds (optional)
    array[UUID] format: uuid
    syncCatalog
    AirbyteCatalog
    schedule (optional)
    ConnectionSchedule
    +
    scheduleType (optional)
    ConnectionScheduleType
    +
    scheduleData (optional)
    ConnectionScheduleData
    status
    ConnectionStatus
    resourceRequirements (optional)
    ResourceRequirements
    withRefreshedCatalog (optional)
    Boolean
    diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index 64612ea1aa4d..ada7a851bf34 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -54,6 +54,11 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= + # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= @@ -64,6 +69,7 @@ JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= # Launch a separate pod to orchestrate sync steps CONTAINER_ORCHESTRATOR_ENABLED=true +CONTAINER_ORCHESTRATOR_IMAGE= # Open Telemetry Configuration METRIC_CLIENT= diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index d62225479835..230efad8e4ec 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -56,6 +56,11 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= + # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= @@ -66,6 +71,7 @@ JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= # Launch a separate pod to orchestrate sync steps CONTAINER_ORCHESTRATOR_ENABLED=true +CONTAINER_ORCHESTRATOR_IMAGE= # Open Telemetry Configuration METRIC_CLIENT= diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index b48947558c89..a410638647ba 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -56,6 +56,11 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= + # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= @@ -66,6 +71,7 @@ JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= # Launch a separate pod to orchestrate sync steps CONTAINER_ORCHESTRATOR_ENABLED=true +CONTAINER_ORCHESTRATOR_IMAGE= # Open Telemetry Configuration METRIC_CLIENT= diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index f5906abd1914..02ceacb266e2 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -119,6 +119,26 @@ spec: configMapKeyRef: name: airbyte-env key: JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST + - name: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: @@ -210,6 +230,11 @@ spec: configMapKeyRef: name: airbyte-env key: CONTAINER_ORCHESTRATOR_ENABLED + - name: CONTAINER_ORCHESTRATOR_IMAGE + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONTAINER_ORCHESTRATOR_IMAGE - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION valueFrom: configMapKeyRef: diff --git a/octavia-cli/README.md b/octavia-cli/README.md index fc35e82670f0..094131bfae1c 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -705,6 +705,7 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment | Version | Date | Description | PR | | ------- | ---------- | --------------------------------------------------------------------------------------| ----------------------------------------------------------- | +| 0.40.0 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | | 0.39.33 | 2022-07-05 | Add `octavia import all` command | [#14374](https://github.com/airbytehq/airbyte/pull/14374) | | 0.39.32 | 2022-06-30 | Create import command to import and manage existing Airbyte resource from octavia-cli | [#14137](https://github.com/airbytehq/airbyte/pull/14137) | | 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | diff --git a/octavia-cli/integration_tests/cassettes/test_api_http_headers/test_api_http_headers.yaml b/octavia-cli/integration_tests/cassettes/test_api_http_headers/test_api_http_headers.yaml index 24332db97353..a7316d679e0c 100644 --- a/octavia-cli/integration_tests/cassettes/test_api_http_headers/test_api_http_headers.yaml +++ b/octavia-cli/integration_tests/cassettes/test_api_http_headers/test_api_http_headers.yaml @@ -9,7 +9,7 @@ interactions: Custom-Header: - Foo User-Agent: - - octavia-cli/0.39.27 + - octavia-cli/0.39.42 method: GET uri: http://localhost:8000/api/v1/health response: @@ -31,9 +31,9 @@ interactions: Content-Type: - application/json Date: - - Wed, 29 Jun 2022 10:53:05 GMT + - Thu, 11 Aug 2022 08:55:08 GMT Server: - - nginx/1.19.10 + - nginx/1.23.1 status: code: 200 message: OK @@ -49,12 +49,12 @@ interactions: Custom-Header: - Foo User-Agent: - - octavia-cli/0.39.27 + - octavia-cli/0.39.42 method: POST uri: http://localhost:8000/api/v1/workspaces/list response: body: - string: '{"workspaces":[{"workspaceId":"75658e4f-e5f0-4e35-be0c-bdad33226c94","customerId":"71257691-1da3-407a-b564-9ce8792b8fe4","email":"augustin@airbyte.io","name":"75658e4f-e5f0-4e35-be0c-bdad33226c94","slug":"75658e4f-e5f0-4e35-be0c-bdad33226c94","initialSetupComplete":true,"displaySetupWizard":true,"anonymousDataCollection":false,"news":false,"securityUpdates":false,"notifications":[]}]}' + string: '{"workspaces":[{"workspaceId":"a70357d2-f753-41a4-a0cc-22944c59e5d2","customerId":"3bc0b3a1-bd3d-4cf6-8bd8-8273addacb42","name":"a70357d2-f753-41a4-a0cc-22944c59e5d2","slug":"a70357d2-f753-41a4-a0cc-22944c59e5d2","initialSetupComplete":false,"displaySetupWizard":true,"notifications":[]}]}' headers: Access-Control-Allow-Headers: - Origin, Content-Type, Accept, Content-Encoding @@ -65,20 +65,20 @@ interactions: Connection: - keep-alive Content-Length: - - "387" + - "289" Content-Security-Policy: - script-src * 'unsafe-inline'; worker-src self blob:; Content-Type: - application/json Date: - - Wed, 29 Jun 2022 10:53:05 GMT + - Thu, 11 Aug 2022 08:55:08 GMT Server: - - nginx/1.19.10 + - nginx/1.23.1 status: code: 200 message: OK - request: - body: '{"workspaceId": "75658e4f-e5f0-4e35-be0c-bdad33226c94"}' + body: '{"workspaceId": "a70357d2-f753-41a4-a0cc-22944c59e5d2"}' headers: Accept: - application/json @@ -89,12 +89,12 @@ interactions: Custom-Header: - Foo User-Agent: - - octavia-cli/0.39.27 + - octavia-cli/0.39.42 method: POST uri: http://localhost:8000/api/v1/workspaces/get response: body: - string: '{"workspaceId":"75658e4f-e5f0-4e35-be0c-bdad33226c94","customerId":"71257691-1da3-407a-b564-9ce8792b8fe4","email":"augustin@airbyte.io","name":"75658e4f-e5f0-4e35-be0c-bdad33226c94","slug":"75658e4f-e5f0-4e35-be0c-bdad33226c94","initialSetupComplete":true,"displaySetupWizard":true,"anonymousDataCollection":false,"news":false,"securityUpdates":false,"notifications":[]}' + string: '{"workspaceId":"a70357d2-f753-41a4-a0cc-22944c59e5d2","customerId":"3bc0b3a1-bd3d-4cf6-8bd8-8273addacb42","name":"a70357d2-f753-41a4-a0cc-22944c59e5d2","slug":"a70357d2-f753-41a4-a0cc-22944c59e5d2","initialSetupComplete":false,"displaySetupWizard":true,"notifications":[]}' headers: Access-Control-Allow-Headers: - Origin, Content-Type, Accept, Content-Encoding @@ -105,15 +105,15 @@ interactions: Connection: - keep-alive Content-Length: - - "370" + - "272" Content-Security-Policy: - script-src * 'unsafe-inline'; worker-src self blob:; Content-Type: - application/json Date: - - Wed, 29 Jun 2022 10:53:05 GMT + - Thu, 11 Aug 2022 08:55:08 GMT Server: - - nginx/1.19.10 + - nginx/1.23.1 status: code: 200 message: OK @@ -129,12427 +129,4452 @@ interactions: Custom-Header: - Foo User-Agent: - - octavia-cli/0.39.27 + - octavia-cli/0.39.42 method: POST uri: http://localhost:8000/api/v1/source_definitions/list response: body: string: - "{\"sourceDefinitions\":[{\"sourceDefinitionId\":\"3052c77e-8b91-47e2-97a0-a29a22794b4b\",\"name\":\"PersistIq\",\"dockerRepository\":\"airbyte/source-persistiq\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/persistiq\",\"icon\":\"\\n - \ \\n \\n \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"435bb9a5-7887-4809-aa58-28c27df0d7ad\",\"name\":\"MySQL\",\"dockerRepository\":\"airbyte/source-mysql\",\"dockerImageTag\":\"0.5.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mysql\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"3981c999-bd7d-4afc-849b-e53dea90c948\",\"name\":\"Lever - Hiring\",\"dockerRepository\":\"airbyte/source-lever-hiring\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/lever-hiring\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"778daa7c-feaf-4db6-96f3-70fd645acc77\",\"name\":\"File\",\"dockerRepository\":\"airbyte/source-file\",\"dockerImageTag\":\"0.2.10\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/file\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"aea2fd0d-377d-465e-86c0-4fdc4f688e51\",\"name\":\"Zoom\",\"dockerRepository\":\"airbyte/source-zoom-singer\",\"dockerImageTag\":\"0.2.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zoom\",\"icon\":\"\\n\\n \\n \\n - \ \\n image/svg+xml\\n - \ \\n \\n \\n \\n - \ \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"b39a7370-74c3-45a6-ac3a-380d48520a83\",\"name\":\"Oracle - DB\",\"dockerRepository\":\"airbyte/source-oracle\",\"dockerImageTag\":\"0.3.17\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/oracle\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"b117307c-14b6-41aa-9422-947e34922962\",\"name\":\"Salesforce\",\"dockerRepository\":\"airbyte/source-salesforce\",\"dockerImageTag\":\"1.0.10\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/salesforce\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"b5ea17b1-f170-46dc-bc31-cc744ca984c1\",\"name\":\"Microsoft - SQL Server (MSSQL)\",\"dockerRepository\":\"airbyte/source-mssql\",\"dockerImageTag\":\"0.4.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mssql\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"c2281cee-86f9-4a86-bb48-d23286b4c7bd\",\"name\":\"Slack\",\"dockerRepository\":\"airbyte/source-slack\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/slack\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"90916976-a132-4ce9-8bce-82a03dd58788\",\"name\":\"BambooHR\",\"dockerRepository\":\"airbyte/source-bamboo-hr\",\"dockerImageTag\":\"0.2.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bamboo-hr\",\"icon\":\"\\n\\n \\n BambooHR\\n Created - with Sketch.\\n \\n \\n \\n - \ \\n \\n \\n \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"eb4c9e00-db83-4d63-a386-39cfa91012a8\",\"name\":\"Google - Search Console\",\"dockerRepository\":\"airbyte/source-google-search-console\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-search-console\",\"icon\":\"\\n\\n \\n Artboard\\n - \ Created with Sketch.\\n \\n \\n - \ \\n \\n - \ \\n \\n - \ \\n \\n - \ \\n - \ \\n - \ \\n \\n \\n \\n \\n - \ \\n - \ \\n \\n \\n \\n \\n \\n - \ \\n \\n - \ \\n \\n \\n - \ \\n \\n \\n\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"eff3616a-f9c3-11eb-9a03-0242ac130003\",\"name\":\"Google - Analytics\",\"dockerRepository\":\"airbyte/source-google-analytics-v4\",\"dockerImageTag\":\"0.1.21\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-analytics-v4\",\"icon\":\"\\n\\n\\n\\n\\t\\n\\t\\t\\n\\t\\n\\t\\n\\t\\t\\n\\t\\n\\t\\n\\t\\t\\n\\t\\n\\n\\n\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"ef69ef6e-aa7f-4af1-a01d-ef775033524e\",\"name\":\"GitHub\",\"dockerRepository\":\"airbyte/source-github\",\"dockerImageTag\":\"0.2.36\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/github\",\"icon\":\"\\n\\n\\n\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"cd06e646-31bf-4dc8-af48-cbc6530fcad3\",\"name\":\"Kustomer\",\"dockerRepository\":\"airbyte/source-kustomer-singer\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/kustomer\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87\",\"name\":\"Sendgrid\",\"dockerRepository\":\"airbyte/source-sendgrid\",\"dockerImageTag\":\"0.2.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/sendgrid\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d6f73702-d7a0-4e95-9758-b0fb1af0bfba\",\"name\":\"Jenkins\",\"dockerRepository\":\"farosai/airbyte-jenkins-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/jenkins\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\n\\t\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"2817b3f0-04e4-4c7a-9f32-7a5e8a83db95\",\"name\":\"PagerDuty\",\"dockerRepository\":\"farosai/airbyte-pagerduty-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pagerduty\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"008b2e26-11a3-11ec-82a8-0242ac130003\",\"name\":\"Commercetools\",\"dockerRepository\":\"airbyte/source-commercetools\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/commercetools\",\"icon\":\"\\n\\n\\n\\n\\n\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\n\\t\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b\",\"name\":\"Snapchat - Marketing\",\"dockerRepository\":\"airbyte/source-snapchat-marketing\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/snapchat-marketing\",\"icon\":\"\\n\\n\\n\\n - \ \\n \\n \\n - \ \\n \\n image/svg+xml\\n - \ \\n \\n \\n \\n - \ \\n \\n - \ \\n - \ \\n - \ \\n \\n \\n \\n \\n \\n \\n - \ \\n\\t\\n\\t\\t\\n\\n\\t\\n\\n\\t\\n\\n\\n \\n \\n\\t.st0{fill:#FFFFFF;}\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"cf40a7f8-71f8-45ce-a7fa-fca053e4028c\",\"name\":\"Confluence\",\"dockerRepository\":\"airbyte/source-confluence\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/confluence\",\"icon\":\"\\n\\n - \ \\n \\n - \ \\n - \ \\n - \ \\n \\n - \ \\n - \ \\n - \ \\n \\n \\n\\t\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"447e0381-3780-4b46-bb62-00a4e3c8b8e2\",\"name\":\"IBM - Db2\",\"dockerRepository\":\"airbyte/source-db2\",\"dockerImageTag\":\"0.1.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/db2\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"2af123bf-0aaf-4e0d-9784-cb497f23741a\",\"name\":\"Appstore\",\"dockerRepository\":\"airbyte/source-appstore-singer\",\"dockerImageTag\":\"0.2.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/appstore\",\"icon\":\"\\n\\n - \ \\n \\n \\n \\n - \ \\n \\n\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"c8630570-086d-4a40-99ae-ea5b18673071\",\"name\":\"Zendesk - Talk\",\"dockerRepository\":\"airbyte/source-zendesk-talk\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-talk\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c\",\"name\":\"BigQuery\",\"dockerRepository\":\"airbyte/source-bigquery\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bigquery\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"eca08d79-7b92-4065-b7f3-79c14836ebe7\",\"name\":\"Freshsales\",\"dockerRepository\":\"airbyte/source-freshsales\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/freshsales\",\"icon\":\"freshsales_logo_color\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"2fed2292-5586-480c-af92-9944e39fe12d\",\"name\":\"Short.io\",\"dockerRepository\":\"airbyte/source-shortio\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/shortio\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"79c1aa37-dae3-42ae-b333-d1c105477715\",\"name\":\"Zendesk - Support\",\"dockerRepository\":\"airbyte/source-zendesk-support\",\"dockerImageTag\":\"0.2.10\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-support\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"010eb12f-837b-4685-892d-0a39f76a98f5\",\"name\":\"Facebook - Pages\",\"dockerRepository\":\"airbyte/source-facebook-pages\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/facebook-pages\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"fa9f58c6-2d03-4237-aaa4-07d75e0c1396\",\"name\":\"Amplitude\",\"dockerRepository\":\"airbyte/source-amplitude\",\"dockerImageTag\":\"0.1.10\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amplitude\",\"icon\":\"\\n\\t\\n\\t\\n\\t\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"8d7ef552-2c0f-11ec-8d3d-0242ac130003\",\"name\":\"SearchMetrics\",\"dockerRepository\":\"airbyte/source-search-metrics\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/search-metrics\",\"icon\":\"\\n\\n\\n\\nCreated by potrace 1.16, written by Peter Selinger - 2001-2019\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"bb6afd81-87d5-47e3-97c4-e2c2901b1cf8\",\"name\":\"OneSignal\",\"dockerRepository\":\"airbyte/source-onesignal\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/onesignal\",\"icon\":\"\\n\\n \\n \\n - \ \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"dfffecb7-9a13-43e9-acdc-b92af7997ca9\",\"name\":\"Close.com\",\"dockerRepository\":\"airbyte/source-close-com\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/close-com\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"29b409d9-30a5-4cc8-ad50-886eb846fea3\",\"name\":\"QuickBooks\",\"dockerRepository\":\"airbyte/source-quickbooks-singer\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/quickbooks\",\"icon\":\" qb-logoCreated with Sketch. - \",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"cd42861b-01fc-4658-a8ab-5d11d0510f01\",\"name\":\"Recurly\",\"dockerRepository\":\"airbyte/source-recurly\",\"dockerImageTag\":\"0.4.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/recurly\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"72d405a3-56d8-499f-a571-667c03406e43\",\"name\":\"Dockerhub\",\"dockerRepository\":\"airbyte/source-dockerhub\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/dockerhub\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d8313939-3782-41b0-be29-b3ca20d8dd3a\",\"name\":\"Intercom\",\"dockerRepository\":\"airbyte/source-intercom\",\"dockerImageTag\":\"0.1.19\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/intercom\",\"icon\":\"\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"ef580275-d9a9-48bb-af5e-db0f5855be04\",\"name\":\"Webflow\",\"dockerRepository\":\"airbyte/source-webflow\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/webflow\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e\",\"name\":\"Linnworks\",\"dockerRepository\":\"airbyte/source-linnworks\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/linnworks\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"6e00b415-b02e-4160-bf02-58176a0ae687\",\"name\":\"Notion\",\"dockerRepository\":\"airbyte/source-notion\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/notion\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212\",\"name\":\"Airtable\",\"dockerRepository\":\"airbyte/source-airtable\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/airtable\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"80a54ea2-9959-4040-aac1-eee42423ec9b\",\"name\":\"Monday\",\"dockerRepository\":\"airbyte/source-monday\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/monday\",\"icon\":\"\\n\\n - \ \\n \\n \\n image/svg+xml\\n - \ \\n \\n \\n \\n \\n \\n Logo / monday.com\\n \\n - \ \\n \\n \\n \\n \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"6ff047c0-f5d5-4ce5-8c81-204a830fa7e1\",\"name\":\"AWS - CloudTrail\",\"dockerRepository\":\"airbyte/source-aws-cloudtrail\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/aws-cloudtrail\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"c47d6804-8b98-449f-970a-5ddb5cb5d7aa\",\"name\":\"Customer.io\",\"dockerRepository\":\"farosai/airbyte-customer-io-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/customer-io\",\"icon\":\"Logo-Color-NEW\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"686473f1-76d9-4994-9cc7-9b13da46147c\",\"name\":\"Chargebee\",\"dockerRepository\":\"airbyte/source-chargebee\",\"dockerImageTag\":\"0.1.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/chargebee\",\"icon\":\"\\n\\n - \ \\n \\n - \ \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e\",\"name\":\"MongoDb\",\"dockerRepository\":\"airbyte/source-mongodb-v2\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mongodb-v2\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"253487c0-2246-43ba-a21f-5116b20a2c50\",\"name\":\"Google - Ads\",\"dockerRepository\":\"airbyte/source-google-ads\",\"dockerImageTag\":\"0.1.42\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-ads\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"95e8cffd-b8c4-4039-968e-d32fb4a69bde\",\"name\":\"Klaviyo\",\"dockerRepository\":\"airbyte/source-klaviyo\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/klaviyo\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"bad83517-5e54-4a3d-9b53-63e85fbd4d7c\",\"name\":\"ClickHouse\",\"dockerRepository\":\"airbyte/source-clickhouse\",\"dockerImageTag\":\"0.1.10\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/clickhouse\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d913b0f2-cc51-4e55-a44c-8ba1697b9239\",\"name\":\"Paypal - Transaction\",\"dockerRepository\":\"airbyte/source-paypal-transaction\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/paypal-transaction\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"374ebc65-6636-4ea0-925c-7d35999a8ffc\",\"name\":\"Smartsheets\",\"dockerRepository\":\"airbyte/source-smartsheets\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/smartsheets\",\"icon\":\"\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"47f25999-dd5e-4636-8c39-e7cea2453331\",\"name\":\"Bing - Ads\",\"dockerRepository\":\"airbyte/source-bing-ads\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bing-ads\",\"icon\":\"\\n\\n \\n \\n \\n\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"d60a46d4-709f-4092-a6b7-2457f7d455f5\",\"name\":\"Prestashop\",\"dockerRepository\":\"airbyte/source-prestashop\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/presta-shop\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"71607ba1-c0ac-4799-8049-7f4b90dd50f7\",\"name\":\"Google - Sheets\",\"dockerRepository\":\"airbyte/source-google-sheets\",\"dockerImageTag\":\"0.2.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-sheets\",\"icon\":\"\\n\\n\\n\\n\\t\\n\\t\\n\\t\\n\\n\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"492b56d1-937c-462e-8076-21ad2031e784\",\"name\":\"Hellobaton\",\"dockerRepository\":\"airbyte/source-hellobaton\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/hellobaton\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d8540a80-6120-485d-b7d6-272bca477d9b\",\"name\":\"OpenWeather\",\"dockerRepository\":\"airbyte/source-openweather\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/openweather\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"45d2e135-2ede-49e1-939f-3e3ec357a65e\",\"name\":\"Recharge\",\"dockerRepository\":\"airbyte/source-recharge\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/recharge\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d53f9084-fa6b-4a5a-976c-5b8392f4ad8a\",\"name\":\"E2E - Testing\",\"dockerRepository\":\"airbyte/source-e2e-test\",\"dockerImageTag\":\"2.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/e2e-test\",\"icon\":\"\\n \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"b9dc6155-672e-42ea-b10d-9f1f1fb95ab1\",\"name\":\"Twilio\",\"dockerRepository\":\"airbyte/source-twilio\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/twilio\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"1d4fdb25-64fc-4569-92da-fcdca79a8372\",\"name\":\"Okta\",\"dockerRepository\":\"airbyte/source-okta\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/okta\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"193bdcb8-1dd9-48d1-aade-91cadfd74f9b\",\"name\":\"Paystack\",\"dockerRepository\":\"airbyte/source-paystack\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/paystack\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"6f2ac653-8623-43c4-8950-19218c7caf3d\",\"name\":\"Firebolt\",\"dockerRepository\":\"airbyte/source-firebolt\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.firebolt.io/\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"63cea06f-1c75-458d-88fe-ad48c7cb27fd\",\"name\":\"Braintree\",\"dockerRepository\":\"airbyte/source-braintree\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/braintree\",\"icon\":\"\\n\\n - \ \\n \\n - \ \\n \\n - \ \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"b6604cbd-1b12-4c08-8767-e140d0fb0877\",\"name\":\"Chartmogul\",\"dockerRepository\":\"airbyte/source-chartmogul\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/chartmogul\",\"icon\":\"\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d8286229-c680-4063-8c59-23b9b391c700\",\"name\":\"Pipedrive\",\"dockerRepository\":\"airbyte/source-pipedrive\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pipedrive\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5\",\"name\":\"Zuora\",\"dockerRepository\":\"airbyte/source-zuora\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zuora\",\"icon\":\"\\n\\n\\nimage/svg+xml\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"59f1e50a-331f-4f09-b3e8-2e8d4d355f44\",\"name\":\"Greenhouse\",\"dockerRepository\":\"airbyte/source-greenhouse\",\"dockerImageTag\":\"0.2.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/greenhouse\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"789f8e7a-2d28-11ec-8d3d-0242ac130003\",\"name\":\"Lemlist\",\"dockerRepository\":\"airbyte/source-lemlist\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/lemlist\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"f1e4c7f6-db5c-4035-981f-d35ab4998794\",\"name\":\"Zenloop\",\"dockerRepository\":\"airbyte/source-zenloop\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zenloop\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"9e0556f4-69df-4522-a3fb-03264d36b348\",\"name\":\"Marketo\",\"dockerRepository\":\"airbyte/source-marketo\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/marketo\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"7f0455fb-4518-4ec0-b7a3-d808bf8081cc\",\"name\":\"Orb\",\"dockerRepository\":\"airbyte/source-orb\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/orb\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"4942d392-c7b5-4271-91f9-3b4f4e51eb3e\",\"name\":\"ZohoCRM\",\"dockerRepository\":\"airbyte/source-zoho-crm\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/zoho-crm\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"9b2d3607-7222-4709-9fa2-c2abdebbdd88\",\"name\":\"Chargify\",\"dockerRepository\":\"airbyte/source-chargify\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/chargify\",\"icon\":\"\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4\",\"name\":\"Zendesk - Chat\",\"dockerRepository\":\"airbyte/source-zendesk-chat\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-chat\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"12928b32-bf0a-4f1e-964f-07e12e37153a\",\"name\":\"Mixpanel\",\"dockerRepository\":\"airbyte/source-mixpanel\",\"dockerImageTag\":\"0.1.17\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mixpanel\",\"icon\":\"\\n\\n\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"d19ae824-e289-4b14-995a-0632eb46d246\",\"name\":\"Google - Directory\",\"dockerRepository\":\"airbyte/source-google-directory\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-directory\",\"icon\":\"\\n\\n\\n\\n - \ \\n\\n\\n \\n\\n\\n - \ \\n\\n\\n\\n \\n\\n\\n - \ \\n\\n\\n - \ \\n\\n\\n\\n \\n\\n\\n - \ \\n\\n\\n - \ \\n\\n\\n\\n \\n\\n\\n - \ \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n - \ \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n - \ \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n - \ \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e2b40e36-aa0e-4bed-b41b-bcea6fa348b1\",\"name\":\"Exchange - Rates Api\",\"dockerRepository\":\"airbyte/source-exchange-rates\",\"dockerImageTag\":\"0.2.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/exchangeratesapi\",\"icon\":\"\\n\\n \\n logo\\n - \ Created with Sketch.\\n \\n \\n - \ \\n \\n \\n \\n - \ \\n \\n \\n \\n \\n \\n - \ \\n \\n \\n \\n - \ \\n \\n \\n \\n - \ \\n \\n - \ \\n \\n \\n - \ \\n \\n \\n \\n - \ \\n \\n \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"0dad1a35-ccf8-4d03-b73e-6788c00b13ae\",\"name\":\"TiDB\",\"dockerRepository\":\"airbyte/source-tidb\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/tidb\",\"icon\":\"\\n - \ \\n \\n - \ \\n \\n \\n \\n \\n - \ \\n \\n - \ \\n - \ \\n - \ \\n - \ \\n \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"dfd88b22-b603-4c3d-aad7-3701784586b1\",\"name\":\"Faker\",\"dockerRepository\":\"airbyte/source-faker\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/source-faker\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"db04ecd1-42e7-4115-9cec-95812905c626\",\"name\":\"Retently\",\"dockerRepository\":\"airbyte/source-retently\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/retently\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d917a47b-8537-4d0d-8c10-36a9928d4265\",\"name\":\"Kafka\",\"dockerRepository\":\"airbyte/source-kafka\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/kafka\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"137ece28-5434-455c-8f34-69dc3782f451\",\"name\":\"LinkedIn - Ads\",\"dockerRepository\":\"airbyte/source-linkedin-ads\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/linkedin-ads\",\"icon\":\"\\n\\n\\n - \ \\n \\n \\n - \ \\n\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"d0243522-dccf-4978-8ba0-37ed47a0bdbf\",\"name\":\"Asana\",\"dockerRepository\":\"airbyte/source-asana\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/asana\",\"icon\":\"\\n\\n - \ \\n \\n \\n \\n - \ \\n - \ \\n \\n \\n \\n - \ \\n \\n \\n - \ \\n - \ \\n - \ \\n - \ \\n \\n - \ \\n \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"9fa5862c-da7c-11eb-8d19-0242ac130003\",\"name\":\"Cockroachdb\",\"dockerRepository\":\"airbyte/source-cockroachdb\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/cockroachdb\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"36c891d9-4bd9-43ac-bad2-10e12756272c\",\"name\":\"HubSpot\",\"dockerRepository\":\"airbyte/source-hubspot\",\"dockerImageTag\":\"0.1.72\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/hubspot\",\"icon\":\"\\n\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"6371b14b-bc68-4236-bfbd-468e8df8e968\",\"name\":\"PokeAPI\",\"dockerRepository\":\"airbyte/source-pokeapi\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pokeapi\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d6f73702-d7a0-4e95-9758-b0fb1af0bfba\",\"name\":\"Jenkins\",\"dockerRepository\":\"farosai/airbyte-jenkins-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/jenkins\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\n\\t\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"2817b3f0-04e4-4c7a-9f32-7a5e8a83db95\",\"name\":\"PagerDuty\",\"dockerRepository\":\"farosai/airbyte-pagerduty-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pagerduty\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"008b2e26-11a3-11ec-82a8-0242ac130003\",\"name\":\"Commercetools\",\"dockerRepository\":\"airbyte/source-commercetools\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/commercetools\",\"icon\":\"\\n\\n\\n\\n\\n\\t\\n\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\t\\n\\t\\t\\n\\t\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b\",\"name\":\"Snapchat + Marketing\",\"dockerRepository\":\"airbyte/source-snapchat-marketing\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/snapchat-marketing\",\"icon\":\"\\n\\n\\n\\n + \ \\n \\n \\n + \ \\n \\n image/svg+xml\\n + \ \\n \\n \\n \\n + \ \\n \\n + \ \\n + \ \\n + \ \\n \\n \\n \\n \\n \\n \\n + \ \\n\\t\\n\\t\\t\\n\\n\\t\\n\\n\\t\\n\\n\\n \\n \\n\\t.st0{fill:#FFFFFF;}\\n\\n\\n\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"cf40a7f8-71f8-45ce-a7fa-fca053e4028c\",\"name\":\"Confluence\",\"dockerRepository\":\"airbyte/source-confluence\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/confluence\",\"icon\":\"\\n\\n + \ \\n \\n + \ \\n + \ \\n + \ \\n \\n + \ \\n + \ \\n + \ \\n \\n \\n\\t\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"447e0381-3780-4b46-bb62-00a4e3c8b8e2\",\"name\":\"IBM + Db2\",\"dockerRepository\":\"airbyte/source-db2\",\"dockerImageTag\":\"0.1.13\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/db2\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"2af123bf-0aaf-4e0d-9784-cb497f23741a\",\"name\":\"Appstore\",\"dockerRepository\":\"airbyte/source-appstore-singer\",\"dockerImageTag\":\"0.2.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/appstore\",\"icon\":\"\\n\\n + \ \\n \\n \\n \\n + \ \\n \\n\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\t\\t\\n\\t\\t\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"c8630570-086d-4a40-99ae-ea5b18673071\",\"name\":\"Zendesk + Talk\",\"dockerRepository\":\"airbyte/source-zendesk-talk\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-talk\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c\",\"name\":\"BigQuery\",\"dockerRepository\":\"airbyte/source-bigquery\",\"dockerImageTag\":\"0.2.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bigquery\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"eca08d79-7b92-4065-b7f3-79c14836ebe7\",\"name\":\"Freshsales\",\"dockerRepository\":\"airbyte/source-freshsales\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/freshsales\",\"icon\":\"freshsales_logo_color\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b39a7370-74c3-45a6-ac3a-380d48520a83\",\"name\":\"Oracle + DB\",\"dockerRepository\":\"airbyte/source-oracle\",\"dockerImageTag\":\"0.3.19\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/oracle\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"2fed2292-5586-480c-af92-9944e39fe12d\",\"name\":\"Short.io\",\"dockerRepository\":\"airbyte/source-shortio\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/shortio\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"79c1aa37-dae3-42ae-b333-d1c105477715\",\"name\":\"Zendesk + Support\",\"dockerRepository\":\"airbyte/source-zendesk-support\",\"dockerImageTag\":\"0.2.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-support\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"010eb12f-837b-4685-892d-0a39f76a98f5\",\"name\":\"Facebook + Pages\",\"dockerRepository\":\"airbyte/source-facebook-pages\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/facebook-pages\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"fa9f58c6-2d03-4237-aaa4-07d75e0c1396\",\"name\":\"Amplitude\",\"dockerRepository\":\"airbyte/source-amplitude\",\"dockerImageTag\":\"0.1.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amplitude\",\"icon\":\"\\n\\t\\n\\t\\n\\t\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"492b56d1-937c-462e-8076-21ad2031e784\",\"name\":\"Hellobaton\",\"dockerRepository\":\"airbyte/source-hellobaton\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/hellobaton\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"8d7ef552-2c0f-11ec-8d3d-0242ac130003\",\"name\":\"SearchMetrics\",\"dockerRepository\":\"airbyte/source-search-metrics\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/search-metrics\",\"icon\":\"\\n\\n\\n\\nCreated by potrace 1.16, written by Peter Selinger + 2001-2019\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"bb6afd81-87d5-47e3-97c4-e2c2901b1cf8\",\"name\":\"OneSignal\",\"dockerRepository\":\"airbyte/source-onesignal\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/onesignal\",\"icon\":\"\\n\\n \\n \\n + \ \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"dfffecb7-9a13-43e9-acdc-b92af7997ca9\",\"name\":\"Close.com\",\"dockerRepository\":\"airbyte/source-close-com\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/close-com\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"29b409d9-30a5-4cc8-ad50-886eb846fea3\",\"name\":\"QuickBooks\",\"dockerRepository\":\"airbyte/source-quickbooks-singer\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/quickbooks\",\"icon\":\" qb-logoCreated with Sketch. + \",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"cd42861b-01fc-4658-a8ab-5d11d0510f01\",\"name\":\"Recurly\",\"dockerRepository\":\"airbyte/source-recurly\",\"dockerImageTag\":\"0.4.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/recurly\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"eff3616a-f9c3-11eb-9a03-0242ac130003\",\"name\":\"Google + Analytics\",\"dockerRepository\":\"airbyte/source-google-analytics-v4\",\"dockerImageTag\":\"0.1.25\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics\",\"icon\":\"\\n\\n\\n\\n\\t\\n\\t\\t\\n\\t\\n\\t\\n\\t\\t\\n\\t\\n\\t\\n\\t\\t\\n\\t\\n\\n\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d8313939-3782-41b0-be29-b3ca20d8dd3a\",\"name\":\"Intercom\",\"dockerRepository\":\"airbyte/source-intercom\",\"dockerImageTag\":\"0.1.24\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/intercom\",\"icon\":\"\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"ef580275-d9a9-48bb-af5e-db0f5855be04\",\"name\":\"Webflow\",\"dockerRepository\":\"airbyte/source-webflow\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/webflow\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"c2281cee-86f9-4a86-bb48-d23286b4c7bd\",\"name\":\"Slack\",\"dockerRepository\":\"airbyte/source-slack\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/slack\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e\",\"name\":\"Linnworks\",\"dockerRepository\":\"airbyte/source-linnworks\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/linnworks\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"6e00b415-b02e-4160-bf02-58176a0ae687\",\"name\":\"Notion\",\"dockerRepository\":\"airbyte/source-notion\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/notion\",\"icon\":\"\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212\",\"name\":\"Airtable\",\"dockerRepository\":\"airbyte/source-airtable\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/airtable\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"6371b14b-bc68-4236-bfbd-468e8df8e968\",\"name\":\"PokeAPI\",\"dockerRepository\":\"airbyte/source-pokeapi\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pokeapi\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"5b9cb09e-1003-4f9c-983d-5779d1b2cd51\",\"name\":\"Mailgun\",\"dockerRepository\":\"airbyte/source-mailgun\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mailgun\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"80a54ea2-9959-4040-aac1-eee42423ec9b\",\"name\":\"Monday\",\"dockerRepository\":\"airbyte/source-monday\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/monday\",\"icon\":\"\\n\\n + \ \\n \\n \\n image/svg+xml\\n + \ \\n \\n \\n \\n \\n \\n Logo / monday.com\\n \\n + \ \\n \\n \\n \\n \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"6ff047c0-f5d5-4ce5-8c81-204a830fa7e1\",\"name\":\"AWS + CloudTrail\",\"dockerRepository\":\"airbyte/source-aws-cloudtrail\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/aws-cloudtrail\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"c47d6804-8b98-449f-970a-5ddb5cb5d7aa\",\"name\":\"Customer.io\",\"dockerRepository\":\"farosai/airbyte-customer-io-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/customer-io\",\"icon\":\"Logo-Color-NEW\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"686473f1-76d9-4994-9cc7-9b13da46147c\",\"name\":\"Chargebee\",\"dockerRepository\":\"airbyte/source-chargebee\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/chargebee\",\"icon\":\"\\n\\n + \ \\n \\n + \ \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"778daa7c-feaf-4db6-96f3-70fd645acc77\",\"name\":\"File\",\"dockerRepository\":\"airbyte/source-file\",\"dockerImageTag\":\"0.2.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/file\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"file\"},{\"sourceDefinitionId\":\"b117307c-14b6-41aa-9422-947e34922962\",\"name\":\"Salesforce\",\"dockerRepository\":\"airbyte/source-salesforce\",\"dockerImageTag\":\"1.0.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/salesforce\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e\",\"name\":\"MongoDb\",\"dockerRepository\":\"airbyte/source-mongodb-v2\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mongodb-v2\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"253487c0-2246-43ba-a21f-5116b20a2c50\",\"name\":\"Google + Ads\",\"dockerRepository\":\"airbyte/source-google-ads\",\"dockerImageTag\":\"0.1.44\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-ads\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"95e8cffd-b8c4-4039-968e-d32fb4a69bde\",\"name\":\"Klaviyo\",\"dockerRepository\":\"airbyte/source-klaviyo\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/klaviyo\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"90916976-a132-4ce9-8bce-82a03dd58788\",\"name\":\"BambooHR\",\"dockerRepository\":\"airbyte/source-bamboo-hr\",\"dockerImageTag\":\"0.2.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bamboo-hr\",\"icon\":\"\\n\\n \\n BambooHR\\n Created + with Sketch.\\n \\n \\n \\n + \ \\n \\n \\n \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"eb4c9e00-db83-4d63-a386-39cfa91012a8\",\"name\":\"Google + Search Console\",\"dockerRepository\":\"airbyte/source-google-search-console\",\"dockerImageTag\":\"0.1.13\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-search-console\",\"icon\":\"\\n\\n \\n Artboard\\n + \ Created with Sketch.\\n \\n \\n + \ \\n \\n + \ \\n \\n + \ \\n \\n + \ \\n + \ \\n + \ \\n \\n \\n \\n \\n + \ \\n + \ \\n \\n \\n \\n \\n \\n + \ \\n \\n + \ \\n \\n \\n + \ \\n \\n \\n\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"bad83517-5e54-4a3d-9b53-63e85fbd4d7c\",\"name\":\"ClickHouse\",\"dockerRepository\":\"airbyte/source-clickhouse\",\"dockerImageTag\":\"0.1.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/clickhouse\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"5b9cb09e-1003-4f9c-983d-5779d1b2cd51\",\"name\":\"Mailgun\",\"dockerRepository\":\"airbyte/source-mailgun\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mailgun\",\"icon\":\"\\n \ \\n \\n - \ \\n \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e094cb9a-26de-4645-8761-65c0c425d1de\",\"name\":\"Stripe\",\"dockerRepository\":\"airbyte/source-stripe\",\"dockerImageTag\":\"0.1.33\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/stripe\",\"icon\":\"\\n \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d913b0f2-cc51-4e55-a44c-8ba1697b9239\",\"name\":\"Paypal + Transaction\",\"dockerRepository\":\"airbyte/source-paypal-transaction\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/paypal-transaction\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"374ebc65-6636-4ea0-925c-7d35999a8ffc\",\"name\":\"Smartsheets\",\"dockerRepository\":\"airbyte/source-smartsheets\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/smartsheets\",\"icon\":\"\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"47f25999-dd5e-4636-8c39-e7cea2453331\",\"name\":\"Bing + Ads\",\"dockerRepository\":\"airbyte/source-bing-ads\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bing-ads\",\"icon\":\"\\n\\n \\n \\n \\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d60a46d4-709f-4092-a6b7-2457f7d455f5\",\"name\":\"Prestashop\",\"dockerRepository\":\"airbyte/source-prestashop\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/presta-shop\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"71607ba1-c0ac-4799-8049-7f4b90dd50f7\",\"name\":\"Google + Sheets\",\"dockerRepository\":\"airbyte/source-google-sheets\",\"dockerImageTag\":\"0.2.17\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-sheets\",\"icon\":\"\\n\\n\\n\\n\\t\\n\\t\\n\\t\\n\\n\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"file\"},{\"sourceDefinitionId\":\"d8540a80-6120-485d-b7d6-272bca477d9b\",\"name\":\"OpenWeather\",\"dockerRepository\":\"airbyte/source-openweather\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/openweather\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"45d2e135-2ede-49e1-939f-3e3ec357a65e\",\"name\":\"Recharge\",\"dockerRepository\":\"airbyte/source-recharge\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/recharge\",\"icon\":\"\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d53f9084-fa6b-4a5a-976c-5b8392f4ad8a\",\"name\":\"E2E + Testing\",\"dockerRepository\":\"airbyte/source-e2e-test\",\"dockerImageTag\":\"2.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/e2e-test\",\"icon\":\"\\n \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"cd06e646-31bf-4dc8-af48-cbc6530fcad3\",\"name\":\"Kustomer\",\"dockerRepository\":\"airbyte/source-kustomer-singer\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/kustomer\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b9dc6155-672e-42ea-b10d-9f1f1fb95ab1\",\"name\":\"Twilio\",\"dockerRepository\":\"airbyte/source-twilio\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/twilio\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"3052c77e-8b91-47e2-97a0-a29a22794b4b\",\"name\":\"PersistIq\",\"dockerRepository\":\"airbyte/source-persistiq\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/persistiq\",\"icon\":\"\\n + \ \\n \\n \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"1d4fdb25-64fc-4569-92da-fcdca79a8372\",\"name\":\"Okta\",\"dockerRepository\":\"airbyte/source-okta\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/okta\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"193bdcb8-1dd9-48d1-aade-91cadfd74f9b\",\"name\":\"Paystack\",\"dockerRepository\":\"airbyte/source-paystack\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/paystack\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"6f2ac653-8623-43c4-8950-19218c7caf3d\",\"name\":\"Firebolt\",\"dockerRepository\":\"airbyte/source-firebolt\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/firebolt\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"63cea06f-1c75-458d-88fe-ad48c7cb27fd\",\"name\":\"Braintree\",\"dockerRepository\":\"airbyte/source-braintree\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/braintree\",\"icon\":\"\\n\\n + \ \\n \\n + \ \\n \\n + \ \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b6604cbd-1b12-4c08-8767-e140d0fb0877\",\"name\":\"Chartmogul\",\"dockerRepository\":\"airbyte/source-chartmogul\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/chartmogul\",\"icon\":\"\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d8286229-c680-4063-8c59-23b9b391c700\",\"name\":\"Pipedrive\",\"dockerRepository\":\"airbyte/source-pipedrive\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pipedrive\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\t\\n\\t\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5\",\"name\":\"Zuora\",\"dockerRepository\":\"airbyte/source-zuora\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zuora\",\"icon\":\"\\n\\n\\nimage/svg+xml\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"59f1e50a-331f-4f09-b3e8-2e8d4d355f44\",\"name\":\"Greenhouse\",\"dockerRepository\":\"airbyte/source-greenhouse\",\"dockerImageTag\":\"0.2.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/greenhouse\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"789f8e7a-2d28-11ec-8d3d-0242ac130003\",\"name\":\"Lemlist\",\"dockerRepository\":\"airbyte/source-lemlist\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/lemlist\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"f1e4c7f6-db5c-4035-981f-d35ab4998794\",\"name\":\"Zenloop\",\"dockerRepository\":\"airbyte/source-zenloop\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zenloop\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"9e0556f4-69df-4522-a3fb-03264d36b348\",\"name\":\"Marketo\",\"dockerRepository\":\"airbyte/source-marketo\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/marketo\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"7f0455fb-4518-4ec0-b7a3-d808bf8081cc\",\"name\":\"Orb\",\"dockerRepository\":\"airbyte/source-orb\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/orb\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"435bb9a5-7887-4809-aa58-28c27df0d7ad\",\"name\":\"MySQL\",\"dockerRepository\":\"airbyte/source-mysql\",\"dockerImageTag\":\"0.6.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mysql\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"4942d392-c7b5-4271-91f9-3b4f4e51eb3e\",\"name\":\"ZohoCRM\",\"dockerRepository\":\"airbyte/source-zoho-crm\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/zoho-crm\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"9b2d3607-7222-4709-9fa2-c2abdebbdd88\",\"name\":\"Chargify\",\"dockerRepository\":\"airbyte/source-chargify\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/chargify\",\"icon\":\"\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b5ea17b1-f170-46dc-bc31-cc744ca984c1\",\"name\":\"Microsoft + SQL Server (MSSQL)\",\"dockerRepository\":\"airbyte/source-mssql\",\"dockerImageTag\":\"0.4.13\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mssql\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4\",\"name\":\"Zendesk + Chat\",\"dockerRepository\":\"airbyte/source-zendesk-chat\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-chat\",\"icon\":\"\\nimage/svg+xml\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"dfd88b22-b603-4c3d-aad7-3701784586b1\",\"name\":\"Faker\",\"dockerRepository\":\"airbyte/source-faker\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/source-faker\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"12928b32-bf0a-4f1e-964f-07e12e37153a\",\"name\":\"Mixpanel\",\"dockerRepository\":\"airbyte/source-mixpanel\",\"dockerImageTag\":\"0.1.18\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mixpanel\",\"icon\":\"\\n\\n\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d19ae824-e289-4b14-995a-0632eb46d246\",\"name\":\"Google + Directory\",\"dockerRepository\":\"airbyte/source-google-directory\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-directory\",\"icon\":\"\\n\\n\\n\\n + \ \\n\\n\\n \\n\\n\\n + \ \\n\\n\\n\\n \\n\\n\\n + \ \\n\\n\\n + \ \\n\\n\\n\\n \\n\\n\\n + \ \\n\\n\\n + \ \\n\\n\\n\\n \\n\\n\\n + \ \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n + \ \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n + \ \\n\\n\\n \\n\\n\\n\\n \\n\\n\\n + \ \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e2b40e36-aa0e-4bed-b41b-bcea6fa348b1\",\"name\":\"Exchange + Rates Api\",\"dockerRepository\":\"airbyte/source-exchange-rates\",\"dockerImageTag\":\"0.2.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/exchangeratesapi\",\"icon\":\"\\n\\n \\n logo\\n + \ Created with Sketch.\\n \\n \\n + \ \\n \\n \\n \\n + \ \\n \\n \\n \\n \\n \\n + \ \\n \\n \\n \\n + \ \\n \\n \\n \\n + \ \\n \\n + \ \\n \\n \\n + \ \\n \\n \\n \\n + \ \\n \\n \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"ef69ef6e-aa7f-4af1-a01d-ef775033524e\",\"name\":\"GitHub\",\"dockerRepository\":\"airbyte/source-github\",\"dockerImageTag\":\"0.2.44\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/github\",\"icon\":\"\\n\\n\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"0dad1a35-ccf8-4d03-b73e-6788c00b13ae\",\"name\":\"TiDB\",\"dockerRepository\":\"airbyte/source-tidb\",\"dockerImageTag\":\"0.2.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/tidb\",\"icon\":\"\\n + \ \\n \\n + \ \\n \\n \\n \\n \\n + \ \\n \\n + \ \\n + \ \\n + \ \\n + \ \\n \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"db04ecd1-42e7-4115-9cec-95812905c626\",\"name\":\"Retently\",\"dockerRepository\":\"airbyte/source-retently\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/retently\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d917a47b-8537-4d0d-8c10-36a9928d4265\",\"name\":\"Kafka\",\"dockerRepository\":\"airbyte/source-kafka\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/kafka\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"137ece28-5434-455c-8f34-69dc3782f451\",\"name\":\"LinkedIn + Ads\",\"dockerRepository\":\"airbyte/source-linkedin-ads\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/linkedin-ads\",\"icon\":\"\\n\\n\\n + \ \\n \\n \\n + \ \\n\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d0243522-dccf-4978-8ba0-37ed47a0bdbf\",\"name\":\"Asana\",\"dockerRepository\":\"airbyte/source-asana\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/asana\",\"icon\":\"\\n\\n + \ \\n \\n \\n \\n + \ \\n + \ \\n \\n \\n \\n + \ \\n \\n \\n + \ \\n + \ \\n + \ \\n + \ \\n \\n + \ \\n \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"9fa5862c-da7c-11eb-8d19-0242ac130003\",\"name\":\"Cockroachdb\",\"dockerRepository\":\"airbyte/source-cockroachdb\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/cockroachdb\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"36c891d9-4bd9-43ac-bad2-10e12756272c\",\"name\":\"HubSpot\",\"dockerRepository\":\"airbyte/source-hubspot\",\"dockerImageTag\":\"0.1.81\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/hubspot\",\"icon\":\"\\n\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e094cb9a-26de-4645-8761-65c0c425d1de\",\"name\":\"Stripe\",\"dockerRepository\":\"airbyte/source-stripe\",\"dockerImageTag\":\"0.1.35\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/stripe\",\"icon\":\"Asset 32Stone - Hub\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"c4cfaeda-c757-489a-8aba-859fb08b6970\",\"name\":\"US + Hub\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"c4cfaeda-c757-489a-8aba-859fb08b6970\",\"name\":\"US Census\",\"dockerRepository\":\"airbyte/source-us-census\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/us-census\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\t\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"fe2b4084-3386-4d3b-9ad6-308f61a6f1e6\",\"name\":\"Harvest\",\"dockerRepository\":\"airbyte/source-harvest\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/harvest\",\"icon\":\"\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"fe2b4084-3386-4d3b-9ad6-308f61a6f1e6\",\"name\":\"Harvest\",\"dockerRepository\":\"airbyte/source-harvest\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/harvest\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"b03a9f3e-22a5-11eb-adc1-0242ac120002\",\"name\":\"Mailchimp\",\"dockerRepository\":\"airbyte/source-mailchimp\",\"dockerImageTag\":\"0.2.14\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mailchimp\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b03a9f3e-22a5-11eb-adc1-0242ac120002\",\"name\":\"Mailchimp\",\"dockerRepository\":\"airbyte/source-mailchimp\",\"dockerImageTag\":\"0.2.14\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/mailchimp\",\"icon\":\"\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"983fd355-6bf3-4709-91b5-37afa391eeb6\",\"name\":\"Amazon - SQS\",\"dockerRepository\":\"airbyte/source-amazon-sqs\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amazon-sqs\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"0b5c867e-1b12-4d02-ab74-97b2184ff6d7\",\"name\":\"Dixa\",\"dockerRepository\":\"airbyte/source-dixa\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/dixa\",\"icon\":\"\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"6fe89830-d04d-401b-aad6-6552ffa5c4af\",\"name\":\"Harness\",\"dockerRepository\":\"farosai/airbyte-harness-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/harness\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"983fd355-6bf3-4709-91b5-37afa391eeb6\",\"name\":\"Amazon + SQS\",\"dockerRepository\":\"airbyte/source-amazon-sqs\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amazon-sqs\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"bc617b5f-1b9e-4a2d-bebe-782fd454a771\",\"name\":\"Timely\",\"dockerRepository\":\"airbyte/source-timely\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/timely\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"0b5c867e-1b12-4d02-ab74-97b2184ff6d7\",\"name\":\"Dixa\",\"dockerRepository\":\"airbyte/source-dixa\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/dixa\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"decd338e-5647-4c0b-adf4-da0e75f5a750\",\"name\":\"Postgres\",\"dockerRepository\":\"airbyte/source-postgres\",\"dockerImageTag\":\"0.4.28\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/postgres\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"decd338e-5647-4c0b-adf4-da0e75f5a750\",\"name\":\"Postgres\",\"dockerRepository\":\"airbyte/source-postgres\",\"dockerImageTag\":\"1.0.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/postgres\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"47f17145-fe20-4ef5-a548-e29b048adf84\",\"name\":\"Apify + style=\\\"stroke-width:3;\\\" d=\\\"M0,60.232\\\"/>\\r\\n\\r\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"47f17145-fe20-4ef5-a548-e29b048adf84\",\"name\":\"Apify Dataset\",\"dockerRepository\":\"airbyte/source-apify-dataset\",\"dockerImageTag\":\"0.1.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/apify-dataset\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e7eff203-90bf-43e5-a240-19ea3056c474\",\"name\":\"Typeform\",\"dockerRepository\":\"airbyte/source-typeform\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/typeform\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e7eff203-90bf-43e5-a240-19ea3056c474\",\"name\":\"Typeform\",\"dockerRepository\":\"airbyte/source-typeform\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/typeform\",\"icon\":\"\\n \ \\n \\n - \ \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"8da67652-004c-11ec-9a03-0242ac130003\",\"name\":\"Trello\",\"dockerRepository\":\"airbyte/source-trello\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/trello\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d78e5de0-aa44-4744-aa4f-74c818ccfe19\",\"name\":\"RKI + Covid\",\"dockerRepository\":\"airbyte/source-rki-covid\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/rki-covid\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"8da67652-004c-11ec-9a03-0242ac130003\",\"name\":\"Trello\",\"dockerRepository\":\"airbyte/source-trello\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/trello\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"445831eb-78db-4b1f-8f1f-0d96ad8739e2\",\"name\":\"Drift\",\"dockerRepository\":\"airbyte/source-drift\",\"dockerImageTag\":\"0.2.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/drift\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"445831eb-78db-4b1f-8f1f-0d96ad8739e2\",\"name\":\"Drift\",\"dockerRepository\":\"airbyte/source-drift\",\"dockerImageTag\":\"0.2.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/drift\",\"icon\":\"\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"5e6175e5-68e1-4c17-bff9-56103bbb0d80\",\"name\":\"Gitlab\",\"dockerRepository\":\"airbyte/source-gitlab\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/gitlab\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"5e6175e5-68e1-4c17-bff9-56103bbb0d80\",\"name\":\"Gitlab\",\"dockerRepository\":\"airbyte/source-gitlab\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/gitlab\",\"icon\":\"\\n\\n\\n\\t\\n\\tH: 2.5 x\\n\\t1/2 - x\\n\\t1x\\n\\t1x\\n\\t\\n\\t1x\\n\\t\\n\\t1x\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"9bb85338-ea95-4c93-b267-6be89125b267\",\"name\":\"Freshservice\",\"dockerRepository\":\"airbyte/source-freshservice\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/freshservice\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"3490c201-5d95-4783-b600-eaf07a4c7787\",\"name\":\"Outreach\",\"dockerRepository\":\"airbyte/source-outreach\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/outreach\",\"icon\":\"H: 2.5 x\\n\\t1/2 + x\\n\\t1x\\n\\t1x\\n\\t\\n\\t1x\\n\\t\\n\\t1x\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"3490c201-5d95-4783-b600-eaf07a4c7787\",\"name\":\"Outreach\",\"dockerRepository\":\"airbyte/source-outreach\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/outreach\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"6fe89830-d04d-401b-aad6-6552ffa5c4af\",\"name\":\"Harness\",\"dockerRepository\":\"farosai/airbyte-harness-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/harness\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"325e0640-e7b3-4e24-b823-3361008f603f\",\"name\":\"Zendesk + 1.549-4.273 3.835s.217 4.276 1.936 4.446z\\\" fill=\\\"#cd3131\\\"/>\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"3cc2eafd-84aa-4dca-93af-322d9dfeec1a\",\"name\":\"Google + Analytics Data API\",\"dockerRepository\":\"airbyte/source-google-analytics-data-api\",\"dockerImageTag\":\"0.0.2\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/google-analytics-v4\",\"icon\":\"\\n\\n\\n\\n\\t\\n\\t\\t\\n\\t\\n\\t\\n\\t\\t\\n\\t\\n\\t\\n\\t\\t\\n\\t\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"325e0640-e7b3-4e24-b823-3361008f603f\",\"name\":\"Zendesk Sunshine\",\"dockerRepository\":\"airbyte/source-zendesk-sunshine\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zendesk-sunshine\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"69589781-7828-43c5-9f63-8925b1c1ccc2\",\"name\":\"S3\",\"dockerRepository\":\"airbyte/source-s3\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/s3\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"69589781-7828-43c5-9f63-8925b1c1ccc2\",\"name\":\"S3\",\"dockerRepository\":\"airbyte/source-s3\",\"dockerImageTag\":\"0.1.18\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/s3\",\"icon\":\"\\n\\n Icon-Resource/Storage/Res_Amazon-Simple-Storage_Service-Standard_48_Light\\n @@ -18084,7 +16046,7 @@ interactions: 34.363,44.573 35.206,44.241 C37.373,43.384 38.493,42.308 38.537,41.047 L40.273,28.151 C41.298,28.392 42.14,28.514 42.826,28.514 C43.817,28.514 44.484,28.262 44.909,27.755 C45.26,27.338 45.397,26.813 45.297,26.277 L45.297,26.277 Z\\\" id=\\\"Amazon-Simple-Storage_Service-Standard_Resource-Icon_light-bg\\\" - fill=\\\"#3F8624\\\">\\n \\n\\n\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"7a4327c4-315a-11ec-8d3d-0242ac130003\",\"name\":\"Strava\",\"dockerRepository\":\"airbyte/source-strava\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/strava\",\"icon\":\"\\n \\n\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"file\"},{\"sourceDefinitionId\":\"7a4327c4-315a-11ec-8d3d-0242ac130003\",\"name\":\"Strava\",\"dockerRepository\":\"airbyte/source-strava\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/strava\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"7e20ce3e-d820-4327-ad7a-88f3927fd97a\",\"name\":\"VictorOps\",\"dockerRepository\":\"farosai/airbyte-victorops-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/victorops\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"77225a51-cd15-4a13-af02-65816bd0ecf4\",\"name\":\"Square\",\"dockerRepository\":\"airbyte/source-square\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/square\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"77225a51-cd15-4a13-af02-65816bd0ecf4\",\"name\":\"Square\",\"dockerRepository\":\"airbyte/source-square\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/square\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"eaf50f04-21dd-4620-913b-2a83f5635227\",\"name\":\"Microsoft + 85.0113 9.09738 82.9277 9.09738Z\\\" fill=\\\"#3E4348\\\"/>\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"eaf50f04-21dd-4620-913b-2a83f5635227\",\"name\":\"Microsoft teams\",\"dockerRepository\":\"airbyte/source-microsoft-teams\",\"dockerImageTag\":\"0.2.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/microsoft-teams\",\"icon\":\"\\n\\n\\n\\t\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d78e5de0-aa44-4744-aa4f-74c818ccfe19\",\"name\":\"RKI - Covid\",\"dockerRepository\":\"airbyte/source-rki-covid\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/rki-covid\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e55879a8-0ef8-4557-abcf-ab34c53ec460\",\"name\":\"Amazon - Seller Partner\",\"dockerRepository\":\"airbyte/source-amazon-seller-partner\",\"dockerImageTag\":\"0.2.22\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amazon-seller-partner\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e55879a8-0ef8-4557-abcf-ab34c53ec460\",\"name\":\"Amazon + Seller Partner\",\"dockerRepository\":\"airbyte/source-amazon-seller-partner\",\"dockerImageTag\":\"0.2.24\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amazon-seller-partner\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c\",\"name\":\"Looker\",\"dockerRepository\":\"airbyte/source-looker\",\"dockerImageTag\":\"0.2.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/looker\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c\",\"name\":\"Looker\",\"dockerRepository\":\"airbyte/source-looker\",\"dockerImageTag\":\"0.2.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/looker\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"ed799e2b-2158-4c66-8da4-b40fe63bc72a\",\"name\":\"Plaid\",\"dockerRepository\":\"airbyte/source-plaid\",\"dockerImageTag\":\"0.3.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/plaid\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"ed799e2b-2158-4c66-8da4-b40fe63bc72a\",\"name\":\"Plaid\",\"dockerRepository\":\"airbyte/source-plaid\",\"dockerImageTag\":\"0.3.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/plaid\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2\",\"name\":\"Snowflake\",\"dockerRepository\":\"airbyte/source-snowflake\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/snowflake\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"7cf88806-25f5-4e1a-b422-b2fa9e1b0090\",\"name\":\"Elasticsearch\",\"dockerRepository\":\"airbyte/source-elasticsearch\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/elasticsearch\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2\",\"name\":\"Snowflake\",\"dockerRepository\":\"airbyte/source-snowflake\",\"dockerImageTag\":\"0.1.15\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/snowflake\",\"icon\":\"\\n\\n \\r\\n\\r\\n\\r\\n\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"c7cb421b-942e-4468-99ee-e369bcabaec5\",\"name\":\"Metabase\",\"dockerRepository\":\"airbyte/source-metabase\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/metabase\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"ec4b9503-13cb-48ab-a4ab-6ade4be46567\",\"name\":\"Freshdesk\",\"dockerRepository\":\"airbyte/source-freshdesk\",\"dockerImageTag\":\"0.3.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/freshdesk\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d60f5393-f99e-4310-8d05-b1876820f40e\",\"name\":\"Pivotal - Tracker\",\"dockerRepository\":\"airbyte/source-pivotal-tracker\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pivotal-tracker\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"afa734e4-3571-11ec-991a-1e0031268139\",\"name\":\"YouTube + \ id=\\\"path16\\\" />\\n\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d60f5393-f99e-4310-8d05-b1876820f40e\",\"name\":\"Pivotal + Tracker\",\"dockerRepository\":\"airbyte/source-pivotal-tracker\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pivotal-tracker\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"afa734e4-3571-11ec-991a-1e0031268139\",\"name\":\"YouTube Analytics\",\"dockerRepository\":\"airbyte/source-youtube-analytics\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/youtube-analytics\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"6acf6b55-4f1e-4fca-944e-1a3caef8aba8\",\"name\":\"Instagram\",\"dockerRepository\":\"airbyte/source-instagram\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/instagram\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"6acf6b55-4f1e-4fca-944e-1a3caef8aba8\",\"name\":\"Instagram\",\"dockerRepository\":\"airbyte/source-instagram\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/instagram\",\"icon\":\"\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"b08e4776-d1de-4e80-ab5c-1e51dad934a2\",\"name\":\"Qualaroo\",\"dockerRepository\":\"airbyte/source-qualaroo\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/qualaroo\",\"icon\":\"\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"b08e4776-d1de-4e80-ab5c-1e51dad934a2\",\"name\":\"Qualaroo\",\"dockerRepository\":\"airbyte/source-qualaroo\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/qualaroo\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"badc5925-0485-42be-8caa-b34096cb71b5\",\"name\":\"SurveyMonkey\",\"dockerRepository\":\"airbyte/source-surveymonkey\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/surveymonkey\",\"icon\":\"\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"badc5925-0485-42be-8caa-b34096cb71b5\",\"name\":\"SurveyMonkey\",\"dockerRepository\":\"airbyte/source-surveymonkey\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/surveymonkey\",\"icon\":\"Horizontal_Sabaeus_RGB\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"722ba4bf-06ec-45a4-8dd5-72e4a5cf3903\",\"name\":\"My + transform=\\\"translate(-31.32 -31.32)\\\"/>\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"722ba4bf-06ec-45a4-8dd5-72e4a5cf3903\",\"name\":\"My Hours\",\"dockerRepository\":\"airbyte/source-my-hours\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/my-hours\",\"icon\":\"\\n\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e7778cfc-e97c-4458-9ecb-b4f2bba8946c\",\"name\":\"Facebook - Marketing\",\"dockerRepository\":\"airbyte/source-facebook-marketing\",\"dockerImageTag\":\"0.2.53\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/facebook-marketing\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"547dc08e-ab51-421d-953b-8f3745201a8c\",\"name\":\"Kyriba\",\"dockerRepository\":\"airbyte/source-kyriba\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/kyriba\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e7778cfc-e97c-4458-9ecb-b4f2bba8946c\",\"name\":\"Facebook + Marketing\",\"dockerRepository\":\"airbyte/source-facebook-marketing\",\"dockerImageTag\":\"0.2.58\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/facebook-marketing\",\"icon\":\"\\n\\n\",\"releaseStage\":\"generally_available\"},{\"sourceDefinitionId\":\"bb1a6d31-6879-4819-a2bd-3eed299ea8e2\",\"name\":\"Cart.com\",\"dockerRepository\":\"airbyte/source-cart\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/cart\",\"icon\":\"\\n\",\"releaseStage\":\"generally_available\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"bb1a6d31-6879-4819-a2bd-3eed299ea8e2\",\"name\":\"Cart.com\",\"dockerRepository\":\"airbyte/source-cart\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/cart\",\"icon\":\"\\r\\n\\r\\n\\r\\n\\t\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"f00d2cf4-3c28-499a-ba93-b50b6f26359e\",\"name\":\"TalkDesk - Explore\",\"dockerRepository\":\"airbyte/source-talkdesk-explore\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/talkdesk-explore\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"c7cb421b-942e-4468-99ee-e369bcabaec5\",\"name\":\"Metabase\",\"dockerRepository\":\"airbyte/source-metabase\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/metabase\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"41991d12-d4b5-439e-afd0-260a31d4c53f\",\"name\":\"SalesLoft\",\"dockerRepository\":\"airbyte/source-salesloft\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/salesloft\",\"icon\":\"\\r\\n\\t\\t\\r\\n\\t\\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"f00d2cf4-3c28-499a-ba93-b50b6f26359e\",\"name\":\"TalkDesk + Explore\",\"dockerRepository\":\"airbyte/source-talkdesk-explore\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/talkdesk-explore\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"7e20ce3e-d820-4327-ad7a-88f3927fd97a\",\"name\":\"VictorOps\",\"dockerRepository\":\"farosai/airbyte-victorops-source\",\"dockerImageTag\":\"0.1.23\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/victorops\",\"icon\":\"\\n\\n\\t\\n\\t\\t\\n\\t\\t\\n\\t\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"41991d12-d4b5-439e-afd0-260a31d4c53f\",\"name\":\"SalesLoft\",\"dockerRepository\":\"airbyte/source-salesloft\",\"dockerImageTag\":\"0.1.3\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/salesloft\",\"icon\":\"\\n\\n \\n \\n - \ \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35\",\"name\":\"TikTok - Marketing\",\"dockerRepository\":\"airbyte/source-tiktok-marketing\",\"dockerImageTag\":\"0.1.12\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/tiktok-marketing\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35\",\"name\":\"TikTok + Marketing\",\"dockerRepository\":\"airbyte/source-tiktok-marketing\",\"dockerImageTag\":\"0.1.14\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/tiktok-marketing\",\"icon\":\"\\n\\n \\r\\n\\r\\n\\r\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"798ae795-5189-42b6-b64e-3cb91db93338\",\"name\":\"Azure - Table Storage\",\"dockerRepository\":\"airbyte/source-azure-table\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/azure-table\",\"icon\":\"\\r\\n\\r\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"3981c999-bd7d-4afc-849b-e53dea90c948\",\"name\":\"Lever + Hiring\",\"dockerRepository\":\"airbyte/source-lever-hiring\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/lever-hiring\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"798ae795-5189-42b6-b64e-3cb91db93338\",\"name\":\"Azure + Table Storage\",\"dockerRepository\":\"airbyte/source-azure-table\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/azure-table\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"9da77001-af33-4bcd-be46-6252bf9342b9\",\"name\":\"Shopify\",\"dockerRepository\":\"airbyte/source-shopify\",\"dockerImageTag\":\"0.1.37\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/shopify\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"9da77001-af33-4bcd-be46-6252bf9342b9\",\"name\":\"Shopify\",\"dockerRepository\":\"airbyte/source-shopify\",\"dockerImageTag\":\"0.1.37\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/shopify\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"d1aa448b-7c54-498e-ad95-263cbebcd2db\",\"name\":\"Tempo\",\"dockerRepository\":\"airbyte/source-tempo\",\"dockerImageTag\":\"0.2.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/tempo\",\"icon\":\"\\n\\n\\n \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"5cb7e5fe-38c2-11ec-8d3d-0242ac130003\",\"name\":\"Pinterest\",\"dockerRepository\":\"airbyte/source-pinterest\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pinterest\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"5cb7e5fe-38c2-11ec-8d3d-0242ac130003\",\"name\":\"Pinterest\",\"dockerRepository\":\"airbyte/source-pinterest\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/pinterest\",\"icon\":\"\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"68e63de2-bb83-4c7e-93fa-a8a9051e3993\",\"name\":\"Jira\",\"dockerRepository\":\"airbyte/source-jira\",\"dockerImageTag\":\"0.2.20\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/jira\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"68e63de2-bb83-4c7e-93fa-a8a9051e3993\",\"name\":\"Jira\",\"dockerRepository\":\"airbyte/source-jira\",\"dockerImageTag\":\"0.2.20\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/jira\",\"icon\":\"\\n\\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"af6d50ee-dddf-4126-a8ee-7faee990774f\",\"name\":\"PostHog\",\"dockerRepository\":\"airbyte/source-posthog\",\"dockerImageTag\":\"0.1.6\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/posthog\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"af6d50ee-dddf-4126-a8ee-7faee990774f\",\"name\":\"PostHog\",\"dockerRepository\":\"airbyte/source-posthog\",\"dockerImageTag\":\"0.1.7\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/posthog\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"cc88c43f-6f53-4e8a-8c4d-b284baaf9635\",\"name\":\"Delighted\",\"dockerRepository\":\"airbyte/source-delighted\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/delighted\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"cc88c43f-6f53-4e8a-8c4d-b284baaf9635\",\"name\":\"Delighted\",\"dockerRepository\":\"airbyte/source-delighted\",\"dockerImageTag\":\"0.1.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/delighted\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"a827c52e-791c-4135-a245-e233c5255199\",\"name\":\"SFTP\",\"dockerRepository\":\"airbyte/source-sftp\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/sftp\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"cdaf146a-9b75-49fd-9dd2-9d64a0bb4781\",\"name\":\"Sentry\",\"dockerRepository\":\"airbyte/source-sentry\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/sentry\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"aea2fd0d-377d-465e-86c0-4fdc4f688e51\",\"name\":\"Zoom\",\"dockerRepository\":\"airbyte/source-zoom-singer\",\"dockerImageTag\":\"0.2.4\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/zoom\",\"icon\":\"\\n\\n \\n \\n + \ \\n image/svg+xml\\n + \ \\n \\n \\n \\n + \ \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"a827c52e-791c-4135-a245-e233c5255199\",\"name\":\"SFTP\",\"dockerRepository\":\"airbyte/source-sftp\",\"dockerImageTag\":\"0.1.2\",\"documentationUrl\":\"https://docs.airbyte.com/integrations/sources/sftp\",\"releaseStage\":\"alpha\",\"sourceType\":\"file\"},{\"sourceDefinitionId\":\"95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a\",\"name\":\"Orbit\",\"dockerRepository\":\"airbyte/source-orbit\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/orbit\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"d1aa448b-7c54-498e-ad95-263cbebcd2db\",\"name\":\"Tempo\",\"dockerRepository\":\"airbyte/source-tempo\",\"dockerImageTag\":\"0.2.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/tempo\",\"icon\":\"\\n\\n\\n \\n \\n \\n \\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"cdaf146a-9b75-49fd-9dd2-9d64a0bb4781\",\"name\":\"Sentry\",\"dockerRepository\":\"airbyte/source-sentry\",\"dockerImageTag\":\"0.1.1\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/sentry\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734\",\"name\":\"Google + 5.274h.842l1.77-5.283h.018c0 .587-.018 1.385-.018 1.568v3.715h.824v-6.291h-1.209v.001z\\\"/>\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734\",\"name\":\"Google Workspace Admin Reports\",\"dockerRepository\":\"airbyte/source-google-workspace-admin-reports\",\"dockerImageTag\":\"0.1.8\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports\",\"icon\":\"\\n \\n \\n \\n \ \\n \\n\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"e87ffa8e-a3b5-f69c-9076-6011339de1f6\",\"name\":\"Redshift\",\"dockerRepository\":\"airbyte/source-redshift\",\"dockerImageTag\":\"0.3.10\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/redshift\",\"icon\":\"\\n \\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"cf8ff320-6272-4faa-89e6-4402dc17e5d5\",\"name\":\"Glassfrog\",\"dockerRepository\":\"airbyte/source-glassfrog\",\"dockerImageTag\":\"0.1.0\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/glassfrog\",\"icon\":\"\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"e87ffa8e-a3b5-f69c-9076-6011339de1f6\",\"name\":\"Redshift\",\"dockerRepository\":\"airbyte/source-redshift\",\"dockerImageTag\":\"0.3.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/redshift\",\"icon\":\"\",\"releaseStage\":\"alpha\"},{\"sourceDefinitionId\":\"c6b0a29e-1da9-4512-9002-7bfd0cba2246\",\"name\":\"Amazon - Ads\",\"dockerRepository\":\"airbyte/source-amazon-ads\",\"dockerImageTag\":\"0.1.9\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amazon-ads\",\"icon\":\"\",\"releaseStage\":\"alpha\",\"sourceType\":\"database\"},{\"sourceDefinitionId\":\"c6b0a29e-1da9-4512-9002-7bfd0cba2246\",\"name\":\"Amazon + Ads\",\"dockerRepository\":\"airbyte/source-amazon-ads\",\"dockerImageTag\":\"0.1.11\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/amazon-ads\",\"icon\":\"\",\"releaseStage\":\"beta\"},{\"sourceDefinitionId\":\"59c5501b-9f95-411e-9269-7143c939adbd\",\"name\":\"BigCommerce\",\"dockerRepository\":\"airbyte/source-bigcommerce\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bigcommerce\",\"icon\":\"\",\"releaseStage\":\"beta\",\"sourceType\":\"api\"},{\"sourceDefinitionId\":\"59c5501b-9f95-411e-9269-7143c939adbd\",\"name\":\"BigCommerce\",\"dockerRepository\":\"airbyte/source-bigcommerce\",\"dockerImageTag\":\"0.1.5\",\"documentationUrl\":\"https://docs.airbyte.io/integrations/sources/bigcommerce\",\"icon\":\"\",\"releaseStage\":\"alpha\"}]}" + 27 10.4-27c.1-.4.5-.6.9-.6h8.6c.3 0 .5.2.5.5v38.4c0 .3-.2.5-.5.5h-6c-.2 0-.4-.2-.4-.5z\\\"/>\",\"releaseStage\":\"alpha\",\"sourceType\":\"api\"}]}" headers: Access-Control-Allow-Headers: - Origin, Content-Type, Accept, Content-Encoding @@ -24117,9 +25459,9 @@ interactions: Content-Type: - application/json Date: - - Wed, 29 Jun 2022 10:53:06 GMT + - Thu, 11 Aug 2022 08:55:10 GMT Server: - - nginx/1.19.10 + - nginx/1.23.1 Transfer-Encoding: - chunked status: diff --git a/octavia-cli/integration_tests/configurations/connections/poke_to_pg/configuration.yaml b/octavia-cli/integration_tests/configurations/connections/poke_to_pg/configuration.yaml index 9788994d1eaf..1e343e606fb6 100644 --- a/octavia-cli/integration_tests/configurations/connections/poke_to_pg/configuration.yaml +++ b/octavia-cli/integration_tests/configurations/connections/poke_to_pg/configuration.yaml @@ -15,9 +15,11 @@ configuration: cpu_request: "" # OPTIONAL memory_limit: "" # OPTIONAL memory_request: "" # OPTIONAL - schedule: # OPTIONAL | object - time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months - units: 1 # REQUIRED | integer + schedule_type: basic + schedule_data: + basic_schedule: + time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed. streams: - config: diff --git a/octavia-cli/integration_tests/configurations/connections/poke_to_pg_normalization/configuration.yaml b/octavia-cli/integration_tests/configurations/connections/poke_to_pg_normalization/configuration.yaml index 876b30803133..a5225215f9db 100644 --- a/octavia-cli/integration_tests/configurations/connections/poke_to_pg_normalization/configuration.yaml +++ b/octavia-cli/integration_tests/configurations/connections/poke_to_pg_normalization/configuration.yaml @@ -15,9 +15,11 @@ configuration: cpu_request: "" # OPTIONAL memory_limit: "" # OPTIONAL memory_request: "" # OPTIONAL - schedule: # OPTIONAL | object - time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months - units: 1 # REQUIRED | integer + schedule_type: basic + schedule_data: + basic_schedule: + time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer operations: - name: "Normalization" operator_configuration: diff --git a/octavia-cli/integration_tests/test_apply/test_resources.py b/octavia-cli/integration_tests/test_apply/test_resources.py index be7dfe19b89d..7ecc65a2d6a6 100644 --- a/octavia-cli/integration_tests/test_apply/test_resources.py +++ b/octavia-cli/integration_tests/test_apply/test_resources.py @@ -42,12 +42,10 @@ def test_connection_lifecycle(source, destination, connection, workspace_id): connection.create() connection.state = connection._get_state_from_file(connection.configuration_path, workspace_id) assert connection.was_created - assert not connection.get_diff_with_remote_resource() connection.raw_configuration["configuration"]["status"] = "inactive" connection.configuration = connection._deserialize_raw_configuration() assert 'changed from "active" to "inactive"' in connection.get_diff_with_remote_resource() connection.update() - assert not connection.get_diff_with_remote_resource() def test_connection_lifecycle_with_normalization(source, destination, connection_with_normalization, workspace_id): @@ -61,9 +59,7 @@ def test_connection_lifecycle_with_normalization(source, destination, connection assert connection_with_normalization.was_created assert connection_with_normalization.remote_resource["operations"][0]["operation_id"] is not None assert connection_with_normalization.remote_resource["operations"][0]["operator_configuration"]["normalization"]["option"] == "basic" - assert not connection_with_normalization.get_diff_with_remote_resource() connection_with_normalization.raw_configuration["configuration"]["status"] = "inactive" connection_with_normalization.configuration = connection_with_normalization._deserialize_raw_configuration() assert 'changed from "active" to "inactive"' in connection_with_normalization.get_diff_with_remote_resource() connection_with_normalization.update() - assert not connection_with_normalization.get_diff_with_remote_resource() diff --git a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_with_normalization.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_with_normalization.yaml index 09f3dacdb314..2ec8abc12173 100644 --- a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_with_normalization.yaml +++ b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_with_normalization.yaml @@ -15,9 +15,14 @@ configuration: cpu_request: "" # OPTIONAL memory_limit: "" # OPTIONAL memory_request: "" # OPTIONAL - schedule: # OPTIONAL | object - time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months - units: 1 # REQUIRED | integer + schedule_type: basic # OPTIONAL | string | Allowed values: basic, cron, manual + schedule_data: # OPTIONAL | object + basic_schedule: + time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer + # cron: + # cron_time_zone: "UTC" # REQUIRED | string + # cron_expression: "* */2 * * * ?" # REQUIRED | string # operations: ## -------- Uncomment and edit the block below if you want to enable Airbyte normalization -------- # - name: "Normalization" diff --git a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_without_normalization.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_without_normalization.yaml index eaf91133166e..6cfe86d70e46 100644 --- a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_without_normalization.yaml +++ b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_without_normalization.yaml @@ -15,9 +15,14 @@ configuration: cpu_request: "" # OPTIONAL memory_limit: "" # OPTIONAL memory_request: "" # OPTIONAL - schedule: # OPTIONAL | object - time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months - units: 1 # REQUIRED | integer + schedule_type: basic # OPTIONAL | string | Allowed values: basic, cron, manual + schedule_data: # OPTIONAL | object + basic_schedule: + time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer + # cron: + # cron_time_zone: "UTC" # REQUIRED | string + # cron_expression: "* */2 * * * ?" # REQUIRED | string sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed. streams: - config: diff --git a/octavia-cli/integration_tests/test_import/octavia_project_to_migrate/connections/poke_to_pg_to_import/configuration.yaml b/octavia-cli/integration_tests/test_import/octavia_project_to_migrate/connections/poke_to_pg_to_import/configuration.yaml index f6e2f67fb7ce..5b4bbfd91e15 100644 --- a/octavia-cli/integration_tests/test_import/octavia_project_to_migrate/connections/poke_to_pg_to_import/configuration.yaml +++ b/octavia-cli/integration_tests/test_import/octavia_project_to_migrate/connections/poke_to_pg_to_import/configuration.yaml @@ -15,9 +15,11 @@ configuration: cpu_request: "" # OPTIONAL memory_limit: "" # OPTIONAL memory_request: "" # OPTIONAL - schedule: # OPTIONAL | object - time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months - units: 1 # REQUIRED | integer + schedule_type: basic + schedule_data: + basic_schedule: + time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer operations: - name: "Normalization" operator_configuration: diff --git a/octavia-cli/octavia_cli/_import/commands.py b/octavia-cli/octavia_cli/_import/commands.py index 33795ccc852c..846d4d33fd48 100644 --- a/octavia-cli/octavia_cli/_import/commands.py +++ b/octavia-cli/octavia_cli/_import/commands.py @@ -85,6 +85,8 @@ def import_connection( str: The generated import message. """ remote_configuration = json.loads(get_json_representation(api_client, workspace_id, UnmanagedConnection, resource_to_get)) + # Since #15253 "schedule" is deprecated + remote_configuration.pop("schedule", None) source_name, destination_name = remote_configuration["source"]["name"], remote_configuration["destination"]["name"] source_configuration_path = renderers.ConnectorSpecificationRenderer.get_output_path( project_path=".", definition_type="source", resource_name=source_name diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py index 101e3ee0de9f..714831798d4e 100644 --- a/octavia-cli/octavia_cli/apply/resources.py +++ b/octavia-cli/octavia_cli/apply/resources.py @@ -24,7 +24,10 @@ from airbyte_api_client.model.airbyte_stream_and_configuration import AirbyteStreamAndConfiguration from airbyte_api_client.model.airbyte_stream_configuration import AirbyteStreamConfiguration from airbyte_api_client.model.connection_read import ConnectionRead -from airbyte_api_client.model.connection_schedule import ConnectionSchedule +from airbyte_api_client.model.connection_schedule_data import ConnectionScheduleData +from airbyte_api_client.model.connection_schedule_data_basic_schedule import ConnectionScheduleDataBasicSchedule +from airbyte_api_client.model.connection_schedule_data_cron import ConnectionScheduleDataCron +from airbyte_api_client.model.connection_schedule_type import ConnectionScheduleType from airbyte_api_client.model.connection_status import ConnectionStatus from airbyte_api_client.model.destination_create import DestinationCreate from airbyte_api_client.model.destination_definition_id_with_workspace_id import DestinationDefinitionIdWithWorkspaceId @@ -577,6 +580,7 @@ class Connection(BaseResource): "is_syncing", "latest_sync_job_status", "latest_sync_job_created_at", + "schedule", ] # We do not allow local editing of these keys # We do not allow local editing of these keys @@ -595,8 +599,20 @@ def _deserialize_raw_configuration(self): self._check_for_legacy_connection_configuration_keys(configuration) configuration["sync_catalog"] = self._create_configured_catalog(configuration["sync_catalog"]) configuration["namespace_definition"] = NamespaceDefinitionType(configuration["namespace_definition"]) - if "schedule" in configuration: - configuration["schedule"] = ConnectionSchedule(**configuration["schedule"]) + + if "schedule_type" in configuration: + # If schedule type is manual we do not expect a schedule_data field to be set + # TODO: sending a WebConnectionCreate payload without schedule_data (for manual) fails. + is_manual = configuration["schedule_type"] == "manual" + configuration["schedule_type"] = ConnectionScheduleType(configuration["schedule_type"]) + if not is_manual: + if "basic_schedule" in configuration["schedule_data"]: + basic_schedule = ConnectionScheduleDataBasicSchedule(**configuration["schedule_data"]["basic_schedule"]) + configuration["schedule_data"]["basic_schedule"] = basic_schedule + if "cron" in configuration["schedule_data"]: + cron = ConnectionScheduleDataCron(**configuration["schedule_data"]["cron"]) + configuration["schedule_data"]["cron"] = cron + configuration["schedule_data"] = ConnectionScheduleData(**configuration["schedule_data"]) if "resource_requirements" in configuration: configuration["resource_requirements"] = ResourceRequirements(**configuration["resource_requirements"]) configuration["status"] = ConnectionStatus(configuration["status"]) @@ -738,8 +754,16 @@ def _deserialize_operations( deserialized_operations.append(operation) return deserialized_operations - # TODO this check can be removed when all our active user are on >= 0.37.0 def _check_for_legacy_connection_configuration_keys(self, configuration_to_check): + self._check_for_wrong_casing_in_connection_configurations_keys(configuration_to_check) + self._check_for_schedule_in_connection_configurations_keys(configuration_to_check) + + # TODO this check can be removed when all our active user are on >= 0.37.0 + def _check_for_schedule_in_connection_configurations_keys(self, configuration_to_check): + error_message = "The schedule key is deprecated since 0.40.0, please use a combination of schedule_type and schedule_data" + self._check_for_invalid_configuration_keys(configuration_to_check, {"schedule"}, error_message) + + def _check_for_wrong_casing_in_connection_configurations_keys(self, configuration_to_check): """We changed connection configuration keys from camelCase to snake_case in 0.37.0. This function check if the connection configuration has some camelCase keys and display a meaningful error message. Args: diff --git a/octavia-cli/octavia_cli/generate/templates/connection.yaml.j2 b/octavia-cli/octavia_cli/generate/templates/connection.yaml.j2 index 78d1fe8ba937..58b36cba53cd 100644 --- a/octavia-cli/octavia_cli/generate/templates/connection.yaml.j2 +++ b/octavia-cli/octavia_cli/generate/templates/connection.yaml.j2 @@ -15,9 +15,14 @@ configuration: cpu_request: "" # OPTIONAL memory_limit: "" # OPTIONAL memory_request: "" # OPTIONAL - schedule: # OPTIONAL | object - time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months - units: 1 # REQUIRED | integer + schedule_type: basic # OPTIONAL | string | Allowed values: basic, cron, manual + schedule_data: # OPTIONAL | object + basic_schedule: + time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer + # cron: + # cron_time_zone: "UTC" # REQUIRED | string + # cron_expression: "* */2 * * * ?" # REQUIRED | string {%- if supports_normalization or supports_dbt%} # operations: {%- endif %} diff --git a/octavia-cli/unit_tests/test_apply/test_resources.py b/octavia-cli/unit_tests/test_apply/test_resources.py index 3d4cfe6c41f2..66014d435a42 100644 --- a/octavia-cli/unit_tests/test_apply/test_resources.py +++ b/octavia-cli/unit_tests/test_apply/test_resources.py @@ -8,7 +8,8 @@ import pytest from airbyte_api_client import ApiException from airbyte_api_client.model.airbyte_catalog import AirbyteCatalog -from airbyte_api_client.model.connection_schedule import ConnectionSchedule +from airbyte_api_client.model.connection_schedule_data_basic_schedule import ConnectionScheduleDataBasicSchedule +from airbyte_api_client.model.connection_schedule_type import ConnectionScheduleType from airbyte_api_client.model.connection_status import ConnectionStatus from airbyte_api_client.model.destination_definition_id_with_workspace_id import DestinationDefinitionIdWithWorkspaceId from airbyte_api_client.model.namespace_definition_type import NamespaceDefinitionType @@ -468,12 +469,20 @@ def connection_configuration(self): } ] }, - "schedule": {"units": 1, "time_unit": "days"}, + "schedule_type": "basic", + "schedule_data": {"units": 1, "time_unit": "days"}, "status": "active", "resource_requirements": {"cpu_request": "foo", "cpu_limit": "foo", "memory_request": "foo", "memory_limit": "foo"}, }, } + @pytest.fixture + def connection_configuration_with_manual_schedule(self, connection_configuration): + connection_configuration_with_manual_schedule = deepcopy(connection_configuration) + connection_configuration_with_manual_schedule["configuration"]["schedule_type"] = "manual" + connection_configuration_with_manual_schedule["configuration"]["schedule_data"] = None + return connection_configuration_with_manual_schedule + @pytest.fixture def connection_configuration_with_normalization(self, connection_configuration): connection_configuration_with_normalization = deepcopy(connection_configuration) @@ -559,6 +568,28 @@ def legacy_connection_configurations(self): "resource_requirements": {"cpu_request": "foo", "cpu_limit": "foo", "memory_request": "foo", "memory_limit": "foo"}, }, }, + { + "definition_type": "connection", + "resource_name": "my_connection", + "source_id": "my_source", + "destination_id": "my_destination", + "configuration": { + "namespace_definition": "customformat", + "namespace_format": "foo", + "prefix": "foo", + "sync_catalog": { + "streams": [ + { + "stream": {}, + "config": {}, + } + ] + }, + "schedule": {"units": 1, "time_unit": "days"}, + "status": "active", + "resource_requirements": {"cpu_request": "foo", "cpu_limit": "foo", "memory_request": "foo", "memory_limit": "foo"}, + }, + }, ] @pytest.mark.parametrize( @@ -769,14 +800,18 @@ def test_update(self, mocker, mock_api_client, connection_configuration): assert update_result == resource._create_or_update.return_value resource._create_or_update.assert_called_with(resource._update_fn, resource.update_payload) - def test__deserialize_raw_configuration(self, mock_api_client, connection_configuration): + def test__deserialize_raw_configuration(self, mock_api_client, connection_configuration, connection_configuration_with_manual_schedule): resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml") configuration = resource._deserialize_raw_configuration() assert isinstance(configuration["sync_catalog"], AirbyteCatalog) assert configuration["namespace_definition"] == NamespaceDefinitionType( connection_configuration["configuration"]["namespace_definition"] ) - assert configuration["schedule"] == ConnectionSchedule(**connection_configuration["configuration"]["schedule"]) + assert configuration["schedule_type"] == ConnectionScheduleType(connection_configuration["configuration"]["schedule_type"]) + assert ( + configuration["schedule_data"].to_dict() + == ConnectionScheduleDataBasicSchedule(**connection_configuration["configuration"]["schedule_data"]).to_dict() + ) assert configuration["resource_requirements"] == ResourceRequirements( **connection_configuration["configuration"]["resource_requirements"] ) @@ -786,11 +821,19 @@ def test__deserialize_raw_configuration(self, mock_api_client, connection_config "namespace_format", "prefix", "sync_catalog", - "schedule", + "schedule_type", + "schedule_data", "status", "resource_requirements", ] + resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration_with_manual_schedule, "bar.yaml") + configuration = resource._deserialize_raw_configuration() + assert configuration["schedule_type"] == ConnectionScheduleType( + connection_configuration_with_manual_schedule["configuration"]["schedule_type"] + ) + assert configuration["schedule_data"] is None + def test__deserialize_operations(self, mock_api_client, connection_configuration): resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml") operations = [ diff --git a/tools/ci_credentials/ci_credentials/secrets_loader.py b/tools/ci_credentials/ci_credentials/secrets_loader.py index 4cf8b70f092a..60979c96b1be 100644 --- a/tools/ci_credentials/ci_credentials/secrets_loader.py +++ b/tools/ci_credentials/ci_credentials/secrets_loader.py @@ -145,7 +145,7 @@ def mask_secrets_from_action_log(self, key, value): for pattern in MASK_KEY_PATTERNS: if re.search(pattern, key): self.logger.info(f"Add mask for key: {key}") - for line in value.splitlines(): + for line in str(value).splitlines(): line = str(line).strip() # don't output } and such if len(line) > 1 and not os.getenv("VERSION") == "dev": diff --git a/tools/ci_credentials/tests/test_secrets.py b/tools/ci_credentials/tests/test_secrets.py index b0436d13e4d2..84173c1df5f9 100644 --- a/tools/ci_credentials/tests/test_secrets.py +++ b/tools/ci_credentials/tests/test_secrets.py @@ -166,3 +166,17 @@ def test_write(connector_name, secrets, expected_files): has = True break assert has, f"incorrect file data: {target_file}" + + +@pytest.mark.parametrize( + "connector_name,dict_json_value,expected_secret", + ( + ("source-default", "{\"org_id\": 111}", "::add-mask::111"), + ("source-default", "{\"org\": 111}", ""), + ) +) +def test_validate_mask_values(connector_name, dict_json_value, expected_secret, capsys): + loader = SecretsLoader(connector_name=connector_name, gsm_credentials={}) + json_value = json.loads(dict_json_value) + loader.mask_secrets_from_action_log(None, json_value) + assert expected_secret in capsys.readouterr().out

    diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx index 1c77e9a7281a..3f8d715c5906 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx @@ -28,7 +28,7 @@ export const FieldRow: React.FC = ({ transform }) => { [styles.mod]: diffType === "update", }); - const contentStyle = classnames(styles.content, { + const contentStyle = classnames(styles.content, styles.cell, { [styles.add]: diffType === "add", [styles.remove]: diffType === "remove", [styles.update]: diffType === "update", @@ -50,16 +50,16 @@ export const FieldRow: React.FC = ({ transform }) => { )} - +
    {fieldName} -
    		+ +
    {oldType && newType && ( {oldType} {newType} )} -
    {namespace}{itemName}{itemName} {diffVerb === "removed" && syncMode && }