diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/SeedType.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/SeedType.java index 373036962109..47c4c419bcf1 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/SeedType.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/SeedType.java @@ -7,9 +7,7 @@ public enum SeedType { STANDARD_SOURCE_DEFINITION("/seed/source_definitions.yaml", "sourceDefinitionId"), - STANDARD_DESTINATION_DEFINITION("/seed/destination_definitions.yaml", "destinationDefinitionId"), - SOURCE_SPEC("/seed/source_specs.yaml", "dockerImage"), - DESTINATION_SPEC("/seed/destination_specs.yaml", "dockerImage"); + STANDARD_DESTINATION_DEFINITION("/seed/destination_definitions.yaml", "destinationDefinitionId"); final String resourcePath; // ID field name diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml deleted file mode 100644 index a248b4eff240..000000000000 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ /dev/null @@ -1,2752 +0,0 @@ -# This file is generated by io.airbyte.config.specs.SeedConnectorSpecGenerator. -# Do NOT edit this file directly. See generator class for more details. ---- -- dockerImage: "airbyte/destination-azure-blob-storage:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/azureblobstorage" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "AzureBlobStorage Destination Spec" - type: "object" - required: - - "azure_blob_storage_account_name" - - "azure_blob_storage_account_key" - - "format" - additionalProperties: false - properties: - azure_blob_storage_endpoint_domain_name: - title: "Endpoint Domain Name" - type: "string" - default: "blob.core.windows.net" - description: "This is Azure Blob Storage endpoint domain name. Leave default\ - \ value (or leave it empty if run container from command line) to use\ - \ Microsoft native from example." - examples: - - "blob.core.windows.net" - azure_blob_storage_container_name: - title: "Azure blob storage container (Bucket) Name" - type: "string" - description: "The name of the Azure blob storage container. If not exists\ - \ - will be created automatically. May be empty, then will be created\ - \ automatically airbytecontainer+timestamp" - examples: - - "airbytetescontainername" - azure_blob_storage_account_name: - title: "Azure Blob Storage account name" - type: "string" - description: "The account's name of the Azure Blob Storage." - examples: - - "airbyte5storage" - azure_blob_storage_account_key: - description: "The Azure blob storage account key." - airbyte_secret: true - type: "string" - examples: - - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd==" - format: - title: "Output Format" - type: "object" - description: "Output data format" - oneOf: - - title: "CSV: Comma-Separated Values" - required: - - "format_type" - - "flattening" - properties: - format_type: - type: "string" - const: "CSV" - flattening: - type: "string" - title: "Normalization (Flattening)" - description: "Whether the input json data should be normalized (flattened)\ - \ in the output CSV. Please refer to docs for details." - default: "No flattening" - enum: - - "No flattening" - - "Root level flattening" - - title: "JSON Lines: newline-delimited JSON" - required: - - "format_type" - properties: - format_type: - type: "string" - const: "JSONL" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-bigquery:0.5.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "BigQuery Destination Spec" - type: "object" - required: - - "project_id" - - "dataset_id" - additionalProperties: true - properties: - big_query_client_buffer_size_mb: - title: "Google BigQuery client chunk size" - description: "Google BigQuery client's chunk(buffer) size (MIN=1, MAX =\ - \ 15) for each table. The default 15MiB value is used if not set explicitly.\ - \ It's recommended to decrease value for big data sets migration for less\ - \ HEAP memory consumption and avoiding crashes. For more details refer\ - \ to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html" - type: "integer" - minimum: 1 - maximum: 15 - default: 15 - examples: - - "15" - project_id: - type: "string" - description: "The GCP project ID for the project containing the target BigQuery\ - \ dataset." - title: "Project ID" - dataset_id: - type: "string" - description: "Default BigQuery Dataset ID tables are replicated to if the\ - \ source does not specify a namespace." - title: "Default Dataset ID" - dataset_location: - type: "string" - description: "The location of the dataset. Warning: Changes made after creation\ - \ will not be applied." - title: "Dataset Location" - default: "US" - enum: - - "US" - - "EU" - - "asia-east1" - - "asia-east2" - - "asia-northeast1" - - "asia-northeast2" - - "asia-northeast3" - - "asia-south1" - - "asia-southeast1" - - "asia-southeast2" - - "australia-southeast1" - - "europe-central1" - - "europe-central2" - - "europe-north1" - - "europe-west1" - - "europe-west2" - - "europe-west3" - - "europe-west4" - - "europe-west5" - - "europe-west6" - - "northamerica-northeast1" - - "southamerica-east1" - - "us-central1" - - "us-east1" - - "us-east4" - - "us-west-1" - - "us-west-2" - - "us-west-3" - - "us-west-4" - credentials_json: - type: "string" - description: "The contents of the JSON service account key. Check out the\ - \ docs if you need help generating this key. Default credentials will\ - \ be used if this field is left empty." - title: "Credentials JSON" - airbyte_secret: true - transformation_priority: - type: "string" - description: "When running custom transformations or Basic normalization,\ - \ running queries on interactive mode can hit BQ limits, choosing batch\ - \ will solve those limitss." - title: "Transformation Query Run Type" - default: "interactive" - enum: - - "interactive" - - "batch" - loading_method: - type: "object" - title: "Loading Method" - description: "Loading method used to send select the way data will be uploaded\ - \ to BigQuery." - oneOf: - - title: "Standard Inserts" - additionalProperties: false - description: "Direct uploading using streams." - required: - - "method" - properties: - method: - type: "string" - const: "Standard" - - title: "GCS Staging" - additionalProperties: false - description: "Writes large batches of records to a file, uploads the file\ - \ to GCS, then uses 
COPY INTO table
to upload the file. Recommended\ - \ for large production workloads for better speed and scalability." - required: - - "method" - - "gcs_bucket_name" - - "gcs_bucket_path" - - "credential" - properties: - method: - type: "string" - const: "GCS Staging" - gcs_bucket_name: - title: "GCS Bucket Name" - type: "string" - description: "The name of the GCS bucket." - examples: - - "airbyte_sync" - gcs_bucket_path: - description: "Directory under the GCS bucket where data will be written." - type: "string" - examples: - - "data_sync/test" - keep_files_in_gcs-bucket: - type: "string" - description: "This upload method is supposed to temporary store records\ - \ in GCS bucket. What do you want to do with data in GCS bucket\ - \ when migration has finished?" - title: "GCS tmp files afterward processing" - default: "Delete all tmp files from GCS" - enum: - - "Delete all tmp files from GCS" - - "Keep all tmp files in GCS" - credential: - title: "Credential" - type: "object" - oneOf: - - title: "HMAC key" - required: - - "credential_type" - - "hmac_key_access_id" - - "hmac_key_secret" - properties: - credential_type: - type: "string" - const: "HMAC_KEY" - hmac_key_access_id: - type: "string" - description: "HMAC key access ID. When linked to a service account,\ - \ this ID is 61 characters long; when linked to a user account,\ - \ it is 24 characters long." - title: "HMAC Key Access ID" - airbyte_secret: true - examples: - - "1234567890abcdefghij1234" - hmac_key_secret: - type: "string" - description: "The corresponding secret for the access ID. It\ - \ is a 40-character base-64 encoded string." - title: "HMAC Key Secret" - airbyte_secret: true - examples: - - "1234567890abcdefghij1234567890ABCDEFGHIJ" - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:0.1.7" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "BigQuery Denormalized Typed Struct Destination Spec" - type: "object" - required: - - "project_id" - - "dataset_id" - additionalProperties: true - properties: - project_id: - type: "string" - description: "The GCP project ID for the project containing the target BigQuery\ - \ dataset." - title: "Project ID" - dataset_id: - type: "string" - description: "Default BigQuery Dataset ID tables are replicated to if the\ - \ source does not specify a namespace." - title: "Default Dataset ID" - dataset_location: - type: "string" - description: "The location of the dataset. Warning: Changes made after creation\ - \ will not be applied." - title: "Dataset Location" - default: "US" - enum: - - "US" - - "EU" - - "asia-east1" - - "asia-east2" - - "asia-northeast1" - - "asia-northeast2" - - "asia-northeast3" - - "asia-south1" - - "asia-southeast1" - - "asia-southeast2" - - "australia-southeast1" - - "europe-central1" - - "europe-central2" - - "europe-north1" - - "europe-west1" - - "europe-west2" - - "europe-west3" - - "europe-west4" - - "europe-west5" - - "europe-west6" - - "northamerica-northeast1" - - "southamerica-east1" - - "us-central1" - - "us-east1" - - "us-east4" - - "us-west-1" - - "us-west-2" - - "us-west-3" - - "us-west-4" - credentials_json: - type: "string" - description: "The contents of the JSON service account key. Check out the\ - \ docs if you need help generating this key. Default credentials will\ - \ be used if this field is left empty." - title: "Credentials JSON" - airbyte_secret: true - supportsIncremental: true - supportsNormalization: false - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-keen:0.2.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/keen" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Keen Spec" - type: "object" - required: - - "project_id" - - "api_key" - additionalProperties: false - properties: - project_id: - description: "Keen Project ID" - type: "string" - examples: - - "58b4acc22ba938934e888322e" - api_key: - title: "API Key" - description: "Keen Master API key" - type: "string" - examples: - - "ABCDEFGHIJKLMNOPRSTUWXYZ" - airbyte_secret: true - infer_timestamp: - title: "Infer Timestamp" - description: "Allow connector to guess keen.timestamp value based on the\ - \ streamed data" - type: "boolean" - default: true - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-dynamodb:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/dynamodb" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "DynamoDB Destination Spec" - type: "object" - required: - - "dynamodb_table_name" - - "dynamodb_region" - - "access_key_id" - - "secret_access_key" - additionalProperties: false - properties: - dynamodb_endpoint: - title: "Endpoint" - type: "string" - default: "" - description: "This is your DynamoDB endpoint url.(if you are working with\ - \ AWS DynamoDB, just leave empty)." - examples: - - "http://localhost:9000" - dynamodb_table_name: - title: "DynamoDB Table Name" - type: "string" - description: "The name of the DynamoDB table." - examples: - - "airbyte_sync" - dynamodb_region: - title: "DynamoDB Region" - type: "string" - default: "" - description: "The region of the DynamoDB." - enum: - - "" - - "us-east-1" - - "us-east-2" - - "us-west-1" - - "us-west-2" - - "af-south-1" - - "ap-east-1" - - "ap-south-1" - - "ap-northeast-1" - - "ap-northeast-2" - - "ap-northeast-3" - - "ap-southeast-1" - - "ap-southeast-2" - - "ca-central-1" - - "cn-north-1" - - "cn-northwest-1" - - "eu-central-1" - - "eu-north-1" - - "eu-south-1" - - "eu-west-1" - - "eu-west-2" - - "eu-west-3" - - "sa-east-1" - - "me-south-1" - - "us-gov-east-1" - - "us-gov-west-1" - access_key_id: - type: "string" - description: "The access key id to access the DynamoDB. Airbyte requires\ - \ Read and Write permissions to the DynamoDB." - title: "DynamoDB Key Id" - airbyte_secret: true - examples: - - "A012345678910EXAMPLE" - secret_access_key: - type: "string" - description: "The corresponding secret to the access key id." - title: "DynamoDB Access Key" - airbyte_secret: true - examples: - - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-gcs:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "GCS Destination Spec" - type: "object" - required: - - "gcs_bucket_name" - - "gcs_bucket_path" - - "gcs_bucket_region" - - "credential" - - "format" - additionalProperties: false - properties: - gcs_bucket_name: - title: "GCS Bucket Name" - type: "string" - description: "The name of the GCS bucket." - examples: - - "airbyte_sync" - gcs_bucket_path: - description: "Directory under the GCS bucket where data will be written." - type: "string" - examples: - - "data_sync/test" - gcs_bucket_region: - title: "GCS Bucket Region" - type: "string" - default: "" - description: "The region of the GCS bucket." - enum: - - "" - - "-- North America --" - - "northamerica-northeast1" - - "us-central1" - - "us-east1" - - "us-east4" - - "us-west1" - - "us-west2" - - "us-west3" - - "us-west4" - - "-- South America --" - - "southamerica-east1" - - "-- Europe --" - - "europe-central2" - - "europe-north1" - - "europe-west1" - - "europe-west2" - - "europe-west3" - - "europe-west4" - - "europe-west6" - - "-- Asia --" - - "asia-east1" - - "asia-east2" - - "asia-northeast1" - - "asia-northeast2" - - "asia-northeast3" - - "asia-south1" - - "asia-south2" - - "asia-southeast1" - - "asia-southeast2" - - "-- Australia --" - - "australia-southeast1" - - "australia-southeast2" - - "-- Multi-regions --" - - "asia" - - "eu" - - "us" - - "-- Dual-regions --" - - "asia1" - - "eur4" - - "nam4" - credential: - title: "Credential" - type: "object" - oneOf: - - title: "HMAC key" - required: - - "credential_type" - - "hmac_key_access_id" - - "hmac_key_secret" - properties: - credential_type: - type: "string" - enum: - - "HMAC_KEY" - default: "HMAC_KEY" - hmac_key_access_id: - type: "string" - description: "HMAC key access ID. When linked to a service account,\ - \ this ID is 61 characters long; when linked to a user account,\ - \ it is 24 characters long." - title: "HMAC Key Access ID" - airbyte_secret: true - examples: - - "1234567890abcdefghij1234" - hmac_key_secret: - type: "string" - description: "The corresponding secret for the access ID. It is a\ - \ 40-character base-64 encoded string." - title: "HMAC Key Secret" - airbyte_secret: true - examples: - - "1234567890abcdefghij1234567890ABCDEFGHIJ" - format: - title: "Output Format" - type: "object" - description: "Output data format" - oneOf: - - title: "Avro: Apache Avro" - required: - - "format_type" - - "compression_codec" - properties: - format_type: - type: "string" - enum: - - "Avro" - default: "Avro" - compression_codec: - title: "Compression Codec" - description: "The compression algorithm used to compress data. Default\ - \ to no compression." - type: "object" - oneOf: - - title: "no compression" - required: - - "codec" - properties: - codec: - type: "string" - enum: - - "no compression" - default: "no compression" - - title: "Deflate" - required: - - "codec" - - "compression_level" - properties: - codec: - type: "string" - enum: - - "Deflate" - default: "Deflate" - compression_level: - title: "Deflate level" - description: "0: no compression & fastest, 9: best compression\ - \ & slowest." - type: "integer" - default: 0 - minimum: 0 - maximum: 9 - - title: "bzip2" - required: - - "codec" - properties: - codec: - type: "string" - enum: - - "bzip2" - default: "bzip2" - - title: "xz" - required: - - "codec" - - "compression_level" - properties: - codec: - type: "string" - enum: - - "xz" - default: "xz" - compression_level: - title: "Compression level" - description: "See here for details." - type: "integer" - default: 6 - minimum: 0 - maximum: 9 - - title: "zstandard" - required: - - "codec" - - "compression_level" - properties: - codec: - type: "string" - enum: - - "zstandard" - default: "zstandard" - compression_level: - title: "Compression level" - description: "Negative levels are 'fast' modes akin to lz4 or\ - \ snappy, levels above 9 are generally for archival purposes,\ - \ and levels above 18 use a lot of memory." - type: "integer" - default: 3 - minimum: -5 - maximum: 22 - include_checksum: - title: "Include checksum" - description: "If true, include a checksum with each data block." - type: "boolean" - default: false - - title: "snappy" - required: - - "codec" - properties: - codec: - type: "string" - enum: - - "snappy" - default: "snappy" - part_size_mb: - title: "Block Size (MB) for GCS multipart upload" - description: "This is the size of a \"Part\" being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will allow\ - \ to upload a bigger files and improve the speed, but consumes9\ - \ more memory. Allowed values: min=5MB, max=525MB Default: 5MB." - type: "integer" - default: 5 - examples: - - 5 - - title: "CSV: Comma-Separated Values" - required: - - "format_type" - - "flattening" - properties: - format_type: - type: "string" - enum: - - "CSV" - default: "CSV" - flattening: - type: "string" - title: "Normalization (Flattening)" - description: "Whether the input json data should be normalized (flattened)\ - \ in the output CSV. Please refer to docs for details." - default: "No flattening" - enum: - - "No flattening" - - "Root level flattening" - part_size_mb: - title: "Block Size (MB) for GCS multipart upload" - description: "This is the size of a \"Part\" being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will allow\ - \ to upload a bigger files and improve the speed, but consumes9\ - \ more memory. Allowed values: min=5MB, max=525MB Default: 5MB." - type: "integer" - default: 5 - examples: - - 5 - - title: "JSON Lines: newline-delimited JSON" - required: - - "format_type" - properties: - format_type: - type: "string" - enum: - - "JSONL" - default: "JSONL" - part_size_mb: - title: "Block Size (MB) for GCS multipart upload" - description: "This is the size of a \"Part\" being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will allow\ - \ to upload a bigger files and improve the speed, but consumes9\ - \ more memory. Allowed values: min=5MB, max=525MB Default: 5MB." - type: "integer" - default: 5 - examples: - - 5 - - title: "Parquet: Columnar Storage" - required: - - "format_type" - properties: - format_type: - type: "string" - enum: - - "Parquet" - default: "Parquet" - compression_codec: - title: "Compression Codec" - description: "The compression algorithm used to compress data pages." - type: "string" - enum: - - "UNCOMPRESSED" - - "SNAPPY" - - "GZIP" - - "LZO" - - "BROTLI" - - "LZ4" - - "ZSTD" - default: "UNCOMPRESSED" - block_size_mb: - title: "Block Size (Row Group Size) (MB)" - description: "This is the size of a row group being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will improve\ - \ the IO when reading, but consume more memory when writing. Default:\ - \ 128 MB." - type: "integer" - default: 128 - examples: - - 128 - max_padding_size_mb: - title: "Max Padding Size (MB)" - description: "Maximum size allowed as padding to align row groups.\ - \ This is also the minimum size of a row group. Default: 8 MB." - type: "integer" - default: 8 - examples: - - 8 - page_size_kb: - title: "Page Size (KB)" - description: "The page size is for compression. A block is composed\ - \ of pages. A page is the smallest unit that must be read fully\ - \ to access a single record. If this value is too small, the compression\ - \ will deteriorate. Default: 1024 KB." - type: "integer" - default: 1024 - examples: - - 1024 - dictionary_page_size_kb: - title: "Dictionary Page Size (KB)" - description: "There is one dictionary page per column per row group\ - \ when dictionary encoding is used. The dictionary page size works\ - \ like the page size but for dictionary. Default: 1024 KB." - type: "integer" - default: 1024 - examples: - - 1024 - dictionary_encoding: - title: "Dictionary Encoding" - description: "Default: true." - type: "boolean" - default: true - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" - $schema: "http://json-schema.org/draft-07/schema#" -- dockerImage: "airbyte/destination-pubsub:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/pubsub" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Google PubSub Destination Spec" - type: "object" - required: - - "project_id" - - "topic_id" - - "credentials_json" - additionalProperties: true - properties: - project_id: - type: "string" - description: "The GCP project ID for the project containing the target PubSub" - title: "Project ID" - topic_id: - type: "string" - description: "PubSub topic ID in the given GCP project ID" - title: "PubSub Topic ID" - credentials_json: - type: "string" - description: "The contents of the JSON service account key. Check out the\ - \ docs if you need help generating this key." - title: "Credentials JSON" - airbyte_secret: true - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" -- dockerImage: "airbyte/destination-kafka:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/kafka" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Kafka Destination Spec" - type: "object" - required: - - "bootstrap_servers" - - "topic_pattern" - - "protocol" - - "acks" - - "enable_idempotence" - - "compression_type" - - "batch_size" - - "linger_ms" - - "max_in_flight_requests_per_connection" - - "client_dns_lookup" - - "buffer_memory" - - "max_request_size" - - "retries" - - "socket_connection_setup_timeout_ms" - - "socket_connection_setup_timeout_max_ms" - - "max_block_ms" - - "request_timeout_ms" - - "delivery_timeout_ms" - - "send_buffer_bytes" - - "receive_buffer_bytes" - additionalProperties: true - properties: - bootstrap_servers: - title: "Bootstrap servers" - description: "A list of host/port pairs to use for establishing the initial\ - \ connection to the Kafka cluster. The client will make use of all servers\ - \ irrespective of which servers are specified here for bootstrapping—this\ - \ list only impacts the initial hosts used to discover the full set of\ - \ servers. This list should be in the form host1:port1,host2:port2,....\ - \ Since these servers are just used for the initial connection to discover\ - \ the full cluster membership (which may change dynamically), this list\ - \ need not contain the full set of servers (you may want more than one,\ - \ though, in case a server is down)." - type: "string" - examples: - - "kafka-broker1:9092,kafka-broker2:9092" - topic_pattern: - title: "Topic pattern" - description: "Topic pattern in which the records will be sent. You can use\ - \ patterns like '{namespace}' and/or '{stream}' to send the message to\ - \ a specific topic based on these values. Notice that the topic name will\ - \ be transformed to a standard naming convention." - type: "string" - examples: - - "sample.topic" - - "{namespace}.{stream}.sample" - test_topic: - title: "Test topic" - description: "Topic to test if Airbyte can produce messages." - type: "string" - examples: - - "test.topic" - sync_producer: - title: "Sync producer" - description: "Wait synchronously until the record has been sent to Kafka." - type: "boolean" - default: false - protocol: - title: "Protocol" - type: "object" - description: "Protocol used to communicate with brokers." - oneOf: - - title: "PLAINTEXT" - required: - - "security_protocol" - properties: - security_protocol: - type: "string" - enum: - - "PLAINTEXT" - default: "PLAINTEXT" - - title: "SASL PLAINTEXT" - required: - - "security_protocol" - - "sasl_mechanism" - - "sasl_jaas_config" - properties: - security_protocol: - type: "string" - enum: - - "SASL_PLAINTEXT" - default: "SASL_PLAINTEXT" - sasl_mechanism: - title: "SASL mechanism" - description: "SASL mechanism used for client connections. This may\ - \ be any mechanism for which a security provider is available." - type: "string" - default: "PLAIN" - enum: - - "PLAIN" - sasl_jaas_config: - title: "SASL JAAS config" - description: "JAAS login context parameters for SASL connections in\ - \ the format used by JAAS configuration files." - type: "string" - default: "" - airbyte_secret: true - - title: "SASL SSL" - required: - - "security_protocol" - - "sasl_mechanism" - - "sasl_jaas_config" - properties: - security_protocol: - type: "string" - enum: - - "SASL_SSL" - default: "SASL_SSL" - sasl_mechanism: - title: "SASL mechanism" - description: "SASL mechanism used for client connections. This may\ - \ be any mechanism for which a security provider is available." - type: "string" - default: "GSSAPI" - enum: - - "GSSAPI" - - "OAUTHBEARER" - - "SCRAM-SHA-256" - sasl_jaas_config: - title: "SASL JAAS config" - description: "JAAS login context parameters for SASL connections in\ - \ the format used by JAAS configuration files." - type: "string" - default: "" - airbyte_secret: true - client_id: - title: "Client ID" - description: "An id string to pass to the server when making requests. The\ - \ purpose of this is to be able to track the source of requests beyond\ - \ just ip/port by allowing a logical application name to be included in\ - \ server-side request logging." - type: "string" - examples: - - "airbyte-producer" - acks: - title: "ACKs" - description: "The number of acknowledgments the producer requires the leader\ - \ to have received before considering a request complete. This controls\ - \ the durability of records that are sent." - type: "string" - default: "1" - enum: - - "0" - - "1" - - "all" - enable_idempotence: - title: "Enable idempotence" - description: "When set to 'true', the producer will ensure that exactly\ - \ one copy of each message is written in the stream. If 'false', producer\ - \ retries due to broker failures, etc., may write duplicates of the retried\ - \ message in the stream." - type: "boolean" - default: false - compression_type: - title: "Compression type" - description: "The compression type for all data generated by the producer." - type: "string" - default: "none" - enum: - - "none" - - "gzip" - - "snappy" - - "lz4" - - "zstd" - batch_size: - title: "Batch size" - description: "The producer will attempt to batch records together into fewer\ - \ requests whenever multiple records are being sent to the same partition." - type: "integer" - examples: - - 16384 - linger_ms: - title: "Linger ms" - description: "The producer groups together any records that arrive in between\ - \ request transmissions into a single batched request." - type: "string" - examples: - - 0 - max_in_flight_requests_per_connection: - title: "Max in flight requests per connection" - description: "The maximum number of unacknowledged requests the client will\ - \ send on a single connection before blocking." - type: "integer" - examples: - - 5 - client_dns_lookup: - title: "Client DNS lookup" - description: "Controls how the client uses DNS lookups. If set to use_all_dns_ips,\ - \ connect to each returned IP address in sequence until a successful connection\ - \ is established. After a disconnection, the next IP is used. Once all\ - \ IPs have been used once, the client resolves the IP(s) from the hostname\ - \ again. If set to resolve_canonical_bootstrap_servers_only, resolve each\ - \ bootstrap address into a list of canonical names. After the bootstrap\ - \ phase, this behaves the same as use_all_dns_ips. If set to default (deprecated),\ - \ attempt to connect to the first IP address returned by the lookup, even\ - \ if the lookup returns multiple IP addresses." - type: "string" - default: "use_all_dns_ips" - enum: - - "default" - - "use_all_dns_ips" - - "resolve_canonical_bootstrap_servers_only" - - "use_all_dns_ips" - buffer_memory: - title: "Buffer memory" - description: "The total bytes of memory the producer can use to buffer records\ - \ waiting to be sent to the server." - type: "string" - examples: 33554432 - max_request_size: - title: "Max request size" - description: "The maximum size of a request in bytes." - type: "integer" - examples: - - 1048576 - retries: - title: "Retries" - description: "Setting a value greater than zero will cause the client to\ - \ resend any record whose send fails with a potentially transient error." - type: "integer" - examples: - - 2147483647 - socket_connection_setup_timeout_ms: - title: "Socket connection setup timeout" - description: "The amount of time the client will wait for the socket connection\ - \ to be established." - type: "string" - examples: - - 10000 - socket_connection_setup_timeout_max_ms: - title: "Socket connection setup max timeout" - description: "The maximum amount of time the client will wait for the socket\ - \ connection to be established. The connection setup timeout will increase\ - \ exponentially for each consecutive connection failure up to this maximum." - type: "string" - examples: - - 30000 - max_block_ms: - title: "Max block ms" - description: "The configuration controls how long the KafkaProducer's send(),\ - \ partitionsFor(), initTransactions(), sendOffsetsToTransaction(), commitTransaction()\ - \ and abortTransaction() methods will block." - type: "string" - examples: - - 60000 - request_timeout_ms: - title: "Request timeout" - description: "The configuration controls the maximum amount of time the\ - \ client will wait for the response of a request. If the response is not\ - \ received before the timeout elapses the client will resend the request\ - \ if necessary or fail the request if retries are exhausted." - type: "integer" - examples: - - 30000 - delivery_timeout_ms: - title: "Delivery timeout" - description: "An upper bound on the time to report success or failure after\ - \ a call to 'send()' returns." - type: "integer" - examples: - - 120000 - send_buffer_bytes: - title: "Send buffer bytes" - description: "The size of the TCP send buffer (SO_SNDBUF) to use when sending\ - \ data. If the value is -1, the OS default will be used." - type: "integer" - examples: - - 131072 - receive_buffer_bytes: - title: "Receive buffer bytes" - description: "The size of the TCP receive buffer (SO_RCVBUF) to use when\ - \ reading data. If the value is -1, the OS default will be used." - type: "integer" - examples: - - 32768 - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" -- dockerImage: "airbyte/destination-csv:0.2.8" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/local-csv" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "CSV Destination Spec" - type: "object" - required: - - "destination_path" - additionalProperties: false - properties: - destination_path: - description: "Path to the directory where csv files will be written. The\ - \ destination uses the local mount \"/local\" and any data files will\ - \ be placed inside that local mount. For more information check out our\ - \ docs" - type: "string" - examples: - - "/local" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-local-json:0.2.8" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/local-json" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Local Json Destination Spec" - type: "object" - required: - - "destination_path" - additionalProperties: false - properties: - destination_path: - description: "Path to the directory where json files will be written. The\ - \ files will be placed inside that local mount. For more information check\ - \ out our docs" - type: "string" - examples: - - "/json_data" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-mssql:0.1.10" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MS SQL Server Destination Spec" - type: "object" - required: - - "host" - - "port" - - "username" - - "database" - - "schema" - additionalProperties: true - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - order: 0 - port: - title: "Port" - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 1433 - examples: - - "1433" - order: 1 - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - schema: - title: "Default Schema" - description: "The default schema tables are written to if the source does\ - \ not specify a namespace. The usual value for this field is \"public\"\ - ." - type: "string" - examples: - - "public" - default: "public" - order: 3 - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 4 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 5 - ssl_method: - title: "SSL Method" - type: "object" - description: "Encryption method to use when communicating with the database" - order: 6 - oneOf: - - title: "Unencrypted" - additionalProperties: false - description: "Data transfer will not be encrypted." - required: - - "ssl_method" - type: "object" - properties: - ssl_method: - type: "string" - enum: - - "unencrypted" - default: "unencrypted" - - title: "Encrypted (trust server certificate)" - additionalProperties: false - description: "Use the cert provided by the server without verification.\ - \ (For testing purposes only!)" - required: - - "ssl_method" - type: "object" - properties: - ssl_method: - type: "string" - enum: - - "encrypted_trust_server_certificate" - default: "encrypted_trust_server_certificate" - - title: "Encrypted (verify certificate)" - additionalProperties: false - description: "Verify and use the cert provided by the server." - required: - - "ssl_method" - - "trustStoreName" - - "trustStorePassword" - type: "object" - properties: - ssl_method: - type: "string" - enum: - - "encrypted_verify_certificate" - default: "encrypted_verify_certificate" - hostNameInCertificate: - title: "Host Name In Certificate" - type: "string" - description: "Specifies the host name of the server. The value of\ - \ this property must match the subject property of the certificate." - order: 7 - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials in RSA PEM\ - \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/destination-meilisearch:0.2.10" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/meilisearch" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MeiliSearch Destination Spec" - type: "object" - required: - - "host" - additionalProperties: true - properties: - host: - title: "Host" - description: "Hostname of the MeiliSearch instance" - type: "string" - order: 0 - api_key: - title: "API Key" - airbyte_secret: true - description: "MeiliSearch instance API Key" - type: "string" - order: 1 - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-mongodb:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/mongodb" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MongoDB Destination Spec" - type: "object" - required: - - "database" - - "auth_type" - additionalProperties: true - properties: - instance_type: - description: "MongoDb instance to connect to. For MongoDB Atlas and Replica\ - \ Set TLS connection is used by default." - title: "MongoDb instance type" - type: "object" - order: 0 - oneOf: - - title: "Standalone MongoDb Instance" - required: - - "instance" - - "host" - - "port" - properties: - instance: - type: "string" - enum: - - "standalone" - default: "standalone" - host: - title: "Host" - type: "string" - description: "Host of a Mongo database to be replicated." - order: 0 - port: - title: "Port" - type: "integer" - description: "Port of a Mongo database to be replicated." - minimum: 0 - maximum: 65536 - default: 27017 - examples: - - "27017" - order: 1 - tls: - title: "TLS connection" - type: "boolean" - description: "Indicates whether TLS encryption protocol will be used\ - \ to connect to MongoDB. It is recommended to use TLS connection\ - \ if possible. For more information see documentation." - default: false - order: 2 - - title: "Replica Set" - required: - - "instance" - - "server_addresses" - properties: - instance: - type: "string" - enum: - - "replica" - default: "replica" - server_addresses: - title: "Server addresses" - type: "string" - description: "The members of a replica set. Please specify `host`:`port`\ - \ of each member seperated by comma." - examples: - - "host1:27017,host2:27017,host3:27017" - order: 0 - replica_set: - title: "Replica Set" - type: "string" - description: "A replica set name." - order: 1 - - title: "MongoDB Atlas" - additionalProperties: false - required: - - "instance" - - "cluster_url" - properties: - instance: - type: "string" - enum: - - "atlas" - default: "atlas" - cluster_url: - title: "Cluster URL" - type: "string" - description: "URL of a cluster to connect to." - order: 0 - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - auth_type: - title: "Authorization type" - type: "object" - description: "Authorization type." - oneOf: - - title: "None" - additionalProperties: false - description: "None." - required: - - "authorization" - type: "object" - properties: - authorization: - type: "string" - const: "none" - - title: "Login/Password" - additionalProperties: false - description: "Login/Password." - required: - - "authorization" - - "username" - - "password" - type: "object" - properties: - authorization: - type: "string" - const: "login/password" - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 1 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 2 - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-mysql:0.1.13" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MySQL Destination Spec" - type: "object" - required: - - "host" - - "port" - - "username" - - "database" - additionalProperties: true - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - order: 0 - port: - title: "Port" - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 3306 - examples: - - "3306" - order: 1 - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 3 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 4 - ssl: - title: "SSL Connection" - description: "Encrypt data using SSL." - type: "boolean" - default: true - order: 5 - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials for logging\ - \ into the jump server host." - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-oracle:0.1.11" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/oracle" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Oracle Destination Spec" - type: "object" - required: - - "host" - - "port" - - "username" - - "sid" - additionalProperties: true - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - order: 0 - port: - title: "Port" - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 1521 - examples: - - "1521" - order: 1 - sid: - title: "SID" - description: "SID" - type: "string" - order: 2 - username: - title: "User" - description: "Username to use to access the database. This user must have\ - \ CREATE USER privileges in the database." - type: "string" - order: 3 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 4 - schema: - title: "Default Schema" - description: "The default schema tables are written to if the source does\ - \ not specify a namespace. The usual value for this field is \"airbyte\"\ - . In Oracle, schemas and users are the same thing, so the \"user\" parameter\ - \ is used as the login credentials and this is used for the default Airbyte\ - \ message schema." - type: "string" - examples: - - "airbyte" - default: "airbyte" - order: 5 - encryption: - title: "Encryption" - type: "object" - description: "Encryption method to use when communicating with the database" - order: 6 - oneOf: - - title: "Unencrypted" - additionalProperties: false - description: "Data transfer will not be encrypted." - required: - - "encryption_method" - properties: - encryption_method: - type: "string" - const: "unencrypted" - enum: - - "unencrypted" - default: "unencrypted" - - title: "Native Network Ecryption (NNE)" - additionalProperties: false - description: "Native network encryption gives you the ability to encrypt\ - \ database connections, without the configuration overhead of TCP/IP\ - \ and SSL/TLS and without the need to open and listen on different ports." - required: - - "encryption_method" - properties: - encryption_method: - type: "string" - const: "client_nne" - enum: - - "client_nne" - default: "client_nne" - encryption_algorithm: - type: "string" - description: "This parameter defines the encryption algorithm to be\ - \ used" - title: "Encryption Algorithm" - default: "AES256" - enum: - - "AES256" - - "RC4_56" - - "3DES168" - - title: "TLS Encrypted (verify certificate)" - additionalProperties: false - description: "Verify and use the cert provided by the server." - required: - - "encryption_method" - - "ssl_certificate" - properties: - encryption_method: - type: "string" - const: "encrypted_verify_certificate" - enum: - - "encrypted_verify_certificate" - default: "encrypted_verify_certificate" - ssl_certificate: - title: "SSL PEM file" - description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\ - \ containers frequently used in certificate installations" - type: "string" - airbyte_secret: true - multiline: true - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials in RSA PEM\ - \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsIncremental: true - supportsNormalization: false - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-postgres:0.3.11" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Postgres Destination Spec" - type: "object" - required: - - "host" - - "port" - - "username" - - "database" - - "schema" - additionalProperties: true - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - order: 0 - port: - title: "Port" - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 5432 - examples: - - "5432" - order: 1 - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - schema: - title: "Default Schema" - description: "The default schema tables are written to if the source does\ - \ not specify a namespace. The usual value for this field is \"public\"\ - ." - type: "string" - examples: - - "public" - default: "public" - order: 3 - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 4 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 5 - ssl: - title: "SSL Connection" - description: "Encrypt data using SSL." - type: "boolean" - default: false - order: 6 - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials for logging\ - \ into the jump server host." - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/destination-redshift:0.3.19" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Redshift Destination Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - - "password" - - "schema" - additionalProperties: true - properties: - host: - description: "Host Endpoint of the Redshift Cluster (must include the cluster-id,\ - \ region and end with .redshift.amazonaws.com)" - type: "string" - title: "Host" - port: - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 5439 - examples: - - "5439" - title: "Port" - username: - description: "Username to use to access the database." - type: "string" - title: "Username" - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - title: "Password" - database: - description: "Name of the database." - type: "string" - title: "Database" - schema: - description: "The default schema tables are written to if the source does\ - \ not specify a namespace. Unless specifically configured, the usual value\ - \ for this field is \"public\"." - type: "string" - examples: - - "public" - default: "public" - title: "Default Schema" - s3_bucket_name: - title: "S3 Bucket Name" - type: "string" - description: "The name of the staging S3 bucket to use if utilising a COPY\ - \ strategy. COPY is recommended for production workloads for better speed\ - \ and scalability. See AWS docs for more details." - examples: - - "airbyte.staging" - s3_bucket_region: - title: "S3 Bucket Region" - type: "string" - default: "" - description: "The region of the S3 staging bucket to use if utilising a\ - \ copy strategy." - enum: - - "" - - "us-east-1" - - "us-east-2" - - "us-west-1" - - "us-west-2" - - "af-south-1" - - "ap-east-1" - - "ap-south-1" - - "ap-northeast-1" - - "ap-northeast-2" - - "ap-northeast-3" - - "ap-southeast-1" - - "ap-southeast-2" - - "ca-central-1" - - "cn-north-1" - - "cn-northwest-1" - - "eu-central-1" - - "eu-north-1" - - "eu-south-1" - - "eu-west-1" - - "eu-west-2" - - "eu-west-3" - - "sa-east-1" - - "me-south-1" - access_key_id: - type: "string" - description: "The Access Key Id granting allow one to access the above S3\ - \ staging bucket. Airbyte requires Read and Write permissions to the given\ - \ bucket." - title: "S3 Key Id" - airbyte_secret: true - secret_access_key: - type: "string" - description: "The corresponding secret to the above access key id." - title: "S3 Access Key" - airbyte_secret: true - part_size: - type: "integer" - minimum: 10 - maximum: 100 - examples: - - "10" - description: "Optional. Increase this if syncing tables larger than 100GB.\ - \ Only relevant for COPY. Files are streamed to S3 in parts. This determines\ - \ the size of each part, in MBs. As S3 has a limit of 10,000 parts per\ - \ file, part size affects the table size. This is 10MB by default, resulting\ - \ in a default limit of 100GB tables. Note, a larger part size will result\ - \ in larger memory requirements. A rule of thumb is to multiply the part\ - \ size by 10 to get the memory requirement. Modify this with care." - title: "Stream Part Size" - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/destination-s3:0.1.12" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "S3 Destination Spec" - type: "object" - required: - - "s3_bucket_name" - - "s3_bucket_path" - - "s3_bucket_region" - - "access_key_id" - - "secret_access_key" - - "format" - additionalProperties: false - properties: - s3_endpoint: - title: "Endpoint" - type: "string" - default: "" - description: "This is your S3 endpoint url.(if you are working with AWS\ - \ S3, just leave empty)." - examples: - - "http://localhost:9000" - s3_bucket_name: - title: "S3 Bucket Name" - type: "string" - description: "The name of the S3 bucket." - examples: - - "airbyte_sync" - s3_bucket_path: - description: "Directory under the S3 bucket where data will be written." - type: "string" - examples: - - "data_sync/test" - s3_bucket_region: - title: "S3 Bucket Region" - type: "string" - default: "" - description: "The region of the S3 bucket." - enum: - - "" - - "us-east-1" - - "us-east-2" - - "us-west-1" - - "us-west-2" - - "af-south-1" - - "ap-east-1" - - "ap-south-1" - - "ap-northeast-1" - - "ap-northeast-2" - - "ap-northeast-3" - - "ap-southeast-1" - - "ap-southeast-2" - - "ca-central-1" - - "cn-north-1" - - "cn-northwest-1" - - "eu-central-1" - - "eu-north-1" - - "eu-south-1" - - "eu-west-1" - - "eu-west-2" - - "eu-west-3" - - "sa-east-1" - - "me-south-1" - - "us-gov-east-1" - - "us-gov-west-1" - access_key_id: - type: "string" - description: "The access key id to access the S3 bucket. Airbyte requires\ - \ Read and Write permissions to the given bucket." - title: "S3 Key Id" - airbyte_secret: true - examples: - - "A012345678910EXAMPLE" - secret_access_key: - type: "string" - description: "The corresponding secret to the access key id." - title: "S3 Access Key" - airbyte_secret: true - examples: - - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" - format: - title: "Output Format" - type: "object" - description: "Output data format" - oneOf: - - title: "Avro: Apache Avro" - required: - - "format_type" - - "compression_codec" - properties: - format_type: - type: "string" - enum: - - "Avro" - default: "Avro" - compression_codec: - title: "Compression Codec" - description: "The compression algorithm used to compress data. Default\ - \ to no compression." - type: "object" - oneOf: - - title: "no compression" - required: - - "codec" - properties: - codec: - type: "string" - enum: - - "no compression" - default: "no compression" - - title: "Deflate" - required: - - "codec" - - "compression_level" - properties: - codec: - type: "string" - enum: - - "Deflate" - default: "Deflate" - compression_level: - title: "Deflate level" - description: "0: no compression & fastest, 9: best compression\ - \ & slowest." - type: "integer" - default: 0 - minimum: 0 - maximum: 9 - - title: "bzip2" - required: - - "codec" - properties: - codec: - type: "string" - enum: - - "bzip2" - default: "bzip2" - - title: "xz" - required: - - "codec" - - "compression_level" - properties: - codec: - type: "string" - enum: - - "xz" - default: "xz" - compression_level: - title: "Compression level" - description: "See here for details." - type: "integer" - default: 6 - minimum: 0 - maximum: 9 - - title: "zstandard" - required: - - "codec" - - "compression_level" - properties: - codec: - type: "string" - enum: - - "zstandard" - default: "zstandard" - compression_level: - title: "Compression level" - description: "Negative levels are 'fast' modes akin to lz4 or\ - \ snappy, levels above 9 are generally for archival purposes,\ - \ and levels above 18 use a lot of memory." - type: "integer" - default: 3 - minimum: -5 - maximum: 22 - include_checksum: - title: "Include checksum" - description: "If true, include a checksum with each data block." - type: "boolean" - default: false - - title: "snappy" - required: - - "codec" - properties: - codec: - type: "string" - enum: - - "snappy" - default: "snappy" - part_size_mb: - title: "Block Size (MB) for Amazon S3 multipart upload" - description: "This is the size of a \"Part\" being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will allow\ - \ to upload a bigger files and improve the speed, but consumes9\ - \ more memory. Allowed values: min=5MB, max=525MB Default: 5MB." - type: "integer" - default: 5 - examples: - - 5 - - title: "CSV: Comma-Separated Values" - required: - - "format_type" - - "flattening" - properties: - format_type: - type: "string" - enum: - - "CSV" - default: "CSV" - flattening: - type: "string" - title: "Normalization (Flattening)" - description: "Whether the input json data should be normalized (flattened)\ - \ in the output CSV. Please refer to docs for details." - default: "No flattening" - enum: - - "No flattening" - - "Root level flattening" - part_size_mb: - title: "Block Size (MB) for Amazon S3 multipart upload" - description: "This is the size of a \"Part\" being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will allow\ - \ to upload a bigger files and improve the speed, but consumes9\ - \ more memory. Allowed values: min=5MB, max=525MB Default: 5MB." - type: "integer" - default: 5 - examples: - - 5 - - title: "JSON Lines: newline-delimited JSON" - required: - - "format_type" - properties: - format_type: - type: "string" - enum: - - "JSONL" - default: "JSONL" - part_size_mb: - title: "Block Size (MB) for Amazon S3 multipart upload" - description: "This is the size of a \"Part\" being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will allow\ - \ to upload a bigger files and improve the speed, but consumes9\ - \ more memory. Allowed values: min=5MB, max=525MB Default: 5MB." - type: "integer" - default: 5 - examples: - - 5 - - title: "Parquet: Columnar Storage" - required: - - "format_type" - properties: - format_type: - type: "string" - enum: - - "Parquet" - default: "Parquet" - compression_codec: - title: "Compression Codec" - description: "The compression algorithm used to compress data pages." - type: "string" - enum: - - "UNCOMPRESSED" - - "SNAPPY" - - "GZIP" - - "LZO" - - "BROTLI" - - "LZ4" - - "ZSTD" - default: "UNCOMPRESSED" - block_size_mb: - title: "Block Size (Row Group Size) (MB)" - description: "This is the size of a row group being buffered in memory.\ - \ It limits the memory usage when writing. Larger values will improve\ - \ the IO when reading, but consume more memory when writing. Default:\ - \ 128 MB." - type: "integer" - default: 128 - examples: - - 128 - max_padding_size_mb: - title: "Max Padding Size (MB)" - description: "Maximum size allowed as padding to align row groups.\ - \ This is also the minimum size of a row group. Default: 8 MB." - type: "integer" - default: 8 - examples: - - 8 - page_size_kb: - title: "Page Size (KB)" - description: "The page size is for compression. A block is composed\ - \ of pages. A page is the smallest unit that must be read fully\ - \ to access a single record. If this value is too small, the compression\ - \ will deteriorate. Default: 1024 KB." - type: "integer" - default: 1024 - examples: - - 1024 - dictionary_page_size_kb: - title: "Dictionary Page Size (KB)" - description: "There is one dictionary page per column per row group\ - \ when dictionary encoding is used. The dictionary page size works\ - \ like the page size but for dictionary. Default: 1024 KB." - type: "integer" - default: 1024 - examples: - - 1024 - dictionary_encoding: - title: "Dictionary Encoding" - description: "Default: true." - type: "boolean" - default: true - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" -- dockerImage: "airbyte/destination-snowflake:0.3.16" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Snowflake Destination Spec" - type: "object" - required: - - "host" - - "role" - - "warehouse" - - "database" - - "schema" - - "username" - - "password" - additionalProperties: true - properties: - host: - description: "Host domain of the snowflake instance (must include the account,\ - \ region, cloud environment, and end with snowflakecomputing.com)." - examples: - - "accountname.us-east-2.aws.snowflakecomputing.com" - type: "string" - title: "Host" - order: 0 - role: - description: "The role you created for Airbyte to access Snowflake." - examples: - - "AIRBYTE_ROLE" - type: "string" - title: "Role" - order: 1 - warehouse: - description: "The warehouse you created for Airbyte to sync data into." - examples: - - "AIRBYTE_WAREHOUSE" - type: "string" - title: "Warehouse" - order: 2 - database: - description: "The database you created for Airbyte to sync data into." - examples: - - "AIRBYTE_DATABASE" - type: "string" - title: "Database" - order: 3 - schema: - description: "The default Snowflake schema tables are written to if the\ - \ source does not specify a namespace." - examples: - - "AIRBYTE_SCHEMA" - type: "string" - title: "Default Schema" - order: 4 - username: - description: "The username you created to allow Airbyte to access the database." - examples: - - "AIRBYTE_USER" - type: "string" - title: "Username" - order: 5 - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - title: "Password" - order: 6 - loading_method: - type: "object" - title: "Loading Method" - description: "Loading method used to send data to Snowflake." - order: 7 - oneOf: - - title: "Standard Inserts" - additionalProperties: false - description: "Uses 
INSERT
statements to send batches of records\ - \ to Snowflake. Easiest (no setup) but not recommended for large production\ - \ workloads due to slow speed." - required: - - "method" - properties: - method: - type: "string" - enum: - - "Standard" - default: "Standard" - - title: "AWS S3 Staging" - additionalProperties: false - description: "Writes large batches of records to a file, uploads the file\ - \ to S3, then uses 
COPY INTO table
to upload the file. Recommended\ - \ for large production workloads for better speed and scalability." - required: - - "method" - - "s3_bucket_name" - - "access_key_id" - - "secret_access_key" - properties: - method: - type: "string" - enum: - - "S3 Staging" - default: "S3 Staging" - order: 0 - s3_bucket_name: - title: "S3 Bucket Name" - type: "string" - description: "The name of the staging S3 bucket. Airbyte will write\ - \ files to this bucket and read them via 
COPY
statements\ - \ on Snowflake." - examples: - - "airbyte.staging" - order: 1 - s3_bucket_region: - title: "S3 Bucket Region" - type: "string" - default: "" - description: "The region of the S3 staging bucket to use if utilising\ - \ a copy strategy." - enum: - - "" - - "us-east-1" - - "us-east-2" - - "us-west-1" - - "us-west-2" - - "af-south-1" - - "ap-east-1" - - "ap-south-1" - - "ap-northeast-1" - - "ap-northeast-2" - - "ap-northeast-3" - - "ap-southeast-1" - - "ap-southeast-2" - - "ca-central-1" - - "cn-north-1" - - "cn-northwest-1" - - "eu-central-1" - - "eu-west-1" - - "eu-west-2" - - "eu-west-3" - - "eu-south-1" - - "eu-north-1" - - "sa-east-1" - - "me-south-1" - order: 2 - access_key_id: - type: "string" - description: "The Access Key Id granting allow one to access the above\ - \ S3 staging bucket. Airbyte requires Read and Write permissions\ - \ to the given bucket." - title: "S3 Key Id" - airbyte_secret: true - order: 3 - secret_access_key: - type: "string" - description: "The corresponding secret to the above access key id." - title: "S3 Access Key" - airbyte_secret: true - order: 4 - - title: "GCS Staging" - additionalProperties: false - description: "Writes large batches of records to a file, uploads the file\ - \ to GCS, then uses 
COPY INTO table
to upload the file. Recommended\ - \ for large production workloads for better speed and scalability." - required: - - "method" - - "project_id" - - "bucket_name" - - "credentials_json" - properties: - method: - type: "string" - enum: - - "GCS Staging" - default: "GCS Staging" - order: 0 - project_id: - title: "GCP Project ID" - type: "string" - description: "The name of the GCP project ID for your credentials." - examples: - - "my-project" - order: 1 - bucket_name: - title: "GCS Bucket Name" - type: "string" - description: "The name of the staging GCS bucket. Airbyte will write\ - \ files to this bucket and read them via 
COPY
statements\ - \ on Snowflake." - examples: - - "airbyte-staging" - order: 2 - credentials_json: - title: "Google Application Credentials" - type: "string" - description: "The contents of the JSON key file that has read/write\ - \ permissions to the staging GCS bucket. You will separately need\ - \ to grant bucket access to your Snowflake GCP service account.\ - \ See the GCP docs for more information on how to generate a JSON key\ - \ for your service account." - airbyte_secret: true - multiline: true - order: 3 - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml deleted file mode 100644 index 71e4a45e6915..000000000000 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ /dev/null @@ -1,5836 +0,0 @@ -# This file is generated by io.airbyte.config.specs.SeedConnectorSpecGenerator. -# Do NOT edit this file directly. See generator class for more details. ---- -- dockerImage: "airbyte/source-aws-cloudtrail:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/aws-cloudtrail" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Aws CloudTrail Spec" - type: "object" - required: - - "aws_key_id" - - "aws_secret_key" - - "aws_region_name" - - "start_date" - additionalProperties: true - properties: - aws_key_id: - type: "string" - description: "Specifies an AWS access key associated with an IAM user or\ - \ role." - airbyte_secret: true - aws_secret_key: - type: "string" - description: "Specifies the secret key associated with the access key. This\ - \ is essentially the 'password' for the access key." - airbyte_secret: true - aws_region_name: - type: "string" - description: "The default AWS Region to use, for example, us-west-1 or us-west-2.\ - \ When specifying a Region inline during client initialization, this property\ - \ is named region_name." - start_date: - type: "string" - description: "The date you would like to replicate data. Data in ClouTraid\ - \ is available for last 90 days only. Format: YYYY-MM-DD." - examples: - - "2021-01-01" - default: "1970-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-ads" - connectionSpecification: - title: "Amazon Ads Spec" - type: "object" - properties: - client_id: - title: "Client Id" - description: "Oauth client id How to create your Login with Amazon" - name: "Client ID" - type: "string" - client_secret: - title: "Client Secret" - description: "Oauth client secret How to create your Login with Amazon" - name: "Client secret" - airbyte_secret: true - type: "string" - scope: - title: "Scope" - description: "By default its advertising::campaign_management, but customers\ - \ may need to set scope to cpc_advertising:campaign_management." - default: "advertising::campaign_management" - name: "Client scope" - examples: - - "cpc_advertising:campaign_management" - type: "string" - refresh_token: - title: "Refresh Token" - description: "Oauth 2.0 refresh_token, read details here" - name: "Oauth refresh token" - airbyte_secret: true - type: "string" - start_date: - title: "Start Date" - description: "Start date for collectiong reports, should not be more than\ - \ 60 days in past. In YYYY-MM-DD format" - name: "Start date" - examples: - - "2022-10-10" - - "2022-10-22" - type: "string" - region: - description: "Region to pull data from (EU/NA/FE/SANDBOX)" - default: "NA" - name: "Region" - title: "AmazonAdsRegion" - enum: - - "NA" - - "EU" - - "FE" - - "SANDBOX" - type: "string" - profiles: - title: "Profiles" - description: "profile Ids you want to fetch data for" - name: "Profile Ids" - type: "array" - items: - type: "integer" - required: - - "client_id" - - "client_secret" - - "refresh_token" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-seller-partner:0.2.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner" - changelogUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner" - connectionSpecification: - title: "Amazon Seller Partner Spec" - type: "object" - properties: - replication_start_date: - title: "Replication Start Date" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-25T00:00:00Z" - type: "string" - refresh_token: - title: "Refresh Token" - description: "The refresh token used obtained via authorization (can be\ - \ passed to the client instead)" - airbyte_secret: true - type: "string" - lwa_app_id: - title: "Lwa App Id" - description: "Your login with amazon app id" - airbyte_secret: true - type: "string" - lwa_client_secret: - title: "Lwa Client Secret" - description: "Your login with amazon client secret" - airbyte_secret: true - type: "string" - aws_access_key: - title: "Aws Access Key" - description: "AWS user access key" - airbyte_secret: true - type: "string" - aws_secret_key: - title: "Aws Secret Key" - description: "AWS user secret key" - airbyte_secret: true - type: "string" - role_arn: - title: "Role Arn" - description: "The role's arn (needs permission to 'Assume Role' STS)" - airbyte_secret: true - type: "string" - aws_environment: - title: "AWSEnvironment" - description: "An enumeration." - enum: - - "PRODUCTION" - - "SANDBOX" - type: "string" - region: - title: "AWSRegion" - description: "An enumeration." - enum: - - "AE" - - "DE" - - "PL" - - "EG" - - "ES" - - "FR" - - "IN" - - "IT" - - "NL" - - "SA" - - "SE" - - "TR" - - "UK" - - "AU" - - "JP" - - "SG" - - "US" - - "BR" - - "CA" - - "MX" - - "GB" - type: "string" - required: - - "replication_start_date" - - "refresh_token" - - "lwa_app_id" - - "lwa_client_secret" - - "aws_access_key" - - "aws_secret_key" - - "role_arn" - - "aws_environment" - - "region" - definitions: - AWSEnvironment: - title: "AWSEnvironment" - description: "An enumeration." - enum: - - "PRODUCTION" - - "SANDBOX" - type: "string" - AWSRegion: - title: "AWSRegion" - description: "An enumeration." - enum: - - "AE" - - "DE" - - "PL" - - "EG" - - "ES" - - "FR" - - "IN" - - "IT" - - "NL" - - "SA" - - "SE" - - "TR" - - "UK" - - "AU" - - "JP" - - "SG" - - "US" - - "BR" - - "CA" - - "MX" - - "GB" - type: "string" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amplitude:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/amplitude" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Amplitude Spec" - type: "object" - required: - - "api_key" - - "secret_key" - - "start_date" - additionalProperties: false - properties: - api_key: - type: "string" - description: "This is the project’s API key, used for calling Amplitude’\ - s APIs" - airbyte_secret: true - secret_key: - type: "string" - description: "This is the project's secret key, which is also used for calling\ - \ Amplitude’s APIs" - airbyte_secret: true - start_date: - type: "string" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - description: "UTC date and time in the format 2021-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2021-01-25T00:00:00Z" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-apify-dataset:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/apify-dataset" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Apify Dataset Spec" - type: "object" - required: - - "datasetId" - additionalProperties: false - properties: - datasetId: - type: "string" - description: "ID of the dataset you would like to load to Airbyte." - clean: - type: "boolean" - description: "If set to true, only clean items will be downloaded from the\ - \ dataset. See description of what clean means in Apify API docs. If not sure, set clean to false." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-appstore-singer:0.2.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/appstore" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Appstore Singer Spec" - type: "object" - required: - - "key_id" - - "private_key" - - "issuer_id" - - "vendor" - - "start_date" - additionalProperties: false - properties: - key_id: - type: "string" - description: "Key_id is the API key you use to connect to appstore's API." - private_key: - type: "string" - description: "Private_key is the contents of the key file you use to connect to appstore's API." - airbyte_secret: true - multiline: true - issuer_id: - type: "string" - description: "Issuer_id is used to generate the credentials to connect to appstore's\ - \ API." - vendor: - type: "string" - description: "This is the Apple ID of your account." - start_date: - type: "string" - description: "Date from which to start pulling data." - examples: - - "2020-11-16T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-asana:0.1.3" - spec: - documentationUrl: "https://docsurl.com" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Asana Spec" - type: "object" - additionalProperties: true - properties: - credentials: - title: "Authentication mechanism" - description: "Choose how to authenticate to Github" - type: "object" - oneOf: - - type: "object" - title: "Authenticate with Personal Access Token" - required: - - "personal_access_token" - properties: - option_title: - type: "string" - title: "Credentials title" - description: "PAT Credentials" - const: "PAT Credentials" - personal_access_token: - type: "string" - title: "Personal Access Token" - description: "Asana Personal Access Token (generate yours here)." - airbyte_secret: true - - type: "object" - title: "Authenticate via Asana (Oauth)" - required: - - "client_id" - - "client_secret" - - "refresh_token" - properties: - option_title: - type: "string" - title: "Credentials title" - description: "OAuth Credentials" - const: "OAuth Credentials" - client_id: - type: "string" - title: "" - description: "" - airbyte_secret: true - client_secret: - type: "string" - title: "" - description: "" - airbyte_secret: true - refresh_token: - type: "string" - title: "" - description: "" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "1" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "refresh_token" -- dockerImage: "airbyte/source-bamboo-hr:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/bamboo-hr" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Bamboo HR Spec" - type: "object" - required: - - "subdomain" - - "api_key" - additionalProperties: false - properties: - subdomain: - type: "string" - description: "Sub Domain of bamboo hr" - api_key: - type: "string" - description: "Api key of bamboo hr" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bigcommerce:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/bigcommerce" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "BigCommerce Source CDK Specifications" - type: "object" - required: - - "start_date" - - "store_hash" - - "access_token" - additionalProperties: false - properties: - start_date: - type: "string" - description: "The date you would like to replicate data. Format: YYYY-MM-DD." - examples: - - "2021-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - store_hash: - type: "string" - description: "The hash code of the store. For https://api.bigcommerce.com/stores/HASH_CODE/v3/,\ - \ The store's hash code is 'HASH_CODE'." - access_token: - type: "string" - description: "The API Access Token." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bigquery:0.1.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/source/bigquery" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "BigQuery Source Spec" - type: "object" - required: - - "project_id" - - "credentials_json" - additionalProperties: false - properties: - project_id: - type: "string" - description: "The GCP project ID for the project containing the target BigQuery\ - \ dataset." - title: "Project ID" - dataset_id: - type: "string" - description: "The BigQuery Dataset ID to look for tables to replicate from." - title: "Default Dataset ID" - credentials_json: - type: "string" - description: "The contents of the JSON service account key. Check out the\ - \ docs\ - \ if you need help generating this key." - title: "Credentials JSON" - airbyte_secret: true - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: [] - supported_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Bing Ads Spec" - type: "object" - required: - - "accounts" - - "client_id" - - "client_secret" - - "customer_id" - - "developer_token" - - "refresh_token" - - "user_id" - - "reports_start_date" - - "hourly_reports" - - "daily_reports" - - "weekly_reports" - - "monthly_reports" - additionalProperties: false - properties: - accounts: - title: "Accounts" - type: "object" - description: "Account selection strategy." - oneOf: - - title: "All accounts assigned to your user" - additionalProperties: false - description: "Fetch data for all available accounts." - required: - - "selection_strategy" - properties: - selection_strategy: - type: "string" - enum: - - "all" - const: "all" - - title: "Subset of your accounts" - additionalProperties: false - description: "Fetch data for subset of account ids." - required: - - "ids" - - "selection_strategy" - properties: - selection_strategy: - type: "string" - enum: - - "subset" - const: "subset" - ids: - type: "array" - description: "List of accounts from which data will be fetched." - items: - type: "string" - minItems: 1 - uniqueItems: true - client_id: - type: "string" - description: "ID of your Microsoft Advertising client application." - airbyte_secret: true - client_secret: - type: "string" - description: "Secret of your Microsoft Advertising client application." - airbyte_secret: true - customer_id: - type: "string" - description: "User's customer ID." - developer_token: - type: "string" - description: "Developer token associated with user." - airbyte_secret: true - refresh_token: - type: "string" - description: "The long-lived Refresh token received via grant_type=refresh_token\ - \ request." - airbyte_secret: true - user_id: - type: "string" - description: "Unique user identifier." - reports_start_date: - type: "string" - format: "date" - default: "2020-01-01" - description: "From which date perform initial sync for report related streams.\ - \ In YYYY-MM-DD format" - hourly_reports: - title: "Hourly reports" - type: "boolean" - description: "The report data will be aggregated by each hour of the day." - default: false - daily_reports: - title: "Daily reports" - type: "boolean" - description: "The report data will be aggregated by each day." - default: false - weekly_reports: - title: "Weekly reports" - type: "boolean" - description: "The report data will be aggregated by each week running from\ - \ Sunday through Saturday." - default: false - monthly_reports: - title: "Monthly reports" - type: "boolean" - description: "The report data will be aggregated by each month." - default: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-braintree:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/braintree" - connectionSpecification: - title: "Braintree Spec" - type: "object" - properties: - merchant_id: - title: "Merchant Id" - description: "Merchant ID is the unique identifier for entire gateway account." - name: "Merchant ID" - type: "string" - public_key: - title: "Public Key" - description: "This is your user-specific public identifier for Braintree." - name: "Public key" - type: "string" - private_key: - title: "Private Key" - description: "This is your user-specific private identifier." - name: "Private Key" - airbyte_secret: true - type: "string" - start_date: - title: "Start Date" - description: "The date from which you'd like to replicate data for Braintree\ - \ API for UTC timezone, All data generated after this date will be replicated." - name: "Start date" - examples: - - "2020" - - "2020-12-30" - - "2020-11-22 20:20:05" - type: "string" - format: "date-time" - environment: - description: "Environment specifies where the data will come from." - name: "Environment" - examples: - - "sandbox" - - "production" - - "qa" - - "development" - allOf: - - $ref: "#/definitions/Environment" - required: - - "merchant_id" - - "public_key" - - "private_key" - - "environment" - definitions: - Environment: - title: "Environment" - description: "An enumeration." - enum: - - "Development" - - "Sandbox" - - "Qa" - - "Production" - type: "string" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cart:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/cart" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Cart Spec" - type: "object" - required: - - "access_token" - - "start_date" - - "store_name" - additionalProperties: true - properties: - access_token: - type: "string" - airbyte_secret: true - description: "API Key. See the docs for information on how to generate this key." - store_name: - type: "string" - description: "Store name. All API URLs start with https://[mystorename.com]/api/v1/,\ - \ where [mystorename.com] is the domain name of your store." - start_date: - title: "Start Date" - type: "string" - description: "The date from which you'd like to replicate the data" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-01-01T00:00:00Z" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-chargebee:0.1.4" - spec: - documentationUrl: "https://apidocs.chargebee.com/docs/api" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Chargebee Spec" - type: "object" - required: - - "site" - - "site_api_key" - - "start_date" - - "product_catalog" - additionalProperties: false - properties: - site: - type: "string" - title: "Site" - description: "The site prefix for your Chargebee instance." - examples: - - "airbyte-test" - site_api_key: - type: "string" - title: "API Key" - description: "The API key from your Chargebee instance." - examples: - - "test_3yzfanAXF66USdWC9wQcM555DQJkSYoppu" - airbyte_secret: true - start_date: - type: "string" - title: "Start Date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - description: "UTC date and time in the format 2021-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2021-01-25T00:00:00Z" - product_catalog: - title: "Product Catalog" - type: "string" - description: "Product Catalog version of your Chargebee site. Instructions\ - \ on how to find your version you may find here under `API Version` section." - enum: - - "1.0" - - "2.0" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-clickhouse:0.1.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "ClickHouse Source Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - additionalProperties: false - properties: - host: - description: "Host Endpoint of the Clickhouse Cluster" - type: "string" - port: - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 8123 - examples: - - "8123" - database: - description: "Name of the database." - type: "string" - examples: - - "default" - username: - description: "Username to use to access the database." - type: "string" - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - ssl: - title: "SSL Connection" - description: "Encrypt data using SSL." - type: "boolean" - default: true - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials in RSA PEM\ - \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-close-com:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/close-com" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Close.com Spec" - type: "object" - required: - - "api_key" - additionalProperties: false - properties: - api_key: - type: "string" - description: "Close.com API key (usually starts with 'api_'; find yours\ - \ here)." - airbyte_secret: true - start_date: - type: "string" - description: "The start date to sync data. Leave blank for full sync. Format:\ - \ YYYY-MM-DD." - examples: - - "2021-01-01" - default: "2021-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cockroachdb:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/postgres" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Cockroach Source Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - additionalProperties: false - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - order: 0 - port: - title: "Port" - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 5432 - examples: - - "5432" - order: 1 - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 3 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 4 - ssl: - title: "Connect using SSL" - description: "Encrypt client/server communications for increased security." - type: "boolean" - default: false - order: 5 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-dixa:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/dixa" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Dixa Spec" - type: "object" - required: - - "api_token" - - "start_date" - additionalProperties: false - properties: - api_token: - type: "string" - description: "Dixa API token" - airbyte_secret: true - start_date: - type: "string" - description: "The connector pulls records updated from this date onwards." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - examples: - - "YYYY-MM-DD" - batch_size: - type: "integer" - description: "Number of days to batch into one request. Max 31." - pattern: "^[0-9]{1,2}$" - examples: - - 1 - - 31 - default: 31 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-drift:0.2.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/drift" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Drift Spec" - type: "object" - required: - - "access_token" - additionalProperties: false - properties: - access_token: - type: "string" - description: "Drift Access Token. See the docs for more information on how to generate this key." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-exchange-rates:0.2.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/exchangeratesapi" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "ratesapi.io Source Spec" - type: "object" - required: - - "start_date" - - "access_key" - additionalProperties: false - properties: - start_date: - type: "string" - description: "Start getting data from that date." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - examples: - - "YYYY-MM-DD" - access_key: - type: "string" - description: "Your API Access Key. See here. The key is case sensitive." - airbyte_secret: true - base: - type: "string" - description: "ISO reference currency. See here. Free plan doesn't support Source Currency Switching, default\ - \ base currency is EUR" - examples: - - "EUR" - - "USD" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.21" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" - changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" - connectionSpecification: - title: "Source Facebook Marketing" - type: "object" - properties: - account_id: - title: "Account Id" - description: "The Facebook Ad account ID to use when pulling data from the\ - \ Facebook Marketing API." - type: "string" - access_token: - title: "Access Token" - description: "The value of the access token generated. See the docs\ - \ for more information" - airbyte_secret: true - type: "string" - start_date: - title: "Start Date" - description: "The date from which you'd like to replicate data for AdCreatives\ - \ and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated\ - \ after this date will be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-25T00:00:00Z" - type: "string" - format: "date-time" - end_date: - title: "End Date" - description: "The date until which you'd like to replicate data for AdCreatives\ - \ and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated\ - \ between start_date and this date will be replicated. Not setting this\ - \ option will result in always syncing the latest data." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-26T00:00:00Z" - type: "string" - format: "date-time" - include_deleted: - title: "Include Deleted" - description: "Include data from deleted campaigns, ads, and adsets." - default: false - type: "boolean" - insights_lookback_window: - title: "Insights Lookback Window" - description: "The attribution window for the actions" - default: 28 - minimum: 0 - maximum: 28 - type: "integer" - insights_days_per_job: - title: "Insights Days Per Job" - description: "Number of days to sync in one job. The more data you have\ - \ - the smaller you want this parameter to be." - default: 7 - minimum: 1 - maximum: 30 - type: "integer" - custom_insights: - title: "Custom Insights" - description: "A list wich contains insights entries, each entry must have\ - \ a name and can contains fields, breakdowns or action_breakdowns)" - type: "array" - items: - title: "InsightConfig" - type: "object" - properties: - name: - title: "Name" - description: "The name value of insight" - type: "string" - fields: - title: "Fields" - description: "A list of chosen fields for fields parameter" - default: [] - type: "array" - items: - type: "string" - breakdowns: - title: "Breakdowns" - description: "A list of chosen breakdowns for breakdowns" - default: [] - type: "array" - items: - type: "string" - action_breakdowns: - title: "Action Breakdowns" - description: "A list of chosen action_breakdowns for action_breakdowns" - default: [] - type: "array" - items: - type: "string" - required: - - "name" - required: - - "account_id" - - "access_token" - - "start_date" - definitions: - InsightConfig: - title: "InsightConfig" - type: "object" - properties: - name: - title: "Name" - description: "The name value of insight" - type: "string" - fields: - title: "Fields" - description: "A list of chosen fields for fields parameter" - default: [] - type: "array" - items: - type: "string" - breakdowns: - title: "Breakdowns" - description: "A list of chosen breakdowns for breakdowns" - default: [] - type: "array" - items: - type: "string" - action_breakdowns: - title: "Action Breakdowns" - description: "A list of chosen action_breakdowns for action_breakdowns" - default: [] - type: "array" - items: - type: "string" - required: - - "name" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: [] - oauthFlowInitParameters: [] - oauthFlowOutputParameters: - - - "access_token" -- dockerImage: "airbyte/source-facebook-pages:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-pages" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Facebook Pages Spec" - type: "object" - required: - - "access_token" - - "page_id" - additionalProperties: false - properties: - access_token: - type: "string" - description: "Facebook Page Access Token" - airbyte_secret: true - page_id: - type: "string" - description: "Page ID" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: [] - oauthFlowInitParameters: [] - oauthFlowOutputParameters: - - - "access_token" -- dockerImage: "airbyte/source-file:0.2.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/file" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "File Source Spec" - type: "object" - additionalProperties: false - required: - - "dataset_name" - - "format" - - "url" - - "provider" - properties: - dataset_name: - type: "string" - description: "Name of the final table where to replicate this file (should\ - \ include only letters, numbers dash and underscores)" - format: - type: "string" - enum: - - "csv" - - "json" - - "jsonl" - - "excel" - - "feather" - - "parquet" - default: "csv" - description: "File Format of the file to be replicated (Warning: some format\ - \ may be experimental, please refer to docs)." - reader_options: - type: "string" - description: "This should be a valid JSON string used by each reader/parser\ - \ to provide additional options and tune its behavior" - examples: - - "{}" - - "{'sep': ' '}" - url: - type: "string" - description: "URL path to access the file to be replicated" - provider: - type: "object" - description: "Storage Provider or Location of the file(s) to be replicated." - default: "Public Web" - oneOf: - - title: "HTTPS: Public Web" - required: - - "storage" - properties: - storage: - type: "string" - enum: - - "HTTPS" - default: "HTTPS" - - title: "GCS: Google Cloud Storage" - required: - - "storage" - properties: - storage: - type: "string" - enum: - - "GCS" - default: "GCS" - service_account_json: - type: "string" - description: "In order to access private Buckets stored on Google\ - \ Cloud, this connector would need a service account json credentials\ - \ with the proper permissions as described here. Please generate the credentials.json\ - \ file and copy/paste its content to this field (expecting JSON\ - \ formats). If accessing publicly available data, this field is\ - \ not necessary." - - title: "S3: Amazon Web Services" - required: - - "storage" - properties: - storage: - type: "string" - enum: - - "S3" - default: "S3" - aws_access_key_id: - type: "string" - description: "In order to access private Buckets stored on AWS S3,\ - \ this connector would need credentials with the proper permissions.\ - \ If accessing publicly available data, this field is not necessary." - aws_secret_access_key: - type: "string" - description: "In order to access private Buckets stored on AWS S3,\ - \ this connector would need credentials with the proper permissions.\ - \ If accessing publicly available data, this field is not necessary." - airbyte_secret: true - - title: "AzBlob: Azure Blob Storage" - required: - - "storage" - - "storage_account" - properties: - storage: - type: "string" - enum: - - "AzBlob" - default: "AzBlob" - storage_account: - type: "string" - description: "The globally unique name of the storage account that\ - \ the desired blob sits within. See here for more details." - sas_token: - type: "string" - description: "To access Azure Blob Storage, this connector would need\ - \ credentials with the proper permissions. One option is a SAS (Shared\ - \ Access Signature) token. If accessing publicly available data,\ - \ this field is not necessary." - airbyte_secret: true - shared_key: - type: "string" - description: "To access Azure Blob Storage, this connector would need\ - \ credentials with the proper permissions. One option is a storage\ - \ account shared key (aka account key or access key). If accessing\ - \ publicly available data, this field is not necessary." - airbyte_secret: true - - title: "SSH: Secure Shell" - required: - - "storage" - - "user" - - "host" - properties: - storage: - type: "string" - enum: - - "SSH" - default: "SSH" - user: - type: "string" - password: - type: "string" - airbyte_secret: true - host: - type: "string" - port: - type: "string" - default: "22" - - title: "SCP: Secure copy protocol" - required: - - "storage" - - "user" - - "host" - properties: - storage: - type: "string" - enum: - - "SCP" - default: "SCP" - user: - type: "string" - password: - type: "string" - airbyte_secret: true - host: - type: "string" - port: - type: "string" - default: "22" - - title: "SFTP: Secure File Transfer Protocol" - required: - - "storage" - - "user" - - "host" - properties: - storage: - type: "string" - enum: - - "SFTP" - default: "SFTP" - user: - type: "string" - password: - type: "string" - airbyte_secret: true - host: - type: "string" - port: - type: "string" - default: "22" - - title: "Local Filesystem (limited)" - required: - - "storage" - properties: - storage: - type: "string" - description: "WARNING: Note that local storage URL available for read\ - \ must start with the local mount \"/local/\" at the moment until\ - \ we implement more advanced docker mounting options..." - enum: - - "local" - default: "local" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-freshdesk:0.2.7" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Freshdesk Spec" - type: "object" - required: - - "domain" - - "api_key" - additionalProperties: false - properties: - domain: - type: "string" - description: "Freshdesk domain" - examples: - - "myaccount.freshdesk.com" - pattern: - - "^[a-zA-Z0-9._-]*\\.freshdesk\\.com$" - api_key: - type: "string" - description: "Freshdesk API Key. See the docs for more information on how to obtain this key." - airbyte_secret: true - requests_per_minute: - title: "Requests per minute" - type: "integer" - description: "Number of requests per minute that this source allowed to\ - \ use." - start_date: - title: "Start date" - description: "Date from which to start pulling data." - format: "date-time" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2020-12-01T00:00:00Z" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-freshservice:0.1.0" - spec: - documentationUrl: "https://hub.docker.com/r/airbyte/source-freshservice" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Freshservice Spec" - type: "object" - required: - - "domain_name" - - "api_key" - - "start_date" - additionalProperties: false - properties: - domain_name: - type: "string" - description: "Freshservice domain" - examples: - - "mydomain.freshservice.com" - api_key: - title: "Api Key" - type: "string" - description: "Your API Access Key. See here. The key is case sensitive." - airbyte_secret: true - start_date: - title: "Replication Start Date" - type: "string" - description: "UTC date and time in the format 2020-10-01T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2020-10-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.2.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/github" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Github Source Spec" - type: "object" - required: - - "start_date" - - "repository" - additionalProperties: true - properties: - credentials: - title: "Authentication mechanism" - description: "Choose how to authenticate to Github" - type: "object" - oneOf: - - type: "object" - title: "Authenticate via Github (Oauth)" - required: - - "access_token" - properties: - option_title: - type: "string" - title: "Credentials title" - description: "OAuth Credentials" - const: "OAuth Credentials" - access_token: - type: "string" - title: "Access Token" - description: "Oauth access token" - airbyte_secret: true - - type: "object" - title: "Authenticate with Personal Access Token" - required: - - "personal_access_token" - properties: - option_title: - type: "string" - title: "Credentials title" - description: "PAT Credentials" - const: "PAT Credentials" - personal_access_token: - type: "string" - title: "Personal Access Tokens" - description: "Log into Github and then generate a personal access token. To load balance your API quota consumption\ - \ across multiple API tokens, input multiple tokens separated with\ - \ \",\"" - airbyte_secret: true - repository: - type: "string" - examples: - - "airbytehq/airbyte" - - "airbytehq/*" - title: "Github repositories" - description: "Space-delimited list of GitHub repositories/organizations,\ - \ e.g. `airbytehq/airbyte` for single repository and `airbytehq/*` for\ - \ get all repositories from organization" - start_date: - type: "string" - title: "Start date" - description: "The date from which you'd like to replicate data for GitHub\ - \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\ - \ will be replicated. Note that it will be used only in the following\ - \ incremental streams: comments, commits and issues." - examples: - - "2021-03-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - branch: - type: "string" - title: "Branch" - examples: - - "airbytehq/airbyte/master" - description: "Space-delimited list of GitHub repository branches to pull\ - \ commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified\ - \ for a repository, the default branch will be pulled." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "0" - oauthFlowInitParameters: [] - oauthFlowOutputParameters: - - - "access_token" -- dockerImage: "airbyte/source-gitlab:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/gitlab" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Gitlab Singer Spec" - type: "object" - required: - - "api_url" - - "private_token" - - "start_date" - additionalProperties: false - properties: - api_url: - type: "string" - examples: - - "gitlab.com" - description: "Please enter your basic URL from Gitlab instance" - private_token: - type: "string" - description: "Log into your Gitlab account and then generate a personal\ - \ Access Token." - airbyte_secret: true - groups: - type: "string" - examples: - - "airbyte.io" - description: "Space-delimited list of groups. e.g. airbyte.io" - projects: - type: "string" - examples: - - "airbyte.io/documentation" - description: "Space-delimited list of projects. e.g. airbyte.io/documentation\ - \ meltano/tap-gitlab" - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Gitlab\ - \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\ - \ date will be replicated." - examples: - - "2021-03-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-google-ads:0.1.15" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/google-ads" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Ads Spec" - type: "object" - required: - - "credentials" - - "start_date" - - "customer_id" - additionalProperties: true - properties: - credentials: - type: "object" - title: "Google Credentials" - required: - - "developer_token" - - "client_id" - - "client_secret" - - "refresh_token" - properties: - developer_token: - type: "string" - title: "Developer Token" - description: "Developer token granted by Google to use their APIs. More\ - \ instruction on how to find this value in our docs" - airbyte_secret: true - client_id: - type: "string" - title: "Client Id" - description: "Google client id. More instruction on how to find this\ - \ value in our docs" - client_secret: - type: "string" - title: "Client Secret" - description: "Google client secret. More instruction on how to find\ - \ this value in our docs" - airbyte_secret: true - access_token: - type: "string" - title: "Access Token" - description: "Access token generated using developer_token, oauth_client_id,\ - \ and oauth_client_secret. More instruction on how to find this value\ - \ in our docs" - airbyte_secret: true - refresh_token: - type: "string" - title: "Refresh Token" - description: "Refresh token generated using developer_token, oauth_client_id,\ - \ and oauth_client_secret. More instruction on how to find this value\ - \ in our docs" - airbyte_secret: true - customer_id: - title: "Customer Id" - type: "string" - description: "Customer id must be specified as a 10-digit number without\ - \ dashes. More instruction on how to find this value in our docs" - login_customer_id: - type: "string" - title: "Login Customer ID" - description: "If your access to the customer account is through a manager\ - \ account, this field is required and must be set to the customer ID of\ - \ the manager account (10-digit number without dashes). More information\ - \ about this field you can see here" - start_date: - type: "string" - title: "Start Date" - description: "UTC date and time in the format 2017-01-25. Any data before\ - \ this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - examples: - - "2017-01-25" - conversion_window_days: - title: "Conversion Window" - type: "integer" - description: "Define the historical replication lookback window in days" - minimum: 0 - maximum: 1095 - default: 14 - examples: - - 14 - custom_queries: - type: "array" - title: "Custom GAQL Queries" - items: - type: "object" - properties: - query: - type: "string" - title: "Custom query" - description: "A custom defined GAQL query for building the report.\ - \ Should not contain segments.date expression as it used by incremental\ - \ streams" - examples: - - "SELECT segments.ad_destination_type, campaign.advertising_channel_sub_type\ - \ FROM campaign WHERE campaign.status = 'PAUSED'" - table_name: - type: "string" - title: "Destination table name" - description: "The table name in your destination database for choosen\ - \ query." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - - - "developer_token" - oauthFlowOutputParameters: - - - "access_token" - - - "refresh_token" -- dockerImage: "airbyte/source-google-analytics-v4:0.1.9" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/google-analytics-v4" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Analytics V4 Spec" - type: "object" - required: - - "view_id" - - "start_date" - additionalProperties: true - properties: - view_id: - type: "string" - title: "View ID" - description: "The ID for the Google Analytics View you want to fetch data\ - \ from. This can be found from the Google Analytics Account Explorer." - airbyte_secret: true - start_date: - type: "string" - title: "Start Date" - description: "A date in the format YYYY-MM-DD." - examples: - - "2020-06-01" - window_in_days: - type: "integer" - description: "The amount of days for each data-chunk begining from start_date.\ - \ Bigger the value - faster the fetch. (Min=1, as for a Day; Max=364,\ - \ as for a Year)." - examples: - - 30 - - 60 - - 90 - - 120 - - 200 - - 364 - default: 90 - custom_reports: - title: "Custom Reports" - type: "string" - description: "A JSON array describing the custom reports you want to sync\ - \ from GA. Check out the docs to get more information about this field." - credentials: - type: "object" - oneOf: - - title: "Authenticate via Google (Oauth)" - type: "object" - required: - - "client_id" - - "client_secret" - - "refresh_token" - properties: - auth_type: - type: "string" - const: "Client" - enum: - - "Client" - default: "Client" - order: 0 - client_id: - title: "Client ID" - type: "string" - description: "The Client ID of your developer application" - airbyte_secret: true - client_secret: - title: "Client Secret" - type: "string" - description: "The client secret of your developer application" - airbyte_secret: true - refresh_token: - title: "Refresh Token" - type: "string" - description: "A refresh token generated using the above client ID\ - \ and secret" - airbyte_secret: true - access_token: - title: "Access Token" - type: "string" - description: "A access token generated using the above client ID,\ - \ secret and refresh_token" - airbyte_secret: true - - type: "object" - title: "Service Account Key Authentication" - required: - - "credentials_json" - properties: - auth_type: - type: "string" - const: "Service" - enum: - - "Service" - default: "Service" - order: 0 - credentials_json: - type: "string" - description: "The JSON key of the service account to use for authorization" - examples: - - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\ - \ \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "0" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "access_token" - - - "refresh_token" -- dockerImage: "airbyte/source-google-directory:0.1.5" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/google-directory" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Directory Spec" - type: "object" - required: - - "credentials_json" - - "email" - additionalProperties: false - properties: - credentials_json: - type: "string" - description: "The contents of the JSON service account key. See the docs for more information on how to generate this key." - airbyte_secret: true - email: - type: "string" - description: "The email of the user, which has permissions to access the\ - \ Google Workspace Admin APIs." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-google-search-console:0.1.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/google-search-console" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Search Console Spec" - type: "object" - additionalProperties: false - required: - - "site_urls" - - "start_date" - - "authorization" - properties: - site_urls: - type: "array" - items: - type: "string" - description: "Website URLs property; do not include the domain-level property\ - \ in the list" - examples: - - "https://example1.com" - - "https://example2.com" - start_date: - type: "string" - description: "The date from which you'd like to replicate data in the format\ - \ YYYY-MM-DD." - examples: - - "2021-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - end_date: - type: "string" - description: "The date from which you'd like to replicate data in the format\ - \ YYYY-MM-DD. Must be greater or equal start_date field" - examples: - - "2021-12-12" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - authorization: - type: "object" - title: "Authentication Type" - oneOf: - - title: "Authenticate via Google (Oauth)" - type: "object" - required: - - "auth_type" - - "client_id" - - "client_secret" - - "refresh_token" - properties: - auth_type: - type: "string" - const: "Client" - enum: - - "Client" - default: "Client" - order: 0 - client_id: - title: "Client ID" - type: "string" - description: "The Client ID of your developer application" - airbyte_secret: true - client_secret: - title: "Client Secret" - type: "string" - description: "The client secret of your developer application" - airbyte_secret: true - access_token: - title: "Access Token" - type: "string" - description: "An access token generated using the above client ID\ - \ and secret" - airbyte_secret: true - refresh_token: - title: "Refresh Token" - type: "string" - description: "A refresh token generated using the above client ID\ - \ and secret" - airbyte_secret: true - - type: "object" - title: "Service Account Key Authentication" - required: - - "auth_type" - - "service_account_info" - - "email" - properties: - auth_type: - type: "string" - const: "Service" - enum: - - "Service" - default: "Service" - order: 0 - service_account_info: - title: "Service Account JSON Key" - type: "string" - description: "The JSON key of the service account to use for authorization" - examples: - - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\ - \ \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - email: - title: "Admin Email" - type: "string" - description: "The email of the user which has permissions to access\ - \ the Google Workspace Admin APIs." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "authorization" - - "0" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "access_token" - - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Stripe Source Spec" - type: "object" - required: - - "spreadsheet_id" - additionalProperties: true - properties: - spreadsheet_id: - type: "string" - description: "The ID of the spreadsheet to be replicated." - credentials: - type: "object" - oneOf: - - title: "Authenticate via Google (Oauth)" - type: "object" - required: - - "auth_type" - - "client_id" - - "client_secret" - - "refresh_token" - properties: - auth_type: - type: "string" - const: "Client" - client_id: - title: "Client ID" - type: "string" - description: "The Client ID of your developer application" - airbyte_secret: true - client_secret: - title: "Client Secret" - type: "string" - description: "The client secret of your developer application" - airbyte_secret: true - refresh_token: - title: "Refresh Token" - type: "string" - description: "A refresh token generated using the above client ID\ - \ and secret" - airbyte_secret: true - - title: "Service Account Key Authentication" - type: "object" - required: - - "auth_type" - - "service_account_info" - properties: - auth_type: - type: "string" - const: "Service" - service_account_info: - type: "string" - description: "The JSON key of the service account to use for authorization" - examples: - - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\ - \ \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - 0 - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "refresh_token" -- dockerImage: "airbyte/source-google-workspace-admin-reports:0.1.5" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Directory Spec" - type: "object" - required: - - "credentials_json" - - "email" - additionalProperties: false - properties: - credentials_json: - type: "string" - description: "The contents of the JSON service account key. See the docs for more information on how to generate this key." - airbyte_secret: true - email: - type: "string" - description: "The email of the user, which has permissions to access the\ - \ Google Workspace Admin APIs." - lookback: - type: "integer" - minimum: 0 - maximum: 180 - description: "Sets the range of time shown in the report. Reports API allows\ - \ from up to 180 days ago. " - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-greenhouse:0.2.5" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/greenhouse" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Greenhouse Spec" - type: "object" - required: - - "api_key" - additionalProperties: false - properties: - api_key: - type: "string" - description: "Greenhouse API Key. See the docs for more information on how to generate this key." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-harvest:0.1.5" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/harvest" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Harvest Spec" - type: "object" - required: - - "api_token" - - "account_id" - - "replication_start_date" - additionalProperties: false - properties: - api_token: - title: "API Token" - description: "Harvest API Token." - airbyte_secret: true - type: "string" - account_id: - title: "Account ID" - description: "Harvest account ID. Required for all Harvest requests in pair\ - \ with API Key" - airbyte_secret: true - type: "string" - replication_start_date: - title: "Replication Start Date" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-25T00:00:00Z" - type: "string" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" -- dockerImage: "airbyte/source-hubspot:0.1.21" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Hubspot Source Spec" - type: "object" - required: - - "start_date" - - "credentials" - additionalProperties: false - properties: - start_date: - type: "string" - title: "Replication start date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2017-01-25T00:00:00Z" - credentials: - title: "Authentication mechanism" - description: "Choose either to provide the API key or the OAuth2.0 credentials" - type: "object" - oneOf: - - type: "object" - title: "Authenticate via Hubspot (Oauth)" - required: - - "redirect_uri" - - "client_id" - - "client_secret" - - "refresh_token" - - "access_token" - - "credentials_title" - properties: - credentials_title: - type: "string" - title: "Credentials title" - description: "Name of the credentials set" - const: "OAuth Credentials" - enum: - - "OAuth Credentials" - default: "OAuth Credentials" - order: 0 - client_id: - title: "Client ID" - description: "Hubspot client_id. See our docs if you need help finding this id." - type: "string" - examples: - - "123456789000" - client_secret: - title: "Client Secret" - description: "Hubspot client_secret. See our docs if you need help finding this secret." - type: "string" - examples: - - "secret" - airbyte_secret: true - refresh_token: - title: "Refresh token" - description: "Hubspot refresh_token. See our docs if you need help generating the token." - type: "string" - examples: - - "refresh_token" - airbyte_secret: true - - type: "object" - title: "API key" - required: - - "api_key" - - "credentials_title" - properties: - credentials_title: - type: "string" - title: "Credentials title" - description: "Name of the credentials set" - const: "API Key Credentials" - enum: - - "API Key Credentials" - default: "API Key Credentials" - order: 0 - api_key: - title: "API key" - description: "Hubspot API Key. See our docs if you need help finding this key." - type: "string" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "0" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - - - "refresh_token" - oauthFlowOutputParameters: - - - "refresh_token" -- dockerImage: "airbyte/source-db2:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/db2" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "IBM Db2 Source Spec" - type: "object" - required: - - "host" - - "port" - - "db" - - "username" - - "password" - additionalProperties: false - properties: - host: - description: "Host of the Db2." - type: "string" - port: - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 8123 - examples: - - "8123" - db: - description: "Name of the database." - type: "string" - examples: - - "default" - username: - description: "Username to use to access the database." - type: "string" - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-instagram:0.1.9" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/instagram" - changelogUrl: "https://docs.airbyte.io/integrations/sources/instagram" - connectionSpecification: - title: "Source Instagram" - type: "object" - properties: - start_date: - title: "Start Date" - description: "The date from which you'd like to replicate data for User\ - \ Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after\ - \ this date will be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-25T00:00:00Z" - type: "string" - format: "date-time" - access_token: - title: "Access Token" - description: "The value of the access token generated. See the docs for\ - \ more information" - airbyte_secret: true - type: "string" - required: - - "start_date" - - "access_token" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: [] - oauthFlowInitParameters: [] - oauthFlowOutputParameters: - - - "access_token" -- dockerImage: "airbyte/source-intercom:0.1.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/intercom" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Intercom Spec" - type: "object" - required: - - "access_token" - - "start_date" - additionalProperties: false - properties: - access_token: - type: "string" - description: "Intercom Access Token. See the docs for more information on how to obtain this key." - airbyte_secret: true - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Intercom\ - \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\ - \ date will be replicated." - examples: - - "2020-11-16T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-iterable:0.1.9" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/iterable" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Iterable Spec" - type: "object" - required: - - "start_date" - - "api_key" - additionalProperties: false - properties: - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Iterable,\ - \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\ - \ will be replicated." - examples: - - "2021-04-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - api_key: - type: "string" - description: "Iterable API Key. See the docs for more information on how to obtain this key." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-jira:0.2.14" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/jira" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Jira Spec" - type: "object" - required: - - "api_token" - - "domain" - - "email" - additionalProperties: true - properties: - api_token: - type: "string" - description: "Jira API Token. See the docs for more information on how to generate this key." - airbyte_secret: true - domain: - type: "string" - examples: - - "domainname.atlassian.net" - pattern: "^[a-zA-Z0-9._-]*\\.atlassian\\.net$" - description: "Domain for your Jira account, e.g. airbyteio.atlassian.net" - email: - type: "string" - description: "The user email for your Jira account" - projects: - type: "array" - title: "Projects" - items: - type: "string" - examples: - - "PROJ1" - - "PROJ2" - description: "Comma-separated list of Jira project keys to replicate data\ - \ for" - start_date: - type: "string" - title: "Start Date" - description: "The date from which you'd like to replicate data for Jira\ - \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\ - \ will be replicated. Note that it will be used only in the following\ - \ incremental streams: issues." - examples: - - "2021-03-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - additional_fields: - type: "array" - title: "Additional Fields" - items: - type: "string" - description: "Comma-separated list of additional fields to include in replicating\ - \ issues" - examples: - - "Field A" - - "Field B" - expand_issue_changelog: - type: "boolean" - title: "Expand Issue Changelog" - description: "Expand the changelog when replicating issues" - default: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-kafka:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/kafka" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Kafka Source Spec" - type: "object" - required: - - "bootstrap_servers" - - "subscription" - - "protocol" - additionalProperties: false - properties: - bootstrap_servers: - title: "Bootstrap servers" - description: "A list of host/port pairs to use for establishing the initial\ - \ connection to the Kafka cluster. The client will make use of all servers\ - \ irrespective of which servers are specified here for bootstrapping—this\ - \ list only impacts the initial hosts used to discover the full set of\ - \ servers. This list should be in the form host1:port1,host2:port2,....\ - \ Since these servers are just used for the initial connection to discover\ - \ the full cluster membership (which may change dynamically), this list\ - \ need not contain the full set of servers (you may want more than one,\ - \ though, in case a server is down)." - type: "string" - examples: - - "kafka-broker1:9092,kafka-broker2:9092" - subscription: - title: "Subscribe method" - type: "object" - description: "You can choose to manually assign a list of partitions, or\ - \ subscribe to all topics matching specified pattern to get dynamically\ - \ assigned partitions" - oneOf: - - title: "Manually assign a list of partitions" - required: - - "subscription_type" - - "topic_partitions" - properties: - subscription_type: - description: "Manually assign a list of partitions to this consumer.\ - \ This interface does not allow for incremental assignment and will\ - \ replace the previous assignment (if there is one).\nIf the given\ - \ list of topic partitions is empty, it is treated the same as unsubscribe()." - type: "string" - const: "assign" - enum: - - "assign" - default: "assign" - topic_partitions: - title: "List of topic:partition pairs" - type: "string" - examples: - - "sample.topic:0, sample.topic:1" - - title: "Subscribe to all topics matching specified pattern" - required: - - "subscription_type" - - "topic_pattern" - properties: - subscription_type: - description: "Topic pattern from which the records will be read." - type: "string" - const: "subscribe" - enum: - - "subscribe" - default: "subscribe" - topic_pattern: - title: "Topic pattern" - type: "string" - examples: - - "sample.topic" - test_topic: - title: "Test topic" - description: "Topic to test if Airbyte can consume messages." - type: "string" - examples: - - "test.topic" - group_id: - title: "Group ID" - description: "Group id." - type: "string" - examples: - - "group.id" - max_poll_records: - title: "Max poll records" - description: "The maximum number of records returned in a single call to\ - \ poll(). Note, that max_poll_records does not impact the underlying fetching\ - \ behavior. The consumer will cache the records from each fetch request\ - \ and returns them incrementally from each poll." - type: "integer" - default: 500 - protocol: - title: "Protocol" - type: "object" - description: "Protocol used to communicate with brokers." - oneOf: - - title: "PLAINTEXT" - required: - - "security_protocol" - properties: - security_protocol: - type: "string" - enum: - - "PLAINTEXT" - default: "PLAINTEXT" - - title: "SASL PLAINTEXT" - required: - - "security_protocol" - - "sasl_mechanism" - - "sasl_jaas_config" - properties: - security_protocol: - type: "string" - enum: - - "SASL_PLAINTEXT" - default: "SASL_PLAINTEXT" - sasl_mechanism: - title: "SASL mechanism" - description: "SASL mechanism used for client connections. This may\ - \ be any mechanism for which a security provider is available." - type: "string" - default: "PLAIN" - enum: - - "PLAIN" - sasl_jaas_config: - title: "SASL JAAS config" - description: "JAAS login context parameters for SASL connections in\ - \ the format used by JAAS configuration files." - type: "string" - default: "" - airbyte_secret: true - - title: "SASL SSL" - required: - - "security_protocol" - - "sasl_mechanism" - - "sasl_jaas_config" - properties: - security_protocol: - type: "string" - enum: - - "SASL_SSL" - default: "SASL_SSL" - sasl_mechanism: - title: "SASL mechanism" - description: "SASL mechanism used for client connections. This may\ - \ be any mechanism for which a security provider is available." - type: "string" - default: "GSSAPI" - enum: - - "GSSAPI" - - "OAUTHBEARER" - - "SCRAM-SHA-256" - sasl_jaas_config: - title: "SASL JAAS config" - description: "JAAS login context parameters for SASL connections in\ - \ the format used by JAAS configuration files." - type: "string" - default: "" - airbyte_secret: true - client_id: - title: "Client ID" - description: "An id string to pass to the server when making requests. The\ - \ purpose of this is to be able to track the source of requests beyond\ - \ just ip/port by allowing a logical application name to be included in\ - \ server-side request logging." - type: "string" - examples: - - "airbyte-consumer" - enable_auto_commit: - title: "Enable auto commit" - description: "If true the consumer's offset will be periodically committed\ - \ in the background." - type: "boolean" - default: true - auto_commit_interval_ms: - title: "Auto commit interval ms" - description: "The frequency in milliseconds that the consumer offsets are\ - \ auto-committed to Kafka if enable.auto.commit is set to true." - type: "integer" - default: 5000 - client_dns_lookup: - title: "Client DNS lookup" - description: "Controls how the client uses DNS lookups. If set to use_all_dns_ips,\ - \ connect to each returned IP address in sequence until a successful connection\ - \ is established. After a disconnection, the next IP is used. Once all\ - \ IPs have been used once, the client resolves the IP(s) from the hostname\ - \ again. If set to resolve_canonical_bootstrap_servers_only, resolve each\ - \ bootstrap address into a list of canonical names. After the bootstrap\ - \ phase, this behaves the same as use_all_dns_ips. If set to default (deprecated),\ - \ attempt to connect to the first IP address returned by the lookup, even\ - \ if the lookup returns multiple IP addresses." - type: "string" - default: "use_all_dns_ips" - enum: - - "default" - - "use_all_dns_ips" - - "resolve_canonical_bootstrap_servers_only" - retry_backoff_ms: - title: "Retry backoff ms" - description: "The amount of time to wait before attempting to retry a failed\ - \ request to a given topic partition. This avoids repeatedly sending requests\ - \ in a tight loop under some failure scenarios." - type: "integer" - default: 100 - request_timeout_ms: - title: "Request timeout ms" - description: "The configuration controls the maximum amount of time the\ - \ client will wait for the response of a request. If the response is not\ - \ received before the timeout elapses the client will resend the request\ - \ if necessary or fail the request if retries are exhausted." - type: "integer" - default: 30000 - receive_buffer_bytes: - title: "Receive buffer bytes" - description: "The size of the TCP receive buffer (SO_RCVBUF) to use when\ - \ reading data. If the value is -1, the OS default will be used." - type: "integer" - default: 32768 - auto_offset_reset: - title: "Auto offset reset" - description: "What to do when there is no initial offset in Kafka or if\ - \ the current offset does not exist any more on the server - earliest:\ - \ automatically reset the offset to the earliest offset, latest: automatically\ - \ reset the offset to the latest offset, none: throw exception to the\ - \ consumer if no previous offset is found for the consumer's group, anything\ - \ else: throw exception to the consumer." - type: "string" - default: "latest" - enum: - - "latest" - - "earliest" - - "none" - repeated_calls: - title: "Repeated calls" - description: "The number of repeated calls to poll() if no messages were\ - \ received." - type: "integer" - default: 3 - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - supported_source_sync_modes: - - "append" -- dockerImage: "airbyte/source-klaviyo:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/klaviyo" - changelogUrl: "https://docs.airbyte.io/integrations/sources/klaviyo" - connectionSpecification: - title: "Klaviyo Spec" - type: "object" - properties: - api_key: - title: "Api Key" - description: "Klaviyo API Key. See our docs if you need help finding this key." - airbyte_secret: true - type: "string" - start_date: - title: "Start Date" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-25T00:00:00Z" - type: "string" - required: - - "api_key" - - "start_date" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" -- dockerImage: "airbyte/source-lever-hiring:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/lever-hiring" - changelogUrl: "https://docs.airbyte.io/integrations/sources/lever-hiring#changelog" - connectionSpecification: - title: "Lever Hiring Spec" - type: "object" - properties: - client_id: - title: "Client Id" - description: "The client application id as provided when registering the\ - \ application with Lever." - type: "string" - client_secret: - title: "Client Secret" - description: "The application secret as provided when registering the application\ - \ with Lever." - airbyte_secret: true - type: "string" - refresh_token: - title: "Refresh Token" - description: "The refresh token your application will need to submit to\ - \ get a new access token after it's expired." - type: "string" - environment: - title: "Environment" - description: "Sandbox or Production environment." - default: "Production" - enum: - - "Sandbox" - - "Production" - type: "string" - start_date: - title: "Start Date" - description: "UTC date and time in the format 2019-02-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-04-25T00:00:00Z" - type: "string" - required: - - "client_id" - - "client_secret" - - "refresh_token" - - "start_date" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: [] - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - - - "refresh_token" - oauthFlowOutputParameters: [] -- dockerImage: "airbyte/source-linkedin-ads:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/linkedin-ads" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Linkedin Ads Spec" - type: "object" - required: - - "start_date" - - "access_token" - additionalProperties: false - properties: - start_date: - type: "string" - title: "Start Date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - description: "Date in the format 2020-09-17. Any data before this date will\ - \ not be replicated." - examples: - - "2021-05-17" - access_token: - type: "string" - title: "Access Token" - description: "The token value ganerated using Auth Code" - airbyte_secret: true - account_ids: - title: "Account IDs" - type: "array" - description: "Specify the Account IDs separated by space, from which to\ - \ pull the data. Leave empty to pull from all associated accounts." - items: - type: "integer" - default: [] - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-looker:0.2.5" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/looker" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Looker Spec" - type: "object" - required: - - "domain" - - "client_id" - - "client_secret" - additionalProperties: false - properties: - domain: - type: "string" - examples: - - "domainname.looker.com" - - "looker.clientname.com" - - "123.123.124.123:8000" - description: "Domain for your Looker account, e.g. airbyte.cloud.looker.com,looker.[clientname].com,IP\ - \ address" - client_id: - title: "Client ID" - type: "string" - description: "The Client ID is first part of an API3 key that is specific\ - \ to each Looker user. See the docs for more information on how to generate this key." - client_secret: - title: "Client Secret" - type: "string" - description: "The Client Secret is second part of an API3 key." - run_look_ids: - title: "Look IDs to Run" - type: "array" - items: - type: "string" - pattern: "^[0-9]*$" - description: "The IDs of any Looks to run (optional)" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mailchimp:0.2.8" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/mailchimp" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Mailchimp Spec" - type: "object" - required: - - "username" - - "apikey" - additionalProperties: false - properties: - username: - type: "string" - description: "The Username or email you use to sign into Mailchimp" - apikey: - type: "string" - airbyte_secret: true - description: "API Key. See the docs for information on how to generate this key." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-marketo:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Marketo Spec" - type: "object" - required: - - "domain_url" - - "client_id" - - "client_secret" - - "start_date" - additionalProperties: false - properties: - domain_url: - type: "string" - description: "Your Marketo Base URL. See the docs for info on how to obtain this." - examples: - - "https://000-AAA-000.mktorest.com" - airbyte_secret: true - client_id: - type: "string" - description: "Your Marketo client_id. See the docs for info on how to obtain this." - airbyte_secret: true - client_secret: - type: "string" - description: "Your Marketo client secret. See the docs for info on how to obtain this." - airbyte_secret: true - start_date: - type: "string" - description: "Data generated in Marketo after this date will be replicated.\ - \ This date must be specified in the format YYYY-MM-DDT00:00:00Z." - examples: - - "2020-09-25T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - window_in_days: - type: "integer" - description: "The amount of days for each data-chunk begining from start_date.\ - \ (Min=1, as for a Day; Max=30, as for a Month)." - examples: - - 1 - - 5 - - 10 - - 15 - - 30 - default: 30 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.3.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MSSQL Source Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - additionalProperties: false - properties: - host: - description: "Hostname of the database." - type: "string" - port: - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - examples: - - "1433" - database: - description: "Name of the database." - type: "string" - examples: - - "master" - username: - description: "Username to use to access the database." - type: "string" - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - ssl_method: - title: "SSL Method" - type: "object" - description: "Encryption method to use when communicating with the database" - order: 6 - oneOf: - - title: "Unencrypted" - additionalProperties: false - description: "Data transfer will not be encrypted." - required: - - "ssl_method" - properties: - ssl_method: - type: "string" - const: "unencrypted" - enum: - - "unencrypted" - default: "unencrypted" - - title: "Encrypted (trust server certificate)" - additionalProperties: false - description: "Use the cert provided by the server without verification.\ - \ (For testing purposes only!)" - required: - - "ssl_method" - properties: - ssl_method: - type: "string" - const: "encrypted_trust_server_certificate" - enum: - - "encrypted_trust_server_certificate" - default: "encrypted_trust_server_certificate" - - title: "Encrypted (verify certificate)" - additionalProperties: false - description: "Verify and use the cert provided by the server." - required: - - "ssl_method" - - "trustStoreName" - - "trustStorePassword" - properties: - ssl_method: - type: "string" - const: "encrypted_verify_certificate" - enum: - - "encrypted_verify_certificate" - default: "encrypted_verify_certificate" - hostNameInCertificate: - title: "Host Name In Certificate" - type: "string" - description: "Specifies the host name of the server. The value of\ - \ this property must match the subject property of the certificate." - order: 7 - replication_method: - type: "string" - title: "Replication Method" - description: "Replication method to use for extracting data from the database.\ - \ STANDARD replication requires no setup on the DB side but will not be\ - \ able to represent deletions incrementally. CDC uses {TBC} to detect\ - \ inserts, updates, and deletes. This needs to be configured on the source\ - \ database itself." - default: "STANDARD" - enum: - - "STANDARD" - - "CDC" - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials for logging\ - \ into the jump server host." - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-microsoft-teams:0.2.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/microsoft-teams" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Microsoft Teams Spec" - type: "object" - required: - - "tenant_id" - - "client_id" - - "client_secret" - - "period" - additionalProperties: false - properties: - tenant_id: - title: "Directory (tenant) ID" - type: "string" - description: "Directory (tenant) ID" - client_id: - title: "Application (client) ID" - type: "string" - description: "Application (client) ID" - client_secret: - title: "Client Secret" - type: "string" - description: "Client secret" - airbyte_secret: true - period: - type: "string" - description: "Specifies the length of time over which the Team Device Report\ - \ stream is aggregated. The supported values are: D7, D30, D90, and D180." - examples: - - "D7" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mixpanel:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Mixpanel Spec" - type: "object" - required: - - "api_secret" - additionalProperties: true - properties: - api_secret: - type: "string" - description: "Mixpanel API Secret. See the docs for more information on how to obtain this key." - airbyte_secret: true - attribution_window: - type: "integer" - description: "Latency minimum number of days to look-back to account for\ - \ delays in attributing accurate results. Default attribution window is\ - \ 5 days." - default: 5 - date_window_size: - type: "integer" - description: "Number of days for date window looping through transactional\ - \ endpoints with from_date and to_date. Default date_window_size is 30\ - \ days. Clients with large volumes of events may want to decrease this\ - \ to 14, 7, or even down to 1-2 days." - default: 30 - project_timezone: - type: "string" - description: "Time zone in which integer date times are stored. The project\ - \ timezone may be found in the project settings in the Mixpanel console." - default: "US/Pacific" - examples: - - "US/Pacific" - - "UTC" - select_properties_by_default: - type: "boolean" - description: "Setting this config parameter to true ensures that new properties\ - \ on events and engage records are captured. Otherwise new properties\ - \ will be ignored" - default: true - start_date: - type: "string" - description: "The default value to use if no bookmark exists for an endpoint.\ - \ Default is 1 year ago." - examples: - - "2021-11-16" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" - region: - type: "string" - enum: - - "US" - - "EU" - default: "US" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mongodb-v2:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2" - changelogUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MongoDb Source Spec" - type: "object" - required: - - "database" - additionalProperties: true - properties: - instance_type: - type: "object" - title: "MongoDb instance type" - description: "MongoDb instance to connect to. For MongoDB Atlas and Replica\ - \ Set TLS connection is used by default." - order: 0 - oneOf: - - title: "Standalone MongoDb Instance" - required: - - "instance" - - "host" - - "port" - properties: - instance: - type: "string" - enum: - - "standalone" - default: "standalone" - host: - title: "Host" - type: "string" - description: "Host of a Mongo database to be replicated." - order: 0 - port: - title: "Port" - type: "integer" - description: "Port of a Mongo database to be replicated." - minimum: 0 - maximum: 65536 - default: 27017 - examples: - - "27017" - order: 1 - tls: - title: "TLS connection" - type: "boolean" - description: "Indicates whether TLS encryption protocol will be used\ - \ to connect to MongoDB. It is recommended to use TLS connection\ - \ if possible. For more information see documentation." - default: false - order: 2 - - title: "Replica Set" - required: - - "instance" - - "server_addresses" - properties: - instance: - type: "string" - enum: - - "replica" - default: "replica" - server_addresses: - title: "Server addresses" - type: "string" - description: "The members of a replica set. Please specify `host`:`port`\ - \ of each member seperated by comma." - examples: - - "host1:27017,host2:27017,host3:27017" - order: 0 - replica_set: - title: "Replica Set" - type: "string" - description: "A replica set name." - order: 1 - - title: "MongoDB Atlas" - additionalProperties: false - required: - - "instance" - - "cluster_url" - properties: - instance: - type: "string" - enum: - - "atlas" - default: "atlas" - cluster_url: - title: "Cluster URL" - type: "string" - description: "URL of a cluster to connect to." - order: 0 - database: - title: "Database name" - type: "string" - description: "Database to be replicated." - order: 1 - user: - title: "User" - type: "string" - description: "User" - order: 2 - password: - title: "Password" - type: "string" - description: "Password" - airbyte_secret: true - order: 3 - auth_source: - title: "Authentication source" - type: "string" - description: "Authentication source where user information is stored" - default: "admin" - examples: - - "admin" - order: 4 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.4.8" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "MySql Source Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - - "replication_method" - additionalProperties: false - properties: - host: - description: "Hostname of the database." - type: "string" - order: 0 - port: - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 3306 - examples: - - "3306" - order: 1 - database: - description: "Name of the database." - type: "string" - order: 2 - username: - description: "Username to use to access the database." - type: "string" - order: 3 - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 4 - jdbc_url_params: - description: "Additional properties to pass to the jdbc url string when\ - \ connecting to the database formatted as 'key=value' pairs separated\ - \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)" - type: "string" - order: 5 - ssl: - title: "SSL Connection" - description: "Encrypt data using SSL." - type: "boolean" - default: true - order: 7 - replication_method: - type: "string" - title: "Replication Method" - description: "Replication method to use for extracting data from the database.\ - \ STANDARD replication requires no setup on the DB side but will not be\ - \ able to represent deletions incrementally. CDC uses the Binlog to detect\ - \ inserts, updates, and deletes. This needs to be configured on the source\ - \ database itself." - order: 6 - default: "STANDARD" - enum: - - "STANDARD" - - "CDC" - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials in RSA PEM\ - \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-okta:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/okta" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Okta Spec" - type: "object" - required: - - "token" - - "base_url" - additionalProperties: false - properties: - token: - type: "string" - title: "API Token" - description: "A Okta token. See the docs for instructions on how to generate it." - airbyte_secret: true - base_url: - type: "string" - title: "Base URL" - description: "The Okta base URL." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-onesignal:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/onesignal" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "OneSignal Source Spec" - type: "object" - required: - - "user_auth_key" - - "start_date" - - "outcome_names" - additionalProperties: false - properties: - user_auth_key: - type: "string" - description: "OneSignal User Auth Key, see the docs for more information on how to obtain this key." - airbyte_secret: true - start_date: - type: "string" - description: "The date from which you'd like to replicate data for OneSignal\ - \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\ - \ date will be replicated." - examples: - - "2020-11-16T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - outcome_names: - type: "string" - description: "Comma-separated list of names and the value (sum/count) for\ - \ the returned outcome data. See the docs for more details" - examples: - - "os__session_duration.count,os__click.count,CustomOutcomeName.sum" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-oracle:0.3.8" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/oracle" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Oracle Source Spec" - type: "object" - required: - - "host" - - "port" - - "sid" - - "username" - additionalProperties: false - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - port: - title: "Port" - description: "Port of the database.\nOracle Corporations recommends the\ - \ following port numbers:\n1521 - Default listening port for client connections\ - \ to the listener. \n2484 - Recommended and officially registered listening\ - \ port for client connections to the listener using TCP/IP with SSL" - type: "integer" - minimum: 0 - maximum: 65536 - default: 1521 - sid: - title: "SID (Oracle System Identifier)" - type: "string" - username: - title: "User" - description: "Username to use to access the database." - type: "string" - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - schemas: - title: "Schemas" - description: "List of schemas to sync from. Defaults to user. Case sensitive." - type: "array" - items: - type: "string" - minItems: 1 - uniqueItems: true - encryption: - title: "Encryption" - type: "object" - description: "Encryption method to use when communicating with the database" - order: 6 - oneOf: - - title: "Unencrypted" - additionalProperties: false - description: "Data transfer will not be encrypted." - required: - - "encryption_method" - properties: - encryption_method: - type: "string" - const: "unencrypted" - enum: - - "unencrypted" - default: "unencrypted" - - title: "Native Network Ecryption (NNE)" - additionalProperties: false - description: "Native network encryption gives you the ability to encrypt\ - \ database connections, without the configuration overhead of TCP/IP\ - \ and SSL/TLS and without the need to open and listen on different ports." - required: - - "encryption_method" - properties: - encryption_method: - type: "string" - const: "client_nne" - enum: - - "client_nne" - default: "client_nne" - encryption_algorithm: - type: "string" - description: "This parameter defines the encryption algorithm to be\ - \ used" - title: "Encryption Algorithm" - default: "AES256" - enum: - - "AES256" - - "RC4_56" - - "3DES168" - - title: "TLS Encrypted (verify certificate)" - additionalProperties: false - description: "Verify and use the cert provided by the server." - required: - - "encryption_method" - - "ssl_certificate" - properties: - encryption_method: - type: "string" - const: "encrypted_verify_certificate" - enum: - - "encrypted_verify_certificate" - default: "encrypted_verify_certificate" - ssl_certificate: - title: "SSL PEM file" - description: "Privacy Enhanced Mail (PEM) files are concatenated certificate\ - \ containers frequently used in certificate installations" - type: "string" - airbyte_secret: true - multiline: true - order: 4 - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials in RSA PEM\ - \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-paypal-transaction:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/paypal-transactions" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Paypal Transaction Search" - type: "object" - required: - - "client_id" - - "secret" - - "start_date" - - "is_sandbox" - additionalProperties: true - properties: - client_id: - title: "Client ID" - type: "string" - description: "The Paypal Client ID for API credentials" - secret: - title: "Secret" - type: "string" - description: "The Secret for a given Client ID." - airbyte_secret: true - start_date: - type: "string" - title: "Start Date" - description: "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before\ - \ present time" - examples: - - "2021-06-11T23:59:59-00:00" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" - is_sandbox: - title: "Is Sandbox" - description: "Whether or not to Sandbox or Production environment to extract\ - \ data from" - type: "boolean" - default: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-pipedrive:0.1.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/pipedrive" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Pipedrive Spec" - type: "object" - required: - - "replication_start_date" - additionalProperties: true - properties: - authorization: - type: "object" - title: "Authentication Type" - oneOf: - - title: "Sign in via Pipedrive (OAuth)" - type: "object" - required: - - "auth_type" - - "client_id" - - "client_secret" - - "refresh_token" - properties: - auth_type: - type: "string" - const: "Client" - enum: - - "Client" - default: "Client" - order: 0 - client_id: - title: "Client ID" - type: "string" - description: "The Client ID of your developer application" - airbyte_secret: true - client_secret: - title: "Client Secret" - type: "string" - description: "The client secret of your developer application" - airbyte_secret: true - access_token: - title: "Access Token" - type: "string" - description: "An access token generated using the above client ID\ - \ and secret" - airbyte_secret: true - refresh_token: - title: "Refresh Token" - type: "string" - description: "A refresh token generated using the above client ID\ - \ and secret" - airbyte_secret: true - - type: "object" - title: "API Key Authentication" - required: - - "auth_type" - - "api_token" - properties: - auth_type: - type: "string" - const: "Token" - enum: - - "Token" - default: "Token" - order: 0 - api_token: - title: "API Token" - type: "string" - description: "Pipedrive API Token" - airbyte_secret: true - replication_start_date: - title: "Replication Start Date" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated. When specified and not\ - \ None, then stream will behave as incremental" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2017-01-25T00:00:00Z" - type: "string" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" -- dockerImage: "airbyte/source-plaid:0.2.1" - spec: - documentationUrl: "https://plaid.com/docs/api/" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - type: "object" - required: - - "access_token" - - "api_key" - - "client_id" - additionalProperties: false - properties: - access_token: - type: "string" - title: "Access Token" - description: "The end-user's Link access token." - api_key: - title: "API Key" - type: "string" - description: "The Plaid API key to use to hit the API." - airbyte_secret: true - client_id: - title: "Client ID" - type: "string" - description: "The Plaid client id" - plaid_env: - title: "Plaid Environment" - type: "string" - enum: - - "sandbox" - - "development" - - "production" - description: "The Plaid environment" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-pokeapi:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/pokeapi" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Pokeapi Spec" - type: "object" - required: - - "pokemon_name" - additionalProperties: false - properties: - pokemon_name: - type: "string" - description: "Pokemon requested from the API." - pattern: "^[a-z0-9_\\-]+$" - examples: - - "ditto, luxray, snorlax" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-posthog:0.1.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/posthog" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "PostHog Spec" - type: "object" - required: - - "api_key" - - "start_date" - additionalProperties: false - properties: - start_date: - title: "Start Date" - type: "string" - description: "The date from which you'd like to replicate the data" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-01-01T00:00:00Z" - api_key: - type: "string" - airbyte_secret: true - description: "API Key. See the docs for information on how to generate this key." - base_url: - type: "string" - default: "https://app.posthog.com" - description: "Base PostHog url. Defaults to PostHog Cloud (https://app.posthog.com)." - examples: - - "https://posthog.example.com" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.3.11" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/postgres" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Postgres Source Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - additionalProperties: false - properties: - host: - title: "Host" - description: "Hostname of the database." - type: "string" - order: 0 - port: - title: "Port" - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 5432 - examples: - - "5432" - order: 1 - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 3 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 4 - ssl: - title: "Connect using SSL" - description: "Encrypt client/server communications for increased security." - type: "boolean" - default: false - order: 5 - replication_method: - type: "object" - title: "Replication Method" - description: "Replication method to use for extracting data from the database." - order: 6 - oneOf: - - title: "Standard" - additionalProperties: false - description: "Standard replication requires no setup on the DB side but\ - \ will not be able to represent deletions incrementally." - required: - - "method" - properties: - method: - type: "string" - const: "Standard" - enum: - - "Standard" - default: "Standard" - order: 0 - - title: "Logical Replication (CDC)" - additionalProperties: false - description: "Logical replication uses the Postgres write-ahead log (WAL)\ - \ to detect inserts, updates, and deletes. This needs to be configured\ - \ on the source database itself. Only available on Postgres 10 and above.\ - \ Read the Postgres Source docs for more information." - required: - - "method" - - "replication_slot" - - "publication" - properties: - method: - type: "string" - const: "CDC" - enum: - - "CDC" - default: "CDC" - order: 0 - plugin: - type: "string" - description: "A logical decoding plug-in installed on the PostgreSQL\ - \ server. `pgoutput` plug-in is used by default.\nIf replication\ - \ table contains a lot of big jsonb values it is recommended to\ - \ use `wal2json` plug-in. For more information about `wal2json`\ - \ plug-in read Postgres Source docs." - enum: - - "pgoutput" - - "wal2json" - default: "pgoutput" - order: 1 - replication_slot: - type: "string" - description: "A plug-in logical replication slot." - order: 2 - publication: - type: "string" - description: "A Postgres publication used for consuming changes." - order: 3 - tunnel_method: - type: "object" - title: "SSH Tunnel Method" - description: "Whether to initiate an SSH tunnel before connecting to the\ - \ database, and if so, which kind of authentication to use." - oneOf: - - title: "No Tunnel" - required: - - "tunnel_method" - properties: - tunnel_method: - description: "No ssh tunnel needed to connect to database" - type: "string" - const: "NO_TUNNEL" - order: 0 - - title: "SSH Key Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "ssh_key" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and ssh key" - type: "string" - const: "SSH_KEY_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host." - type: "string" - order: 3 - ssh_key: - title: "SSH Private Key" - description: "OS-level user account ssh key credentials for logging\ - \ into the jump server host." - type: "string" - airbyte_secret: true - multiline: true - order: 4 - - title: "Password Authentication" - required: - - "tunnel_method" - - "tunnel_host" - - "tunnel_port" - - "tunnel_user" - - "tunnel_user_password" - properties: - tunnel_method: - description: "Connect through a jump server tunnel host using username\ - \ and password authentication" - type: "string" - const: "SSH_PASSWORD_AUTH" - order: 0 - tunnel_host: - title: "SSH Tunnel Jump Server Host" - description: "Hostname of the jump server host that allows inbound\ - \ ssh tunnel." - type: "string" - order: 1 - tunnel_port: - title: "SSH Connection Port" - description: "Port on the proxy/jump server that accepts inbound ssh\ - \ connections." - type: "integer" - minimum: 0 - maximum: 65536 - default: 22 - examples: - - "22" - order: 2 - tunnel_user: - title: "SSH Login Username" - description: "OS-level username for logging into the jump server host" - type: "string" - order: 3 - tunnel_user_password: - title: "Password" - description: "OS-level password for logging into the jump server host" - type: "string" - airbyte_secret: true - order: 4 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-prestashop:0.1.0" - spec: - documentationUrl: "https://docsurl.com" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "PrestaShop Spec" - type: "object" - required: - - "url" - - "access_key" - additionalProperties: false - properties: - url: - type: "string" - description: "Shop URL without trailing slash (domain name or IP address)" - access_key: - type: "string" - description: "Your PrestaShop access key. See the docs for info on how to obtain this." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-quickbooks-singer:0.1.3" - spec: - documentationUrl: "https://docsurl.com" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Quickbooks Singer Spec" - type: "object" - required: - - "client_id" - - "client_secret" - - "refresh_token" - - "realm_id" - - "user_agent" - - "start_date" - - "sandbox" - additionalProperties: false - properties: - client_id: - type: "string" - description: "Identifies which app is making the request. Obtain this value\ - \ from the Keys tab on the app profile via My Apps on the developer site.\ - \ There are two versions of this key: development and production" - client_secret: - description: " Obtain this value from the Keys tab on the app profile via\ - \ My Apps on the developer site. There are two versions of this key: development\ - \ and production" - type: "string" - airbyte_secret: true - refresh_token: - description: "A token used when refreshing the access token." - type: "string" - airbyte_secret: true - realm_id: - description: "Labeled Company ID. The Make API Calls panel is populated\ - \ with the realm id and the current access token" - type: "string" - airbyte_secret: true - user_agent: - type: "string" - description: "Process and email for API logging purposes. Example: tap-quickbooks\ - \ " - start_date: - description: "The default value to use if no bookmark exists for an endpoint\ - \ (rfc3339 date string) E.g, 2021-03-20T00:00:00Z" - type: "string" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-03-20T00:00:00Z" - sandbox: - description: "Development or Production." - type: "boolean" - default: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-recharge:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/recharge" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Recharge Spec" - type: "object" - required: - - "start_date" - - "access_token" - additionalProperties: false - properties: - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Recharge\ - \ API, in the format YYYY-MM-DDT00:00:00Z." - examples: - - "2021-05-14T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - access_token: - type: "string" - description: "The value of the Access Token generated. See the docs for more\ - \ information" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-recurly:0.2.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/recurly" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Recurly Source Spec" - type: "object" - required: - - "api_key" - additionalProperties: false - properties: - api_key: - type: "string" - description: "Recurly API Key. See the docs for more information on how to generate this key." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-redshift:0.3.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Redshift Source Spec" - type: "object" - required: - - "host" - - "port" - - "database" - - "username" - - "password" - additionalProperties: false - properties: - host: - description: "Host Endpoint of the Redshift Cluster (must include the cluster-id,\ - \ region and end with .redshift.amazonaws.com)" - type: "string" - port: - description: "Port of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 5439 - examples: - - "5439" - database: - description: "Name of the database." - type: "string" - examples: - - "master" - username: - description: "Username to use to access the database." - type: "string" - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:0.1.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/s3" - changelogUrl: "https://docs.airbyte.io/integrations/sources/s3" - connectionSpecification: - title: "S3 Source Spec" - type: "object" - properties: - dataset: - title: "Dataset" - description: "This source creates one table per connection, this field is\ - \ the name of that table. This should include only letters, numbers, dash\ - \ and underscores. Note that this may be altered according to destination." - pattern: "^([A-Za-z0-9-_]+)$" - type: "string" - path_pattern: - title: "Path Pattern" - description: "Add at least 1 pattern here to match filepaths against. Use\ - \ | to separate multiple patterns. Airbyte uses these patterns to determine\ - \ which files to pick up from the provider storage. See wcmatch.glob to understand pattern syntax (GLOBSTAR\ - \ and SPLIT flags are enabled). Use pattern ** to pick\ - \ up all files." - examples: - - "**" - - "myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv" - type: "string" - schema: - title: "Schema" - description: "Optionally provide a schema to enforce, as a valid JSON string.\ - \ Ensure this is a mapping of { \"column\" : \"type\" },\ - \ where types are valid JSON Schema datatypes. Leave as {} to auto-infer\ - \ the schema." - default: "{}" - examples: - - "{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"\ - array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}" - type: "string" - format: - title: "Format" - default: "csv" - type: "object" - oneOf: - - title: "csv" - description: "This connector utilises PyArrow (Apache Arrow) for CSV parsing." - type: "object" - properties: - filetype: - title: "Filetype" - const: "csv" - type: "string" - delimiter: - title: "Delimiter" - description: "The character delimiting individual cells in the CSV\ - \ data. This may only be a 1-character string." - default: "," - minLength: 1 - type: "string" - quote_char: - title: "Quote Char" - description: "The character used optionally for quoting CSV values.\ - \ To disallow quoting, make this field blank." - default: "\"" - type: "string" - escape_char: - title: "Escape Char" - description: "The character used optionally for escaping special characters.\ - \ To disallow escaping, leave this field blank." - type: "string" - encoding: - title: "Encoding" - description: "The character encoding of the CSV data. Leave blank\ - \ to default to UTF-8. See list of python encodings for allowable options." - type: "string" - double_quote: - title: "Double Quote" - description: "Whether two quotes in a quoted CSV value denote a single\ - \ quote in the data." - default: true - type: "boolean" - newlines_in_values: - title: "Newlines In Values" - description: "Whether newline characters are allowed in CSV values.\ - \ Turning this on may affect performance. Leave blank to default\ - \ to False." - default: false - type: "boolean" - block_size: - title: "Block Size" - description: "The chunk size in bytes to process at a time in memory\ - \ from each file. If your data is particularly wide and failing\ - \ during schema detection, increasing this should solve it. Beware\ - \ of raising this too high as you could hit OOM errors." - default: 10000 - type: "integer" - additional_reader_options: - title: "Additional Reader Options" - description: "Optionally add a valid JSON string here to provide additional\ - \ options to the csv reader. Mappings must correspond to options\ - \ detailed here. 'column_types' is used internally\ - \ to handle schema so overriding that would likely cause problems." - default: "{}" - examples: - - "{\"timestamp_parsers\": [\"%m/%d/%Y %H:%M\", \"%Y/%m/%d %H:%M\"\ - ], \"strings_can_be_null\": true, \"null_values\": [\"NA\", \"NULL\"\ - ]}" - type: "string" - advanced_options: - title: "Advanced Options" - description: "Optionally add a valid JSON string here to provide additional\ - \ Pyarrow ReadOptions. Specify 'column_names'\ - \ here if your CSV doesn't have header, or if you want to use custom\ - \ column names. 'block_size' and 'encoding' are already used above,\ - \ specify them again here will override the values above." - default: "{}" - examples: - - "{\"column_names\": [\"column1\", \"column2\"]}" - type: "string" - - title: "parquet" - description: "This connector utilises PyArrow (Apache Arrow) for Parquet parsing." - type: "object" - properties: - filetype: - title: "Filetype" - const: "parquet" - type: "string" - buffer_size: - title: "Buffer Size" - description: "Perform read buffering when deserializing individual\ - \ column chunks. By default every group column will be loaded fully\ - \ to memory. This option can help to optimize a work with memory\ - \ if your data is particularly wide or failing during detection\ - \ of OOM errors." - default: 0 - type: "integer" - columns: - title: "Columns" - description: "If you only want to sync a subset of the columns from\ - \ the file(s), add the columns you want here. Leave it empty to\ - \ sync all columns." - type: "array" - items: - type: "string" - batch_size: - title: "Batch Size" - description: "Maximum number of records per batch. Batches may be\ - \ smaller if there aren’t enough rows in the file. This option can\ - \ help to optimize a work with memory if your data is particularly\ - \ wide or failing during detection of OOM errors." - default: 65536 - type: "integer" - provider: - title: "S3: Amazon Web Services" - type: "object" - properties: - bucket: - title: "Bucket" - description: "Name of the S3 bucket where the file(s) exist." - type: "string" - aws_access_key_id: - title: "Aws Access Key Id" - description: "In order to access private Buckets stored on AWS S3, this\ - \ connector requires credentials with the proper permissions. If accessing\ - \ publicly available data, this field is not necessary." - airbyte_secret: true - type: "string" - aws_secret_access_key: - title: "Aws Secret Access Key" - description: "In order to access private Buckets stored on AWS S3, this\ - \ connector requires credentials with the proper permissions. If accessing\ - \ publicly available data, this field is not necessary." - airbyte_secret: true - type: "string" - path_prefix: - title: "Path Prefix" - description: "By providing a path-like prefix (e.g. myFolder/thisTable/)\ - \ under which all the relevant files sit, we can optimise finding\ - \ these in S3. This is optional but recommended if your bucket contains\ - \ many folders/files." - default: "" - type: "string" - endpoint: - title: "Endpoint" - description: "Endpoint to an S3 compatible service. Leave empty to use\ - \ AWS." - default: "" - type: "string" - use_ssl: - title: "Use Ssl" - description: "Is remote server using secure SSL/TLS connection" - type: "boolean" - verify_ssl_cert: - title: "Verify Ssl Cert" - description: "Allow self signed certificates" - type: "boolean" - required: - - "bucket" - required: - - "dataset" - - "path_pattern" - - "provider" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/source-salesloft:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/salesloft" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Salesloft Spec" - type: "object" - required: - - "client_id" - - "client_secret" - - "refresh_token" - - "start_date" - additionalProperties: false - properties: - client_id: - type: "string" - description: "Salesloft client id." - client_secret: - type: "string" - description: "Salesloft client secret." - airbyte_secret: true - refresh_token: - type: "string" - description: "Salesloft refresh token." - airbyte_secret: true - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Salesloft\ - \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\ - \ date will be replicated." - examples: - - "2020-11-16T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/salesforce" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Salesforce Source Spec" - type: "object" - required: - - "client_id" - - "client_secret" - - "refresh_token" - - "start_date" - - "api_type" - additionalProperties: false - properties: - client_id: - description: "The Consumer Key that can be found when viewing your app in\ - \ Salesforce" - type: "string" - client_secret: - description: "The Consumer Secret that can be found when viewing your app\ - \ in Salesforce" - type: "string" - airbyte_secret: true - refresh_token: - description: "Salesforce Refresh Token used for Airbyte to access your Salesforce\ - \ account. If you don't know what this is, follow this guide to retrieve it." - type: "string" - airbyte_secret: true - start_date: - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - type: "string" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-07-25T00:00:00Z" - is_sandbox: - description: "Whether or not the the app is in a Salesforce sandbox. If\ - \ you do not know what this, assume it is false. We provide more info\ - \ on this field in the docs." - type: "boolean" - default: false - api_type: - description: "Unless you know that you are transferring a very small amount\ - \ of data, prefer using the BULK API. This will help avoid using up all\ - \ of your API call quota with Salesforce. Valid values are BULK or REST." - type: "string" - enum: - - "BULK" - - "REST" - default: "BULK" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: [] - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "refresh_token" -- dockerImage: "airbyte/source-sendgrid:0.2.6" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/sendgrid" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Sendgrid Spec" - type: "object" - required: - - "apikey" - additionalProperties: false - properties: - apikey: - type: "string" - description: "API Key, use admin to generate this key." - start_time: - type: "integer" - description: "Start time in timestamp integer format. Any data before this\ - \ timestamp will not be replicated." - examples: - - 1558359837 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-shopify:0.1.21" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/shopify" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Shopify Source CDK Specifications" - type: "object" - required: - - "shop" - - "start_date" - - "auth_method" - additionalProperties: false - properties: - shop: - type: "string" - description: "The name of the shopify store. For https://EXAMPLE.myshopify.com,\ - \ the shop name is 'EXAMPLE'." - start_date: - type: "string" - description: "The date you would like to replicate data. Format: YYYY-MM-DD." - examples: - - "2021-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - auth_method: - title: "Shopify Authorization Method" - type: "object" - oneOf: - - type: "object" - title: "OAuth2.0" - required: - - "client_id" - - "client_secret" - - "access_token" - properties: - auth_method: - type: "string" - const: "access_token" - enum: - - "access_token" - default: "access_token" - order: 0 - client_id: - type: "string" - description: "The API Key of the Shopify developer application." - airbyte_secret: true - client_secret: - type: "string" - description: "The API Secret the Shopify developer application." - airbyte_secret: true - access_token: - type: "string" - description: "Access Token for making authenticated requests." - airbyte_secret: true - - title: "API Password" - type: "object" - required: - - "api_password" - properties: - auth_method: - type: "string" - const: "api_password" - enum: - - "api_password" - default: "api_password" - order: 0 - api_password: - type: "string" - description: "The API PASSWORD for your private application in `Shopify`\ - \ shop." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "auth_method" - - "0" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "access_token" -- dockerImage: "airbyte/source-shortio:0.1.0" - spec: - documentationUrl: "https://developers.short.io/reference" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Shortio Spec" - type: "object" - required: - - "domain_id" - - "secret_key" - - "start_date" - additionalProperties: false - properties: - domain_id: - type: "string" - description: "Domain ID" - airbyte_secret: false - secret_key: - type: "string" - description: "Short.io Secret key" - airbyte_secret: true - start_date: - type: "string" - description: "Start Date, YYYY-MM-DD" - airbyte_secret: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-slack:0.1.12" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/slack" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Slack Spec" - type: "object" - required: - - "start_date" - - "lookback_window" - - "join_channels" - additionalProperties: true - properties: - start_date: - type: "string" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2017-01-25T00:00:00Z" - title: "Start Date" - lookback_window: - type: "integer" - title: "Threads Lookback window (Days)" - description: "How far into the past to look for messages in threads." - examples: - - 7 - - 14 - join_channels: - type: "boolean" - default: true - title: "Join all channels" - description: "Whether to join all channels or to sync data only from channels\ - \ the bot is already in. If false, you'll need to manually add the bot\ - \ to all the channels from which you'd like to sync messages. " - credentials: - title: "Authentication mechanism" - description: "Choose how to authenticate into Slack" - type: "object" - oneOf: - - type: "object" - title: "Sign in via Slack (OAuth)" - required: - - "access_token" - - "client_id" - - "client_secret" - - "option_title" - properties: - option_title: - type: "string" - const: "Default OAuth2.0 authorization" - client_id: - title: "Client ID" - description: "Slack client_id. See our docs if you need help finding this id." - type: "string" - examples: - - "slack-client-id-example" - client_secret: - title: "Client Secret" - description: "Slack client_secret. See our docs if you need help finding this secret." - type: "string" - examples: - - "slack-client-secret-example" - airbyte_secret: true - access_token: - title: "Access token" - description: "Slack access_token. See our docs if you need help generating the token." - type: "string" - examples: - - "slack-access-token-example" - airbyte_secret: true - refresh_token: - title: "Refresh token" - description: "Slack refresh_token. See our docs if you need help generating the token." - type: "string" - examples: - - "slack-refresh-token-example" - airbyte_secret: true - order: 0 - - type: "object" - title: "API Token" - required: - - "api_token" - - "option_title" - properties: - option_title: - type: "string" - const: "API Token Credentials" - api_token: - type: "string" - title: "API Token" - description: "A Slack bot token. See the docs for instructions on how to generate it." - airbyte_secret: true - order: 1 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "0" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "access_token" - - - "refresh_token" -- dockerImage: "airbyte/source-smartsheets:0.1.5" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/smartsheets" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Smartsheets Source Spec" - type: "object" - required: - - "access_token" - - "spreadsheet_id" - additionalProperties: false - properties: - access_token: - title: "API Access token" - description: "Found in Profile > Apps & Integrations > API Access within\ - \ Smartsheet app" - type: "string" - airbyte_secret: true - spreadsheet_id: - title: "Smartsheet ID" - description: "Found in File > Properties" - type: "string" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-snapchat-marketing:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/snapchat-marketing" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Snapchat Marketing Spec" - type: "object" - required: - - "client_id" - - "client_secret" - - "refresh_token" - additionalProperties: false - properties: - client_id: - title: "Client ID" - type: "string" - description: "The Snapchat Client ID for API credentials." - airbyte_secret: true - client_secret: - title: "Client Secret" - type: "string" - description: "The Client Secret for a given Client ID." - airbyte_secret: true - refresh_token: - title: "API Refresh Token" - type: "string" - description: "Refresh Token to get next api key after expiration. Is given\ - \ with API Key" - airbyte_secret: true - start_date: - title: "Start Date" - type: "string" - description: "The start date to sync data. Leave blank for full sync. Format:\ - \ YYYY-MM-DD." - examples: - - "2021-01-01" - default: "1970-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-snowflake:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Snowflake Source Spec" - type: "object" - required: - - "host" - - "role" - - "warehouse" - - "database" - - "schema" - - "username" - - "password" - additionalProperties: false - properties: - host: - description: "Host domain of the snowflake instance (must include the account,\ - \ region, cloud environment, and end with snowflakecomputing.com)." - examples: - - "accountname.us-east-2.aws.snowflakecomputing.com" - type: "string" - title: "Account name" - order: 0 - role: - description: "The role you created for Airbyte to access Snowflake." - examples: - - "AIRBYTE_ROLE" - type: "string" - title: "Role" - order: 1 - warehouse: - description: "The warehouse you created for Airbyte to access data into." - examples: - - "AIRBYTE_WAREHOUSE" - type: "string" - title: "Warehouse" - order: 2 - database: - description: "The database you created for Airbyte to access data into." - examples: - - "AIRBYTE_DATABASE" - type: "string" - title: "Database" - order: 3 - schema: - description: "The source Snowflake schema tables." - examples: - - "AIRBYTE_SCHEMA" - type: "string" - title: "Schema" - order: 4 - username: - description: "The username you created to allow Airbyte to access the database." - examples: - - "AIRBYTE_USER" - type: "string" - title: "Username" - order: 5 - password: - description: "Password associated with the username." - type: "string" - airbyte_secret: true - title: "Password" - order: 6 - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-square:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/square" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Square Source CDK Specifications" - type: "object" - required: - - "api_key" - - "is_sandbox" - additionalProperties: false - properties: - api_key: - type: "string" - description: "The API key for a Square application" - airbyte_secret: true - is_sandbox: - type: "boolean" - description: "Determines the sandbox (true) or production (false) API version" - examples: - - true - - false - default: true - start_date: - type: "string" - description: "The start date to sync data. Leave blank for full sync. Format:\ - \ YYYY-MM-DD." - examples: - - "2021-01-01" - default: "1970-01-01" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - include_deleted_objects: - type: "boolean" - description: "In some streams there is and option to include deleted objects\ - \ (Items, Categories, Discounts, Taxes)" - examples: - - true - - false - default: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-strava:0.1.0" - spec: - documentationUrl: "https://docsurl.com" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Strava Spec" - type: "object" - required: - - "client_id" - - "client_secret" - - "refresh_token" - - "athlete_id" - - "start_date" - additionalProperties: false - properties: - client_id: - type: "string" - description: "Strava Client ID" - pattern: "^[0-9_\\-]+$" - examples: - - "12345" - client_secret: - type: "string" - description: "Strava Client Secret" - pattern: "^[0-9a-fA-F]+$" - examples: - - "fc6243f283e51f6ca989aab298b17da125496f50" - airbyte_secret: true - refresh_token: - type: "string" - description: "Strava Refresh Token with activity:read_all permissions" - pattern: "^[0-9a-fA-F]+$" - examples: - - "fc6243f283e51f6ca989aab298b17da125496f50" - airbyte_secret: true - athlete_id: - type: "integer" - description: "Strava Athlete ID" - pattern: "^[0-9_\\-]+$" - examples: - - "17831421" - start_date: - type: "string" - description: "Start Query Timestamp in UTC" - examples: - - "2016-12-31 23:59:59" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-stripe:0.1.21" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/stripe" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Stripe Source Spec" - type: "object" - required: - - "client_secret" - - "account_id" - - "start_date" - additionalProperties: false - properties: - client_secret: - type: "string" - pattern: "^(s|r)k_(live|test)_[a-zA-Z0-9]+$" - description: "Stripe API key (usually starts with 'sk_live_'; find yours\ - \ here)." - airbyte_secret: true - account_id: - type: "string" - description: "Your Stripe account ID (starts with 'acct_', find yours here)." - start_date: - type: "string" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2017-01-25T00:00:00Z" - lookback_window_days: - type: "integer" - title: "Lookback Window (in days)" - default: 0 - minimum: 0 - description: "When set, the connector will always reload data from the past\ - \ N days, where N is the value set here. This is useful if your data is\ - \ updated after creation." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-surveymonkey:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/surveymonkey" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "SurveyMonkey Spec" - type: "object" - required: - - "start_date" - additionalProperties: true - properties: - start_date: - title: "Start Date" - type: "string" - description: "The date from which you'd like to replicate the data" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$" - examples: - - "2021-01-01T00:00:00Z" - credentials: - type: "object" - title: "Authentication Type" - oneOf: - - title: "Authenticate via OAuth" - type: "object" - required: - - "client_id" - - "client_secret" - - "access_token" - - "auth_type" - properties: - auth_type: - type: "string" - const: "OAuth" - enum: - - "OAuth" - default: "OAuth" - order: 0 - client_id: - title: "Client ID" - type: "string" - description: "The Client ID of your developer application" - airbyte_secret: true - client_secret: - title: "Client Secret" - type: "string" - description: "The client secret of your developer application" - airbyte_secret: true - access_token: - title: "Access Token" - type: "string" - description: "An access token generated using the above client ID\ - \ and secret" - airbyte_secret: true - - type: "object" - title: "Token Authentication" - additionalProperties: false - required: - - "access_token" - - "auth_type" - properties: - auth_type: - type: "string" - const: "Token" - enum: - - "Token" - default: "Token" - order: 0 - access_token: - type: "string" - airbyte_secret: true - description: "API Token. See the docs for information on how to generate this key." - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "0" - oauthFlowInitParameters: - - - "client_id" - - - "client_secret" - oauthFlowOutputParameters: - - - "access_token" -- dockerImage: "airbyte/source-tempo:0.2.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Tempo Spec" - type: "object" - required: - - "api_token" - additionalProperties: false - properties: - api_token: - type: "string" - description: "Tempo API Token." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tiktok-marketing:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" - changelogUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" - connectionSpecification: - title: "TikTok Marketing Source Spec" - type: "object" - properties: - environment: - title: "Environment" - default: "Production" - oneOf: - - title: "Production" - type: "object" - properties: - environment: - title: "Environment" - const: "prod" - type: "string" - app_id: - title: "App Id" - description: "The App id applied by the developer." - type: "string" - secret: - title: "Secret" - description: "The private key of the developer's application." - airbyte_secret: true - type: "string" - required: - - "app_id" - - "secret" - - title: "Sandbox" - type: "object" - properties: - environment: - title: "Environment" - const: "sandbox" - type: "string" - advertiser_id: - title: "Advertiser Id" - description: "The Advertiser ID which generated for the developer's\ - \ Sandbox application." - type: "string" - required: - - "advertiser_id" - type: "object" - access_token: - title: "Access Token" - description: "Long-term Authorized Access Token." - airbyte_secret: true - type: "string" - start_date: - title: "Start Date" - description: "Start Date in format: YYYY-MM-DD." - default: "01-09-2016" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - type: "string" - required: - - "access_token" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" -- dockerImage: "airbyte/source-trello:0.1.1" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/trello" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Trello Spec" - type: "object" - required: - - "token" - - "key" - - "start_date" - additionalProperties: true - properties: - token: - type: "string" - title: "API token" - description: "A Trello token. See the docs for instructions on how to generate it." - airbyte_secret: true - key: - type: "string" - title: "API key" - description: "A Trello token. See the docs for instructions on how to generate it." - airbyte_secret: true - start_date: - type: "string" - title: "Start date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$" - description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ - \ data before this date will not be replicated." - examples: - - "2021-03-01T00:00:00.000Z" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: [] - oauthFlowInitParameters: [] - oauthFlowOutputParameters: - - - "token" - - - "key" -- dockerImage: "airbyte/source-twilio:0.1.1" - spec: - documentationUrl: "https://hub.docker.com/r/airbyte/source-twilio" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Twilio Spec" - type: "object" - required: - - "account_sid" - - "auth_token" - - "start_date" - additionalProperties: false - properties: - account_sid: - title: "Account ID" - description: "Twilio account SID" - airbyte_secret: true - type: "string" - auth_token: - title: "Auth Token" - description: "Twilio Auth Token." - airbyte_secret: true - type: "string" - start_date: - title: "Replication Start Date" - description: "UTC date and time in the format 2020-10-01T00:00:00Z. Any\ - \ data before this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2020-10-01T00:00:00Z" - type: "string" - supportsIncremental: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: - - "append" -- dockerImage: "airbyte/source-typeform:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/typeform" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Typeform Spec" - type: "object" - required: - - "token" - - "start_date" - additionalProperties: true - properties: - start_date: - type: "string" - description: "The date you would like to replicate data. Format: YYYY-MM-DDTHH:mm:ss[Z]." - examples: - - "2020-01-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - token: - type: "string" - description: "The API Token for a Typeform account." - airbyte_secret: true - form_ids: - title: "Form IDs to replicate" - description: "When this parameter is set, the connector will replicate data\ - \ only from the input forms. Otherwise, all forms in your Typeform account\ - \ will be replicated. You can find form IDs in your form URLs. For example,\ - \ in the URL \"https://mysite.typeform.com/to/u6nXL7\" the form_id is\ - \ u6nXL7. You can find form URLs on Share panel" - type: "array" - items: - type: "string" - uniqueItems: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-us-census:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/us-census" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "https://api.census.gov/ Source Spec" - type: "object" - required: - - "api_key" - - "query_path" - additionalProperties: false - properties: - query_params: - type: "string" - description: "The query parameters portion of the GET request, without the\ - \ api key" - pattern: "^\\w+=[\\w,:*]+(&(?!key)\\w+=[\\w,:*]+)*$" - examples: - - "get=NAME,NAICS2017_LABEL,LFO_LABEL,EMPSZES_LABEL,ESTAB,PAYANN,PAYQTR1,EMP&for=us:*&NAICS2017=72&LFO=001&EMPSZES=001" - - "get=MOVEDIN,GEOID1,GEOID2,MOVEDOUT,FULL1_NAME,FULL2_NAME,MOVEDNET&for=county:*" - query_path: - type: "string" - description: "The path portion of the GET request" - pattern: "^data(\\/[\\w\\d]+)+$" - examples: - - "data/2019/cbp" - - "data/2018/acs" - - "data/timeseries/healthins/sahie" - api_key: - type: "string" - description: "Your API Key. Get your key here." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zendesk-chat:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-chat" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Zendesk Chat Spec" - type: "object" - required: - - "start_date" - - "access_token" - additionalProperties: false - properties: - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Zendesk\ - \ Chat API, in the format YYYY-MM-DDT00:00:00Z." - examples: - - "2021-02-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - access_token: - type: "string" - description: "The value of the Access Token generated. See the docs for\ - \ more information" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zendesk-sunshine:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk_sunshine" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Zendesk Sunshine Spec" - type: "object" - required: - - "api_token" - - "email" - - "start_date" - - "subdomain" - additionalProperties: false - properties: - api_token: - type: "string" - airbyte_secret: true - description: "API Token. See the docs for information on how to generate this key." - email: - type: "string" - description: "The user email for your Zendesk account" - subdomain: - type: "string" - description: "The subdomain for your Zendesk Account" - start_date: - title: "Start Date" - type: "string" - description: "The date from which you'd like to replicate the data" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: "2021-01-01T00:00:00.000000Z" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zendesk-support:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Zendesk Support Spec" - type: "object" - required: - - "start_date" - - "subdomain" - - "auth_method" - additionalProperties: false - properties: - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Zendesk\ - \ Support API, in the format YYYY-MM-DDT00:00:00Z. All data generated\ - \ after this date will be replicated." - examples: - - "2020-10-15T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - subdomain: - type: "string" - description: "The subdomain for your Zendesk Support" - auth_method: - title: "ZenDesk Authorization Method" - type: "object" - default: "api_token" - description: "Zendesk service provides 2 auth method: API token and oAuth2.\ - \ Now only the first one is available. Another one will be added in the\ - \ future" - oneOf: - - title: "API Token" - type: "object" - required: - - "email" - - "api_token" - additionalProperties: false - properties: - auth_method: - type: "string" - const: "api_token" - email: - type: "string" - description: "The user email for your Zendesk account" - api_token: - type: "string" - description: "The value of the API token generated. See the docs\ - \ for more information" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zendesk-talk:0.1.2" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-talk" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Zendesk Talk Spec" - type: "object" - required: - - "start_date" - - "subdomain" - - "access_token" - - "email" - additionalProperties: false - properties: - start_date: - type: "string" - description: "The date from which you'd like to replicate data for Zendesk\ - \ Talk API, in the format YYYY-MM-DDT00:00:00Z." - examples: - - "2021-04-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - subdomain: - type: "string" - description: "The subdomain for your Zendesk Talk" - access_token: - type: "string" - description: "The value of the API token generated. See the docs for more information" - airbyte_secret: true - email: - type: "string" - description: "The user email for your Zendesk account" - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zoom-singer:0.2.4" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/zoom" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Zoom Singer Spec" - type: "object" - required: - - "jwt" - additionalProperties: false - properties: - jwt: - title: "JWT Token" - type: "string" - description: "Zoom JWT Token. See the docs for more information on how to obtain this key." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zuora:0.1.3" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/zuora" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Zuora Connector Configuration" - type: "object" - required: - - "start_date" - - "tenant_endpoint" - - "data_query" - - "client_id" - - "client_secret" - properties: - start_date: - type: "string" - title: "Start Date" - description: "Start Date in format: YYYY-MM-DD" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - window_in_days: - type: "string" - title: "Query Window (in days)" - description: "The amount of days for each data-chunk begining from start_date.\ - \ Bigger the value - faster the fetch. (0.1 - as for couple of hours,\ - \ 1 - as for a Day; 364 - as for a Year)." - examples: - - "0.5" - - "1" - - "30" - - "60" - - "90" - - "120" - - "200" - - "364" - pattern: "^(0|[1-9]\\d*)(\\.\\d+)?$" - default: "90" - tenant_endpoint: - title: "Tenant Endpoint Location" - type: "string" - description: "Please choose the right endpoint where your Tenant is located.\ - \ More info by this Link" - enum: - - "US Production" - - "US Cloud Production" - - "US API Sandbox" - - "US Cloud API Sandbox" - - "US Central Sandbox" - - "US Performance Test" - - "EU Production" - - "EU API Sandbox" - - "EU Central Sandbox" - data_query: - title: "Data Query Type" - type: "string" - description: "Choose between `Live`, or `Unlimited` - the optimized, replicated\ - \ database at 12 hours freshness for high volume extraction Link" - enum: - - "Live" - - "Unlimited" - default: "Live" - client_id: - type: "string" - title: "Client ID" - description: "Your OAuth user Client ID" - airbyte_secret: true - client_secret: - type: "string" - title: "Client Secret" - description: "Your OAuth user Client Secret" - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] diff --git a/airbyte-config/models/build.gradle b/airbyte-config/models/build.gradle index d62c88c7e163..271b3fe685e7 100644 --- a/airbyte-config/models/build.gradle +++ b/airbyte-config/models/build.gradle @@ -7,11 +7,10 @@ plugins { dependencies { implementation project(':airbyte-json-validation') implementation project(':airbyte-protocol:models') - implementation project(':airbyte-commons') } jsonSchema2Pojo { - sourceType = SourceType.YAMLSCHEMA + sourceType = SourceType.YAMLSCHEMA source = files("${sourceSets.main.output.resourcesDir}/types") targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') diff --git a/airbyte-config/models/src/main/resources/types/DockerImageSpec.yaml b/airbyte-config/models/src/main/resources/types/DockerImageSpec.yaml deleted file mode 100644 index 0d3becf8e74c..000000000000 --- a/airbyte-config/models/src/main/resources/types/DockerImageSpec.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -"$schema": http://json-schema.org/draft-07/schema# -"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/DockerImageSpec.yaml -title: DockerImageSpec -description: docker image name and the connector specification associated with it -type: object -required: - - dockerImage - - spec -additionalProperties: false -properties: - dockerImage: - type: string - spec: - type: object - existingJavaType: io.airbyte.protocol.models.ConnectorSpecification diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java index 2181bcb16264..19035ed42295 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java @@ -366,14 +366,7 @@ ConnectorCounter updateConnectorDefinitions(final DSLContext ctx, final ConnectorInfo connectorInfo = connectorRepositoryToIdVersionMap.get(repository); final JsonNode currentDefinition = connectorInfo.definition; - - // todo (lmossman) - this logic to remove the "spec" field is temporary; it is necessary to avoid - // breaking users who are actively using an old connector version, otherwise specs from the most - // recent connector versions may be inserted into the db which could be incompatible with the - // version they are actually using. - // Once the faux major version bump has been merged, this "new field" logic will be removed - // entirely. - final Set newFields = Sets.difference(getNewFields(currentDefinition, latestDefinition), Set.of("spec")); + final Set newFields = getNewFields(currentDefinition, latestDefinition); // Process connector in use if (connectorRepositoriesInUse.contains(repository)) { diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/YamlSeedConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/YamlSeedConfigPersistence.java index 2902d15ef1e9..3bca71d57c2d 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/YamlSeedConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/YamlSeedConfigPersistence.java @@ -5,10 +5,8 @@ package io.airbyte.config.persistence; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; -import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.commons.yaml.Yamls; @@ -21,7 +19,6 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -48,40 +45,11 @@ public static YamlSeedConfigPersistence get(final Class seedDefinitionsResour return new YamlSeedConfigPersistence(seedDefinitionsResourceClass); } - private YamlSeedConfigPersistence(final Class seedResourceClass) throws IOException { - final Map sourceDefinitionConfigs = getConfigs(seedResourceClass, SeedType.STANDARD_SOURCE_DEFINITION); - final Map sourceSpecConfigs = getConfigs(seedResourceClass, SeedType.SOURCE_SPEC); - final Map fullSourceDefinitionConfigs = sourceDefinitionConfigs.entrySet().stream() - .collect(Collectors.toMap(Entry::getKey, e -> mergeSpecIntoDefinition(e.getValue(), sourceSpecConfigs))); - - final Map destinationDefinitionConfigs = getConfigs(seedResourceClass, SeedType.STANDARD_DESTINATION_DEFINITION); - final Map destinationSpecConfigs = getConfigs(seedResourceClass, SeedType.DESTINATION_SPEC); - final Map fullDestinationDefinitionConfigs = destinationDefinitionConfigs.entrySet().stream() - .collect(Collectors.toMap(Entry::getKey, e -> mergeSpecIntoDefinition(e.getValue(), destinationSpecConfigs))); - + private YamlSeedConfigPersistence(final Class seedDefinitionsResourceClass) throws IOException { this.allSeedConfigs = ImmutableMap.>builder() - .put(SeedType.STANDARD_SOURCE_DEFINITION, fullSourceDefinitionConfigs) - .put(SeedType.STANDARD_DESTINATION_DEFINITION, fullDestinationDefinitionConfigs).build(); - } - - /** - * Merges the corresponding spec JSON into the definition JSON. This is necessary because specs are - * stored in a separate resource file from definitions. - * - * @param definitionJson JSON of connector definition that is missing a spec - * @param specConfigs map of docker image to JSON of docker image/connector spec pair - * @return JSON of connector definition including the connector spec - */ - private JsonNode mergeSpecIntoDefinition(final JsonNode definitionJson, final Map specConfigs) { - final String dockerImage = DockerUtils.getTaggedImageName( - definitionJson.get("dockerRepository").asText(), - definitionJson.get("dockerImageTag").asText()); - final JsonNode specConfigJson = specConfigs.get(dockerImage); - if (specConfigJson == null || specConfigJson.get("spec") == null) { - throw new UnsupportedOperationException(String.format("There is no seed spec for docker image %s", dockerImage)); - } - ((ObjectNode) definitionJson).set("spec", specConfigJson.get("spec")); - return definitionJson; + .put(SeedType.STANDARD_SOURCE_DEFINITION, getConfigs(seedDefinitionsResourceClass, SeedType.STANDARD_SOURCE_DEFINITION)) + .put(SeedType.STANDARD_DESTINATION_DEFINITION, getConfigs(seedDefinitionsResourceClass, SeedType.STANDARD_DESTINATION_DEFINITION)) + .build(); } @SuppressWarnings("UnstableApiUsage") diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/YamlSeedConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/YamlSeedConfigPersistenceTest.java index a6f261628046..8a740ba53568 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/YamlSeedConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/YamlSeedConfigPersistenceTest.java @@ -15,7 +15,6 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.StandardWorkspace; import java.io.IOException; -import java.net.URI; import java.util.Collections; import java.util.Map; import java.util.stream.Stream; @@ -42,7 +41,6 @@ public void testGetConfig() throws Exception { assertEquals("airbyte/source-mysql", mysqlSource.getDockerRepository()); assertEquals("https://docs.airbyte.io/integrations/sources/mysql", mysqlSource.getDocumentationUrl()); assertEquals("mysql.svg", mysqlSource.getIcon()); - assertEquals(URI.create("https://docs.airbyte.io/integrations/sources/mysql"), mysqlSource.getSpec().getDocumentationUrl()); // destination final String s3DestinationId = "4816b78f-1489-44c1-9060-4b19d5fa9362"; @@ -52,16 +50,13 @@ public void testGetConfig() throws Exception { assertEquals("S3", s3Destination.getName()); assertEquals("airbyte/destination-s3", s3Destination.getDockerRepository()); assertEquals("https://docs.airbyte.io/integrations/destinations/s3", s3Destination.getDocumentationUrl()); - assertEquals(URI.create("https://docs.airbyte.io/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); } @Test public void testGetInvalidConfig() { - assertThrows( - UnsupportedOperationException.class, + assertThrows(UnsupportedOperationException.class, () -> PERSISTENCE.getConfig(ConfigSchema.STANDARD_SYNC, "invalid_id", StandardSync.class)); - assertThrows( - ConfigNotFoundException.class, + assertThrows(ConfigNotFoundException.class, () -> PERSISTENCE.getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, "invalid_id", StandardWorkspace.class)); } diff --git a/airbyte-config/specs/README.md b/airbyte-config/specs/README.md deleted file mode 100644 index 8d043e1ec972..000000000000 --- a/airbyte-config/specs/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Generating Seed Connector Specs - -The catalog of seeded connector definitions is stored and manually updated in the `airbyte-config/init/src/main/resources/seed/*_definitions.yaml` -files. These manually-maintained connector definitions intentionally _do not_ contain the connector specs, in an effort to keep these files -human-readable and easily-editable, and because specs can be automatically fetched. - -This automatic fetching of connector specs is the goal of the SeedConnectorSpecGenerator. This class reads the connector definitions in -the `airbyte-config/init/src/main/resources/seed/*_definitions.yaml` files, fetches the corresponding specs from the GCS bucket cache, and writes the -specs to the `airbyte-config/init/src/main/resources/seed/*_specs.yaml` files. See the -[SeedConnectorSpecGenerator](src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java) class for more details. - -Therefore, whenever a connector definition is updated in the `airbyte-config/init/src/main/resources/seed/*_definitions.yaml` files, the -SeedConnectorSpecGenerator should be re-ran to generate the updated connector specs files. To do so, -run `./gradlew :airbyte-config:init:processResources`, or just build the platform project, and commit the changes to your PR. If you do not do this, -the build in the CI will fail because there will be a diff in the generated files as you have not checked in the changes that were applied by the -generator. diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle deleted file mode 100644 index 91d1fd092170..000000000000 --- a/airbyte-config/specs/build.gradle +++ /dev/null @@ -1,24 +0,0 @@ -plugins { - id 'java' -} - -dependencies { - implementation 'commons-cli:commons-cli:1.4' - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-cli') - implementation project(':airbyte-config:models') - implementation project(':airbyte-protocol:models') - implementation project(':airbyte-json-validation') -} - -task generateSeedConnectorSpecs(type: JavaExec, dependsOn: compileJava) { - classpath = sourceSets.main.runtimeClasspath - - mainClass = 'io.airbyte.config.specs.SeedConnectorSpecGenerator' - - args '--seed-root' - args new File(project(":airbyte-config:init").projectDir, '/src/main/resources/seed') -} - -project(":airbyte-config:init").tasks.processResources.dependsOn(generateSeedConnectorSpecs) diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/GcsBucketSpecFetcher.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/GcsBucketSpecFetcher.java deleted file mode 100644 index 832326c551c4..000000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/GcsBucketSpecFetcher.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.api.client.util.Preconditions; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.Storage; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteProtocolSchema; -import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.validation.json.JsonSchemaValidator; -import io.airbyte.validation.json.JsonValidationException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class GcsBucketSpecFetcher { - - private static final Logger LOGGER = LoggerFactory.getLogger(GcsBucketSpecFetcher.class); - - private final Storage storage; - private final String bucketName; - - public GcsBucketSpecFetcher(final Storage storage, final String bucketName) { - this.storage = storage; - this.bucketName = bucketName; - } - - public String getBucketName() { - return bucketName; - } - - public Optional attemptFetch(final String dockerImage) { - final String[] dockerImageComponents = dockerImage.split(":"); - Preconditions.checkArgument(dockerImageComponents.length == 2, "Invalidate docker image: " + dockerImage); - final String dockerImageName = dockerImageComponents[0]; - final String dockerImageTag = dockerImageComponents[1]; - - final Path specPath = Path.of("specs").resolve(dockerImageName).resolve(dockerImageTag).resolve("spec.json"); - LOGGER.debug("Checking path for cached spec: {} {}", bucketName, specPath); - final Blob specAsBlob = storage.get(bucketName, specPath.toString()); - - // if null it means the object was not found. - if (specAsBlob == null) { - LOGGER.debug("Spec not found in bucket storage"); - return Optional.empty(); - } - - final String specAsString = new String(specAsBlob.getContent(), StandardCharsets.UTF_8); - try { - validateConfig(Jsons.deserialize(specAsString)); - } catch (final JsonValidationException e) { - LOGGER.error("Received invalid spec from bucket store. {}", e.toString()); - return Optional.empty(); - } - return Optional.of(Jsons.deserialize(specAsString, ConnectorSpecification.class)); - } - - private static void validateConfig(final JsonNode json) throws JsonValidationException { - final JsonSchemaValidator jsonSchemaValidator = new JsonSchemaValidator(); - final JsonNode specJsonSchema = JsonSchemaValidator.getSchema(AirbyteProtocolSchema.PROTOCOL.getFile(), "ConnectorSpecification"); - jsonSchemaValidator.ensure(specJsonSchema, json); - } - -} diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java deleted file mode 100644 index 05a273b08d84..000000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.cloud.storage.StorageOptions; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.cli.Clis; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.DockerImageSpec; -import io.airbyte.config.EnvConfigs; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This script is responsible for ensuring that up-to-date {@link ConnectorSpecification}s for every - * connector definition in the seed are stored in a corresponding resource file, for the purpose of - * seeding the specs into the config database on server startup. See - * ./airbyte-config/specs/readme.md for more details on how this class is run and how it fits into - * the project. - *

- * Specs are stored in a separate file from the definitions in an effort to keep the definitions - * yaml files human-readable and easily-editable, as specs can be rather large. - *

- * Specs are fetched from the GCS spec cache bucket, so if any specs are missing from the bucket - * then this will fail. Note that this script only pulls specs from the bucket cache; it never - * pushes specs to the bucket. Since this script runs at build time, the decision was to depend on - * the bucket cache rather than running a docker container to fetch the spec during the build which - * could be slow and unwieldy. If there is a failure, check the bucket cache and figure out how to - * get the correct spec in there. - */ -public class SeedConnectorSpecGenerator { - - private static final String DOCKER_REPOSITORY_FIELD = "dockerRepository"; - private static final String DOCKER_IMAGE_TAG_FIELD = "dockerImageTag"; - private static final String DOCKER_IMAGE_FIELD = "dockerImage"; - private static final String SPEC_FIELD = "spec"; - private static final String SPEC_BUCKET_NAME = new EnvConfigs().getSpecCacheBucket(); - - private static final Logger LOGGER = LoggerFactory.getLogger(SeedConnectorSpecGenerator.class); - - private static final Option SEED_ROOT_OPTION = Option.builder("s").longOpt("seed-root").hasArg(true).required(true) - .desc("path to where seed resource files are stored").build(); - private static final Options OPTIONS = new Options().addOption(SEED_ROOT_OPTION); - - private final GcsBucketSpecFetcher bucketSpecFetcher; - - public SeedConnectorSpecGenerator(final GcsBucketSpecFetcher bucketSpecFetcher) { - this.bucketSpecFetcher = bucketSpecFetcher; - } - - public static void main(final String[] args) throws Exception { - final CommandLine parsed = Clis.parse(args, OPTIONS); - final Path outputRoot = Path.of(parsed.getOptionValue(SEED_ROOT_OPTION.getOpt())); - - final GcsBucketSpecFetcher bucketSpecFetcher = new GcsBucketSpecFetcher(StorageOptions.getDefaultInstance().getService(), SPEC_BUCKET_NAME); - final SeedConnectorSpecGenerator seedConnectorSpecGenerator = new SeedConnectorSpecGenerator(bucketSpecFetcher); - seedConnectorSpecGenerator.run(outputRoot, SeedConnectorType.SOURCE); - seedConnectorSpecGenerator.run(outputRoot, SeedConnectorType.DESTINATION); - } - - public void run(final Path seedRoot, final SeedConnectorType seedConnectorType) throws IOException { - LOGGER.info("Updating seeded {} definition specs if necessary...", seedConnectorType.name()); - - final JsonNode seedDefinitionsJson = yamlToJson(seedRoot, seedConnectorType.getDefinitionFileName()); - final JsonNode seedSpecsJson = yamlToJson(seedRoot, seedConnectorType.getSpecFileName()); - - final List updatedSeedSpecs = fetchUpdatedSeedSpecs(seedDefinitionsJson, seedSpecsJson); - - final String outputString = String.format("# This file is generated by %s.\n", this.getClass().getName()) - + "# Do NOT edit this file directly. See generator class for more details.\n" - + Yamls.serialize(updatedSeedSpecs); - final Path outputPath = IOs.writeFile(seedRoot.resolve(seedConnectorType.getSpecFileName()), outputString); - - LOGGER.info("Finished updating {}", outputPath); - } - - private JsonNode yamlToJson(final Path root, final String fileName) { - final String yamlString = IOs.readFile(root, fileName); - return Yamls.deserialize(yamlString); - } - - @VisibleForTesting - final List fetchUpdatedSeedSpecs(final JsonNode seedDefinitions, final JsonNode currentSeedSpecs) { - final List seedDefinitionsDockerImages = MoreIterators.toList(seedDefinitions.elements()) - .stream() - .map(json -> String.format("%s:%s", json.get(DOCKER_REPOSITORY_FIELD).asText(), json.get(DOCKER_IMAGE_TAG_FIELD).asText())) - .collect(Collectors.toList()); - - final Map currentSeedImageToSpec = MoreIterators.toList(currentSeedSpecs.elements()) - .stream() - .collect(Collectors.toMap( - json -> json.get(DOCKER_IMAGE_FIELD).asText(), - json -> new DockerImageSpec().withDockerImage(json.get(DOCKER_IMAGE_FIELD).asText()) - .withSpec(Jsons.object(json.get(SPEC_FIELD), ConnectorSpecification.class)))); - - return seedDefinitionsDockerImages - .stream() - .map(dockerImage -> currentSeedImageToSpec.containsKey(dockerImage) ? currentSeedImageToSpec.get(dockerImage) : fetchSpecFromGCS(dockerImage)) - .collect(Collectors.toList()); - } - - private DockerImageSpec fetchSpecFromGCS(final String dockerImage) { - LOGGER.info("Seeded spec not found for docker image {} - fetching from GCS bucket {}...", dockerImage, bucketSpecFetcher.getBucketName()); - final ConnectorSpecification spec = bucketSpecFetcher.attemptFetch(dockerImage) - .orElseThrow(() -> new RuntimeException(String.format( - "Failed to fetch valid spec file for docker image %s from GCS bucket %s", - dockerImage, - bucketSpecFetcher.getBucketName()))); - return new DockerImageSpec().withDockerImage(dockerImage).withSpec(spec); - } - -} diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorType.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorType.java deleted file mode 100644 index 36d1326af215..000000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorType.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -public enum SeedConnectorType { - - SOURCE( - "source_definitions.yaml", - "source_specs.yaml"), - DESTINATION( - "destination_definitions.yaml", - "destination_specs.yaml"); - - private final String definitionFileName; - private final String specFileName; - - SeedConnectorType(final String definitionFileName, - final String specFileName) { - this.definitionFileName = definitionFileName; - this.specFileName = specFileName; - } - - public String getDefinitionFileName() { - return definitionFileName; - } - - public String getSpecFileName() { - return specFileName; - } - -} diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/GcsBucketSpecFetcherTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/GcsBucketSpecFetcherTest.java deleted file mode 100644 index 25e16bea545b..000000000000 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/GcsBucketSpecFetcherTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.Storage; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class GcsBucketSpecFetcherTest { - - private static final String BUCKET_NAME = "bucket"; - private static final String DOCKER_REPOSITORY = "image"; - private static final String DOCKER_IMAGE_TAG = "0.1.0"; - private static final String DOCKER_IMAGE = DOCKER_REPOSITORY + ":" + DOCKER_IMAGE_TAG; - private static final String SPEC_PATH = Path.of("specs").resolve(DOCKER_REPOSITORY).resolve(DOCKER_IMAGE_TAG).resolve("spec.json").toString(); - - private Storage storage; - private Blob specBlob; - private final ConnectorSpecification spec = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar"))); - - @SuppressWarnings("unchecked") - @BeforeEach - void setup() throws IOException { - storage = mock(Storage.class); - - final byte[] specBytes = Jsons.toBytes(Jsons.jsonNode(spec)); - specBlob = mock(Blob.class); - when(specBlob.getContent()).thenReturn(specBytes); - } - - @Test - void testGetsSpecIfPresent() throws IOException { - when(storage.get(BUCKET_NAME, SPEC_PATH)).thenReturn(specBlob); - - final GcsBucketSpecFetcher bucketSpecFetcher = new GcsBucketSpecFetcher(storage, BUCKET_NAME); - final Optional returnedSpec = bucketSpecFetcher.attemptFetch(DOCKER_IMAGE); - - assertTrue(returnedSpec.isPresent()); - assertEquals(spec, returnedSpec.get()); - } - - @Test - void testReturnsEmptyIfNotPresent() throws IOException { - when(storage.get(BUCKET_NAME, SPEC_PATH)).thenReturn(null); - - final GcsBucketSpecFetcher bucketSpecFetcher = new GcsBucketSpecFetcher(storage, BUCKET_NAME); - final Optional returnedSpec = bucketSpecFetcher.attemptFetch(DOCKER_IMAGE); - - assertTrue(returnedSpec.isEmpty()); - } - - @Test - void testReturnsEmptyIfInvalidSpec() throws IOException { - final Blob invalidSpecBlob = mock(Blob.class); - when(invalidSpecBlob.getContent()).thenReturn("{\"notASpec\": true}".getBytes(StandardCharsets.UTF_8)); - when(storage.get(BUCKET_NAME, SPEC_PATH)).thenReturn(invalidSpecBlob); - - final GcsBucketSpecFetcher bucketSpecFetcher = new GcsBucketSpecFetcher(storage, BUCKET_NAME); - final Optional returnedSpec = bucketSpecFetcher.attemptFetch(DOCKER_IMAGE); - - assertTrue(returnedSpec.isEmpty()); - } - -} diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java deleted file mode 100644 index 0925608a2f62..000000000000 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DockerImageSpec; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class SeedConnectorSpecGeneratorTest { - - private static final UUID DEF_ID1 = java.util.UUID.randomUUID(); - private static final UUID DEF_ID2 = java.util.UUID.randomUUID(); - private static final String CONNECTOR_NAME1 = "connector1"; - private static final String CONNECTOR_NAME2 = "connector2"; - private static final String DOCUMENTATION_URL = "https://wwww.example.com"; - private static final String DOCKER_REPOSITORY1 = "airbyte/connector1"; - private static final String DOCKER_REPOSITORY2 = "airbyte/connector2"; - private static final String DOCKER_TAG1 = "0.1.0"; - private static final String DOCKER_TAG2 = "0.2.0"; - private static final String BUCKET_NAME = "bucket"; - - private SeedConnectorSpecGenerator seedConnectorSpecGenerator; - private GcsBucketSpecFetcher bucketSpecFetcherMock; - - @BeforeEach - void setup() { - bucketSpecFetcherMock = mock(GcsBucketSpecFetcher.class); - when(bucketSpecFetcherMock.getBucketName()).thenReturn(BUCKET_NAME); - - seedConnectorSpecGenerator = new SeedConnectorSpecGenerator(bucketSpecFetcherMock); - } - - @Test - void testMissingSpecIsFetched() { - final StandardDestinationDefinition sourceDefinition1 = new StandardDestinationDefinition() - .withDestinationDefinitionId(DEF_ID1) - .withDockerRepository(DOCKER_REPOSITORY1) - .withDockerImageTag(DOCKER_TAG1) - .withName(CONNECTOR_NAME1) - .withDocumentationUrl(DOCUMENTATION_URL); - final ConnectorSpecification spec1 = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo1", "bar1"))); - final DockerImageSpec dockerImageSpec1 = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1).withSpec(spec1); - - final StandardDestinationDefinition sourceDefinition2 = new StandardDestinationDefinition() - .withDestinationDefinitionId(DEF_ID2) - .withDockerRepository(DOCKER_REPOSITORY2) - .withDockerImageTag(DOCKER_TAG2) - .withName(CONNECTOR_NAME2) - .withDocumentationUrl(DOCUMENTATION_URL); - final ConnectorSpecification spec2 = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))); - final DockerImageSpec dockerImageSpec2 = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY2 + ":" + DOCKER_TAG2).withSpec(spec2); - - final JsonNode seedDefinitions = Jsons.jsonNode(Arrays.asList(sourceDefinition1, sourceDefinition2)); - final JsonNode seedSpecs = Jsons.jsonNode(List.of(dockerImageSpec1)); - - when(bucketSpecFetcherMock.attemptFetch(DOCKER_REPOSITORY2 + ":" + DOCKER_TAG2)).thenReturn(Optional.of(spec2)); - - final List actualSeedSpecs = seedConnectorSpecGenerator.fetchUpdatedSeedSpecs(seedDefinitions, seedSpecs); - final List expectedSeedSpecs = Arrays.asList(dockerImageSpec1, dockerImageSpec2); - - assertEquals(expectedSeedSpecs, actualSeedSpecs); - } - - @Test - void testOutdatedSpecIsFetched() { - final StandardDestinationDefinition sourceDefinition = new StandardDestinationDefinition() - .withDestinationDefinitionId(DEF_ID1) - .withDockerRepository(DOCKER_REPOSITORY1) - .withDockerImageTag(DOCKER_TAG2) - .withName(CONNECTOR_NAME1) - .withDocumentationUrl(DOCUMENTATION_URL); - final ConnectorSpecification outdatedSpec = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of( - "foo1", - "bar1"))); - final DockerImageSpec outdatedDockerImageSpec = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1) - .withSpec(outdatedSpec); - - final JsonNode seedDefinitions = Jsons.jsonNode(List.of(sourceDefinition)); - final JsonNode seedSpecs = Jsons.jsonNode(List.of(outdatedDockerImageSpec)); - - final ConnectorSpecification newSpec = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))); - final DockerImageSpec newDockerImageSpec = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG2).withSpec(newSpec); - - when(bucketSpecFetcherMock.attemptFetch(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG2)).thenReturn(Optional.of(newSpec)); - - final List actualSeedSpecs = seedConnectorSpecGenerator.fetchUpdatedSeedSpecs(seedDefinitions, seedSpecs); - final List expectedSeedSpecs = List.of(newDockerImageSpec); - - assertEquals(expectedSeedSpecs, actualSeedSpecs); - } - - @Test - void testExtraneousSpecIsRemoved() { - final StandardDestinationDefinition sourceDefinition = new StandardDestinationDefinition() - .withDestinationDefinitionId(DEF_ID1) - .withDockerRepository(DOCKER_REPOSITORY1) - .withDockerImageTag(DOCKER_TAG1) - .withName(CONNECTOR_NAME1) - .withDocumentationUrl(DOCUMENTATION_URL); - final ConnectorSpecification spec1 = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo1", "bar1"))); - final DockerImageSpec dockerImageSpec1 = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1).withSpec(spec1); - - final ConnectorSpecification spec2 = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2"))); - final DockerImageSpec dockerImageSpec2 = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY2 + ":" + DOCKER_TAG2).withSpec(spec2); - - final JsonNode seedDefinitions = Jsons.jsonNode(List.of(sourceDefinition)); - final JsonNode seedSpecs = Jsons.jsonNode(Arrays.asList(dockerImageSpec1, dockerImageSpec2)); - - final List actualSeedSpecs = seedConnectorSpecGenerator.fetchUpdatedSeedSpecs(seedDefinitions, seedSpecs); - final List expectedSeedSpecs = List.of(dockerImageSpec1); - - assertEquals(expectedSeedSpecs, actualSeedSpecs); - } - - @Test - void testNoFetchIsPerformedIfAllSpecsUpToDate() { - final StandardDestinationDefinition sourceDefinition = new StandardDestinationDefinition() - .withDestinationDefinitionId(DEF_ID1) - .withDockerRepository(DOCKER_REPOSITORY1) - .withDockerImageTag(DOCKER_TAG1) - .withName(CONNECTOR_NAME1) - .withDocumentationUrl(DOCUMENTATION_URL); - final ConnectorSpecification spec = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar"))); - final DockerImageSpec dockerImageSpec = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1).withSpec(spec); - - final JsonNode seedDefinitions = Jsons.jsonNode(List.of(sourceDefinition)); - final JsonNode seedSpecs = Jsons.jsonNode(List.of(dockerImageSpec)); - - final List actualSeedSpecs = seedConnectorSpecGenerator.fetchUpdatedSeedSpecs(seedDefinitions, seedSpecs); - final List expectedSeedSpecs = List.of(dockerImageSpec); - - assertEquals(expectedSeedSpecs, actualSeedSpecs); - verify(bucketSpecFetcherMock, never()).attemptFetch(any()); - } - -} diff --git a/airbyte-json-validation/build.gradle b/airbyte-json-validation/build.gradle index 2114a4a053e8..4881fc079a46 100644 --- a/airbyte-json-validation/build.gradle +++ b/airbyte-json-validation/build.gradle @@ -6,6 +6,4 @@ dependencies { implementation 'com.networknt:json-schema-validator:1.0.42' // needed so that we can follow $ref when parsing json. jackson does not support this natively. implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' - - implementation project(':airbyte-commons') } diff --git a/airbyte-protocol/models/build.gradle b/airbyte-protocol/models/build.gradle index 85f8d48cac5a..e4199332b848 100644 --- a/airbyte-protocol/models/build.gradle +++ b/airbyte-protocol/models/build.gradle @@ -7,8 +7,6 @@ plugins { dependencies { implementation 'javax.validation:validation-api:1.1.0.Final' implementation 'org.apache.commons:commons-lang3:3.11' - - implementation project(':airbyte-commons') } jsonSchema2Pojo { diff --git a/airbyte-scheduler/client/build.gradle b/airbyte-scheduler/client/build.gradle index 5e319c0418ef..d90a0262c97c 100644 --- a/airbyte-scheduler/client/build.gradle +++ b/airbyte-scheduler/client/build.gradle @@ -5,7 +5,6 @@ plugins { dependencies { implementation project(':airbyte-config:models') implementation project(':airbyte-config:persistence') - implementation project(':airbyte-config:specs') implementation project(':airbyte-json-validation') implementation project(':airbyte-protocol:models') implementation project(':airbyte-scheduler:models') diff --git a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClient.java b/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClient.java index bcdc972c2cb4..a615643d0830 100644 --- a/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClient.java +++ b/airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClient.java @@ -4,17 +4,27 @@ package io.airbyte.scheduler.client; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.api.client.util.Preconditions; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.DestinationConnection; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardCheckConnectionOutput; -import io.airbyte.config.specs.GcsBucketSpecFetcher; import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteProtocolSchema; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.validation.json.JsonSchemaValidator; +import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.Optional; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,15 +33,17 @@ public class BucketSpecCacheSchedulerClient implements SynchronousSchedulerClien private static final Logger LOGGER = LoggerFactory.getLogger(BucketSpecCacheSchedulerClient.class); private final SynchronousSchedulerClient client; - private final GcsBucketSpecFetcher bucketSpecFetcher; + private final Function> bucketSpecFetcher; public BucketSpecCacheSchedulerClient(final SynchronousSchedulerClient client, final String bucketName) { - this.client = client; - this.bucketSpecFetcher = new GcsBucketSpecFetcher(StorageOptions.getDefaultInstance().getService(), bucketName); + this( + client, + dockerImage -> attemptToFetchSpecFromBucket(StorageOptions.getDefaultInstance().getService(), bucketName, dockerImage)); } @VisibleForTesting - BucketSpecCacheSchedulerClient(final SynchronousSchedulerClient client, final GcsBucketSpecFetcher bucketSpecFetcher) { + BucketSpecCacheSchedulerClient(final SynchronousSchedulerClient client, + final Function> bucketSpecFetcher) { this.client = client; this.bucketSpecFetcher = bucketSpecFetcher; } @@ -60,7 +72,7 @@ public SynchronousResponse createGetSpecJob(final String Optional cachedSpecOptional; // never want to fail because we could not fetch from off board storage. try { - cachedSpecOptional = bucketSpecFetcher.attemptFetch(dockerImage); + cachedSpecOptional = bucketSpecFetcher.apply(dockerImage); LOGGER.debug("Spec bucket cache: Call to cache did not fail."); } catch (final RuntimeException e) { cachedSpecOptional = Optional.empty(); @@ -76,4 +88,38 @@ public SynchronousResponse createGetSpecJob(final String } } + private static void validateConfig(final JsonNode json) throws JsonValidationException { + final JsonSchemaValidator jsonSchemaValidator = new JsonSchemaValidator(); + final JsonNode specJsonSchema = JsonSchemaValidator.getSchema(AirbyteProtocolSchema.PROTOCOL.getFile(), "ConnectorSpecification"); + jsonSchemaValidator.ensure(specJsonSchema, json); + } + + public static Optional attemptToFetchSpecFromBucket(final Storage storage, + final String bucketName, + final String dockerImage) { + final String[] dockerImageComponents = dockerImage.split(":"); + Preconditions.checkArgument(dockerImageComponents.length == 2, "Invalidate docker image: " + dockerImage); + final String dockerImageName = dockerImageComponents[0]; + final String dockerImageTag = dockerImageComponents[1]; + + final Path specPath = Path.of("specs").resolve(dockerImageName).resolve(dockerImageTag).resolve("spec.json"); + LOGGER.debug("Checking path for cached spec: {} {}", bucketName, specPath); + final Blob specAsBlob = storage.get(bucketName, specPath.toString()); + + // if null it means the object was not found. + if (specAsBlob == null) { + LOGGER.debug("Spec not found in bucket storage"); + return Optional.empty(); + } + + final String specAsString = new String(specAsBlob.getContent(), StandardCharsets.UTF_8); + try { + validateConfig(Jsons.deserialize(specAsString)); + } catch (final JsonValidationException e) { + LOGGER.error("Received invalid spec from bucket store. {}", e.toString()); + return Optional.empty(); + } + return Optional.of(Jsons.deserialize(specAsString, ConnectorSpecification.class)); + } + } diff --git a/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClientTest.java b/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClientTest.java index 01f4595b9468..cf21fd2b160d 100644 --- a/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClientTest.java +++ b/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/BucketSpecCacheSchedulerClientTest.java @@ -10,10 +10,10 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import io.airbyte.config.specs.GcsBucketSpecFetcher; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; import java.util.Optional; +import java.util.function.Function; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -21,18 +21,18 @@ class BucketSpecCacheSchedulerClientTest { private SynchronousSchedulerClient defaultClientMock; - private GcsBucketSpecFetcher bucketSpecFetcherMock; + private Function> bucketSpecFetcherMock; @SuppressWarnings("unchecked") @BeforeEach void setup() { defaultClientMock = mock(SynchronousSchedulerClient.class); - bucketSpecFetcherMock = mock(GcsBucketSpecFetcher.class); + bucketSpecFetcherMock = mock(Function.class); } @Test void testGetsSpecIfPresent() throws IOException { - when(bucketSpecFetcherMock.attemptFetch("source-pokeapi:0.1.0")).thenReturn(Optional.of(new ConnectorSpecification())); + when(bucketSpecFetcherMock.apply("source-pokeapi:0.1.0")).thenReturn(Optional.of(new ConnectorSpecification())); final BucketSpecCacheSchedulerClient client = new BucketSpecCacheSchedulerClient(defaultClientMock, bucketSpecFetcherMock); assertEquals(new ConnectorSpecification(), client.createGetSpecJob("source-pokeapi:0.1.0").getOutput()); verifyNoInteractions(defaultClientMock); @@ -40,7 +40,7 @@ void testGetsSpecIfPresent() throws IOException { @Test void testCallsDelegateIfNotPresent() throws IOException { - when(bucketSpecFetcherMock.attemptFetch("source-pokeapi:0.1.0")).thenReturn(Optional.empty()); + when(bucketSpecFetcherMock.apply("source-pokeapi:0.1.0")).thenReturn(Optional.empty()); when(defaultClientMock.createGetSpecJob("source-pokeapi:0.1.0")) .thenReturn(new SynchronousResponse<>(new ConnectorSpecification(), mock(SynchronousJobMetadata.class))); final BucketSpecCacheSchedulerClient client = new BucketSpecCacheSchedulerClient(defaultClientMock, bucketSpecFetcherMock); @@ -49,7 +49,7 @@ void testCallsDelegateIfNotPresent() throws IOException { @Test void testCallsDelegateIfException() throws IOException { - when(bucketSpecFetcherMock.attemptFetch("source-pokeapi:0.1.0")).thenThrow(new RuntimeException("induced exception")); + when(bucketSpecFetcherMock.apply("source-pokeapi:0.1.0")).thenThrow(new RuntimeException("induced exception")); when(defaultClientMock.createGetSpecJob("source-pokeapi:0.1.0")) .thenReturn(new SynchronousResponse<>(new ConnectorSpecification(), mock(SynchronousJobMetadata.class))); final BucketSpecCacheSchedulerClient client = new BucketSpecCacheSchedulerClient(defaultClientMock, bucketSpecFetcherMock); @@ -62,7 +62,7 @@ void testCallsDelegateIfException() throws IOException { @Disabled @Test void testGetsSpecFromBucket() throws IOException { - when(bucketSpecFetcherMock.attemptFetch("source-pokeapi:0.1.0")).thenReturn(Optional.of(new ConnectorSpecification())); + when(bucketSpecFetcherMock.apply("source-pokeapi:0.1.0")).thenReturn(Optional.of(new ConnectorSpecification())); // todo (cgardens) - replace with prod bucket. final BucketSpecCacheSchedulerClient client = new BucketSpecCacheSchedulerClient(defaultClientMock, "cg-specs"); final ConnectorSpecification actualSpec = client.createGetSpecJob("source-pokeapi:0.1.0").getOutput(); diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index a38db0edc5db..edc7c55fb250 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -66,7 +66,6 @@ dependencies { implementation project(':airbyte-config:init') implementation project(':airbyte-config:models') implementation project(':airbyte-config:persistence') - implementation project(':airbyte-config:specs') implementation project(':airbyte-db:lib') implementation project(":airbyte-json-validation") implementation project(':airbyte-migration') diff --git a/build.gradle b/build.gradle index 39d9957cd26b..271b27c23feb 100644 --- a/build.gradle +++ b/build.gradle @@ -73,8 +73,7 @@ def createSpotlessTarget = { pattern -> 'normalization_test_output', 'tools', 'secrets', - 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - 'resources/seed/*_specs.yaml' + 'charts' // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. ] if (System.getenv().containsKey("SUB_BUILD")) { diff --git a/settings.gradle b/settings.gradle index 44baedba5f04..4d5b7d79afa0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,7 +42,6 @@ include ':airbyte-workers' // reused by acceptance tests in connector base. include ':airbyte-analytics' // transitively used by airbyte-workers. include ':airbyte-config:init' // transitively used by airbyte-workers. include ':airbyte-config:persistence' // transitively used by airbyte-workers. -include ':airbyte-config:specs' // transitively used by airbyte-workers. include ':airbyte-db:jooq' // transitively used by airbyte-workers. include ':airbyte-notification' // transitively used by airbyte-workers. include ':airbyte-scheduler:models' // transitively used by airbyte-workers.