diff --git a/airbyte-integrations/connector-templates/generator/package.json b/airbyte-integrations/connector-templates/generator/package.json index 0a4ae5ac873f..f7bd9559da45 100644 --- a/airbyte-integrations/connector-templates/generator/package.json +++ b/airbyte-integrations/connector-templates/generator/package.json @@ -7,6 +7,7 @@ }, "devDependencies": { "capital-case": "^1.0.4", + "change-case": "^4.1.2", "handlebars": "^4.7.7", "plop": "^3.0.5", "set-value": ">=4.0.1", diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index 86c5f2157552..d9914238b4a4 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -2,7 +2,8 @@ const path = require('path'); const uuid = require('uuid'); const capitalCase = require('capital-case'); - +const changeCase = require('change-case') + const getSuccessMessage = function(connectorName, outputPath, additionalMessage){ return ` 🚀 🚀 🚀 🚀 🚀 🚀 @@ -27,6 +28,8 @@ module.exports = function (plop) { const docRoot = '../../../docs/integrations'; const definitionRoot = '../../../airbyte-config/init/src/main/resources'; + const sourceAcceptanceTestFilesInputRoot = '../source_acceptance_test_files'; + const pythonSourceInputRoot = '../source-python'; const singerSourceInputRoot = '../source-singer'; const genericSourceInputRoot = '../source-generic'; @@ -43,11 +46,40 @@ module.exports = function (plop) { const httpApiOutputRoot = `${outputDir}/source-{{dashCase name}}`; const javaDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`; const pythonDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`; + const sourceConnectorImagePrefix = 'airbyte/source-' + const sourceConnectorImageTag = 'dev' + const defaultSpecPathFolderPrefix = 'source_' + const specFileName = 'spec.yaml' + plop.setHelper('capitalCase', function(name) { return capitalCase.capitalCase(name); }); + plop.setHelper('connectorImage', function() { + let suffix = "" + if (typeof this.connectorImageNameSuffix !== 'undefined') { + suffix = this.connectorImageNameSuffix + } + return `${sourceConnectorImagePrefix}${changeCase.paramCase(this.name)}${suffix}:${sourceConnectorImageTag}` + }); + + plop.setHelper('specPath', function() { + let suffix = "" + if (typeof this.specPathFolderSuffix !== 'undefined') { + suffix = this.specPathFolderSuffix + } + let inSubFolder = true + if (typeof this.inSubFolder !== 'undefined') { + inSubFolder = this.inSubFolder + } + if (inSubFolder) { + return `${defaultSpecPathFolderPrefix}${changeCase.snakeCase(this.name)}${suffix}/${specFileName}` + } else { + return specFileName + } + }); + plop.setActionType('emitSuccess', function(answers, config, plopApi){ console.log(getSuccessMessage(answers.name, plopApi.renderString(config.outputPath, answers), config.message)); }); @@ -86,6 +118,14 @@ module.exports = function (plop) { base: httpApiInputRoot, templateFiles: `${httpApiInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: httpApiOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + }, // plop doesn't add dotfiles by default so we manually add them { type:'add', @@ -113,6 +153,18 @@ module.exports = function (plop) { base: singerSourceInputRoot, templateFiles: `${singerSourceInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: singerSourceOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + data: { + connectorImageNameSuffix: "-singer", + specPathFolderSuffix: "_singer" + } + }, { type:'add', abortOnFail: true, @@ -140,6 +192,14 @@ module.exports = function (plop) { base: pythonSourceInputRoot, templateFiles: `${pythonSourceInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: pythonSourceOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + }, { type:'add', abortOnFail: true, @@ -175,6 +235,17 @@ module.exports = function (plop) { base: genericSourceInputRoot, templateFiles: `${genericSourceInputRoot}/**/**`, }, + // common acceptance tests + { + abortOnFail: true, + type:'addMany', + destination: genericSourceOutputRoot, + base: sourceAcceptanceTestFilesInputRoot, + templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`, + data: { + inSubFolder: false + } + }, {type: 'emitSuccess', outputPath: genericSourceOutputRoot} ] }); diff --git a/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml b/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml deleted file mode 100644 index 4d2eeb3e00e6..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml +++ /dev/null @@ -1,25 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev -tests: - spec: - - spec_path: "spec.json" - config_path: "secrets/valid_config.json" # TODO add this file - connection: - - config_path: "secrets/valid_config.json" # TODO add this file - status: "succeed" - - config_path: "secrets/invalid_config.json" # TODO add this file - status: "failed" - discovery: - - config_path: "secrets/valid_config.json" - basic_read: - - config_path: "secrets/valid_config.json" - configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file - empty_streams: [] - full_refresh: - - config_path: "secrets/valid_config.json" - configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file -# incremental: # TODO uncomment this once you implement incremental sync -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-config.yml.hbs deleted file mode 100644 index 49acab015f3f..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-config.yml.hbs +++ /dev/null @@ -1,30 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev -tests: - spec: - - spec_path: "source_{{snakeCase name}}/spec.yaml" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connector-templates/source-python/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-python/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690..000000000000 --- a/airbyte-integrations/connector-templates/source-python/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connector-templates/source-singer/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source-singer/acceptance-test-config.yml.hbs deleted file mode 100644 index f485a8c6460d..000000000000 --- a/airbyte-integrations/connector-templates/source-singer/acceptance-test-config.yml.hbs +++ /dev/null @@ -1,30 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}-singer:dev -tests: - spec: - - spec_path: "source_{{snakeCase name}}_singer/spec.yaml" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "exception" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connector-templates/source-singer/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-singer/acceptance-test-docker.sh deleted file mode 100644 index e4d8b1cef896..000000000000 --- a/airbyte-integrations/connector-templates/source-singer/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs similarity index 92% rename from airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs rename to airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs index 2ec2ab2694f8..3981383e5d76 100644 --- a/airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs +++ b/airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs @@ -1,9 +1,9 @@ # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev +connector_image: {{ connectorImage }} tests: spec: - - spec_path: "source_{{snakeCase name}}/spec.yaml" + - spec_path: "{{ specPath }}" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-docker.sh similarity index 100% rename from airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh rename to airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-docker.sh diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml index a625390b4d5e..97cf9c7d8e1c 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml @@ -15,12 +15,12 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes incremental: # TODO if your connector does not implement incremental sync, remove this block - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json"