CDK: DRY acceptance test templates (#12067)

airbyte-integrations/connector-templates/generator/package.json

@@ -7,6 +7,7 @@
   },
   "devDependencies": {
     "capital-case": "^1.0.4",
+    "change-case": "^4.1.2",
     "handlebars": "^4.7.7",
     "plop": "^3.0.5",
     "set-value": ">=4.0.1",

airbyte-integrations/connector-templates/generator/plopfile.js

@@ -2,7 +2,8 @@
 const path = require('path');
 const uuid = require('uuid');
 const capitalCase = require('capital-case');
-
+const changeCase = require('change-case')
+
 const getSuccessMessage = function(connectorName, outputPath, additionalMessage){
     return `
 🚀 🚀 🚀 🚀 🚀 🚀
@@ -27,6 +28,8 @@ module.exports = function (plop) {
   const docRoot = '../../../docs/integrations';
   const definitionRoot = '../../../airbyte-config/init/src/main/resources';
 
+  const sourceAcceptanceTestFilesInputRoot = '../source_acceptance_test_files';
+
   const pythonSourceInputRoot = '../source-python';
   const singerSourceInputRoot = '../source-singer';
   const genericSourceInputRoot = '../source-generic';
@@ -43,11 +46,40 @@ module.exports = function (plop) {
   const httpApiOutputRoot = `${outputDir}/source-{{dashCase name}}`;
   const javaDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`;
   const pythonDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`;
+  const sourceConnectorImagePrefix = 'airbyte/source-'
+  const sourceConnectorImageTag = 'dev'
+  const defaultSpecPathFolderPrefix = 'source_'
+  const specFileName = 'spec.yaml'
+
 
   plop.setHelper('capitalCase', function(name) {
     return capitalCase.capitalCase(name);
   });
 
+  plop.setHelper('connectorImage', function() {
+    let suffix = ""
+    if (typeof this.connectorImageNameSuffix !== 'undefined') {
+      suffix = this.connectorImageNameSuffix
+    }
+    return `${sourceConnectorImagePrefix}${changeCase.paramCase(this.name)}${suffix}:${sourceConnectorImageTag}`
+  });
+
+  plop.setHelper('specPath', function() {
+    let suffix = ""
+    if (typeof this.specPathFolderSuffix !== 'undefined') {
+      suffix = this.specPathFolderSuffix
+    }
+    let inSubFolder = true
+    if (typeof this.inSubFolder !== 'undefined') {
+      inSubFolder = this.inSubFolder
+    }
+    if (inSubFolder) {
+      return `${defaultSpecPathFolderPrefix}${changeCase.snakeCase(this.name)}${suffix}/${specFileName}`
+    } else {
+      return specFileName
+    }
+  });
+
   plop.setActionType('emitSuccess', function(answers, config, plopApi){
       console.log(getSuccessMessage(answers.name, plopApi.renderString(config.outputPath, answers), config.message));
   });
@@ -86,6 +118,14 @@ module.exports = function (plop) {
         base: httpApiInputRoot,
         templateFiles: `${httpApiInputRoot}/**/**`,
       },
+      // common acceptance tests
+      {
+        abortOnFail: true,
+        type:'addMany',
+        destination: httpApiOutputRoot,
+        base: sourceAcceptanceTestFilesInputRoot,
+        templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`,
+      },
       // plop doesn't add dotfiles by default so we manually add them
       {
         type:'add',
@@ -113,6 +153,18 @@ module.exports = function (plop) {
          base: singerSourceInputRoot,
          templateFiles: `${singerSourceInputRoot}/**/**`,
        },
+       // common acceptance tests
+       {
+         abortOnFail: true,
+         type:'addMany',
+         destination: singerSourceOutputRoot,
+         base: sourceAcceptanceTestFilesInputRoot,
+         templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`,
+         data: {
+          connectorImageNameSuffix: "-singer",
+          specPathFolderSuffix: "_singer"
+        }
+       },
        {
          type:'add',
          abortOnFail: true,
@@ -140,6 +192,14 @@ module.exports = function (plop) {
                 base: pythonSourceInputRoot,
                 templateFiles: `${pythonSourceInputRoot}/**/**`,
             },
+            // common acceptance tests
+            {
+              abortOnFail: true,
+              type:'addMany',
+              destination: pythonSourceOutputRoot,
+              base: sourceAcceptanceTestFilesInputRoot,
+              templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`,
+            },
             {
                 type:'add',
                 abortOnFail: true,
@@ -175,6 +235,17 @@ module.exports = function (plop) {
           base: genericSourceInputRoot,
           templateFiles: `${genericSourceInputRoot}/**/**`,
         },
+        // common acceptance tests
+        {
+          abortOnFail: true,
+          type:'addMany',
+          destination: genericSourceOutputRoot,
+          base: sourceAcceptanceTestFilesInputRoot,
+          templateFiles: `${sourceAcceptanceTestFilesInputRoot}/**/**`,
+          data: {
+            inSubFolder: false
+          }
+        },
         {type: 'emitSuccess', outputPath: genericSourceOutputRoot}
       ]
     });

airbyte-integrations/connector-templates/source-python/acceptance-test-config.yml.hbs → airbyte-integrations/connector-templates/source_acceptance_test_files/acceptance-test-config.yml.hbs

@@ -1,9 +1,9 @@
 # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
-connector_image: airbyte/source-{{dashCase name}}:dev
+connector_image: {{ connectorImage }}
 tests:
   spec:
-    - spec_path: "source_{{snakeCase name}}/spec.yaml"
+    - spec_path: "{{ specPath }}"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"

airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml

@@ -15,12 +15,12 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       empty_streams: []
-# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
-#      expect_records:
-#        path: "integration_tests/expected_records.txt"
-#        extra_fields: no
-#        exact_order: no
-#        extra_records: yes
+  # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
+  #      expect_records:
+  #        path: "integration_tests/expected_records.txt"
+  #        extra_fields: no
+  #        exact_order: no
+  #        extra_records: yes
   incremental: # TODO if your connector does not implement incremental sync, remove this block
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"