🎉 New Source: Elasticsearch (#14118)

* New source Elasticsearch * Update README.md * Update tests * File additions and changes in test * Deleting spec.json * File additions and changes in test * Fixed multiple streams bug in discover * run format * format files * eof and update doc * correct spec test * auto-bump connector version Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jul 12, 2022 · bfa54ac · bfa54ac
1 parent 767992e
commit bfa54ac
Show file tree

Hide file tree

Showing 27 changed files with 1,574 additions and 0 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1113,3 +1113,10 @@
   documentationUrl: https://docs.airbyte.io/integrations/sources/firebolt
   sourceType: database
   releaseStage: alpha
+- name: Elasticsearch
+  sourceDefinitionId: 7cf88806-25f5-4e1a-b422-b2fa9e1b0090
+  dockerRepository: airbyte/source-elasticsearch
+  dockerImageTag: 0.1.0
+  documentationUrl: https://docs.airbyte.io/integrations/sources/elasticsearch
+  sourceType: api
+  releaseStage: alpha
diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -10450,3 +10450,78 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
+- dockerImage: "airbyte/source-elasticsearch:0.1.0"
+  spec:
+    documentationUrl: "https://docs.airbyte.io/integrations/source/elasticsearch"
+    connectionSpecification:
+      $schema: "http://json-schema.org/draft-07/schema#"
+      title: "Elasticsearch Connection Configuration"
+      type: "object"
+      required:
+      - "endpoint"
+      additionalProperties: false
+      properties:
+        endpoint:
+          title: "Server Endpoint"
+          type: "string"
+          description: "The full url of the Elasticsearch server"
+        authenticationMethod:
+          title: "Authentication Method"
+          type: "object"
+          description: "The type of authentication to be used"
+          oneOf:
+          - title: "None"
+            additionalProperties: false
+            description: "No authentication will be used"
+            required:
+            - "method"
+            properties:
+              method:
+                type: "string"
+                const: "none"
+          - title: "Api Key/Secret"
+            additionalProperties: false
+            description: "Use a api key and secret combination to authenticate"
+            required:
+            - "method"
+            - "apiKeyId"
+            - "apiKeySecret"
+            properties:
+              method:
+                type: "string"
+                const: "secret"
+              apiKeyId:
+                title: "API Key ID"
+                description: "The Key ID to used when accessing an enterprise Elasticsearch\
+                  \ instance."
+                type: "string"
+              apiKeySecret:
+                title: "API Key Secret"
+                description: "The secret associated with the API Key ID."
+                type: "string"
+                airbyte_secret: true
+          - title: "Username/Password"
+            additionalProperties: false
+            description: "Basic auth header with a username and password"
+            required:
+            - "method"
+            - "username"
+            - "password"
+            properties:
+              method:
+                type: "string"
+                const: "basic"
+              username:
+                title: "Username"
+                description: "Basic auth username to access a secure Elasticsearch\
+                  \ server"
+                type: "string"
+              password:
+                title: "Password"
+                description: "Basic auth password to access a secure Elasticsearch\
+                  \ server"
+                type: "string"
+                airbyte_secret: true
+    supportsNormalization: false
+    supportsDBT: false
+    supported_destination_sync_modes: []
diff --git a/airbyte-integrations/connectors/source-elasticsearch/.dockerignore b/airbyte-integrations/connectors/source-elasticsearch/.dockerignore
@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build
diff --git a/airbyte-integrations/connectors/source-elasticsearch/Dockerfile b/airbyte-integrations/connectors/source-elasticsearch/Dockerfile
@@ -0,0 +1,21 @@
+FROM airbyte/integration-base-java:dev AS build
+
+WORKDIR /airbyte
+
+ENV APPLICATION source-elasticsearch
+
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+
+RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar
+
+FROM airbyte/integration-base-java:dev
+
+WORKDIR /airbyte
+
+ENV APPLICATION source-elasticsearch
+ENV ENABLE_SENTRY true
+
+COPY --from=build /airbyte /airbyte
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-elasticsearch
diff --git a/airbyte-integrations/connectors/source-elasticsearch/README.md b/airbyte-integrations/connectors/source-elasticsearch/README.md
@@ -0,0 +1,70 @@
+# Elasticsearch source
+
+This is the repository for the Elasticsearch source connector, written in Java using Elasticsearch's High Level Rest Client([HLRC](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html)).
+For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/elasticsearch).
+
+## Local development
+
+#### Building via Gradle
+From the Airbyte repository root, run:
+```
+./gradlew :airbyte-integrations:connectors:source-elasticsearch:build
+```
+
+#### Create credentials
+Credentials can be provided in three ways:
+1. Basic
+2. 
+
+### Locally running the connector docker image
+
+#### Build
+Build the connector image via Gradle:
+```
+./gradlew :airbyte-integrations:connectors:source-elasticsearch:airbyteDocker
+```
+When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
+the Dockerfile.
+
+#### Run
+Then run any of the connector commands as follows:
+```
+docker run --rm airbyte/source-elasticsearch:dev spec
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-elasticsearch:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-elasticsearch:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-elasticsearch:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+```
+
+#### Sync Mode Support
+Current version of this connector only allows the `FULL REFRESH` mode.
+
+## Testing
+We use `JUnit` for Java tests.
+
+### Unit and Integration Tests
+Place unit tests under `src/test/io/airbyte/integrations/sources/elasticsearch-test`.
+
+#### Acceptance Tests
+Airbyte has a standard test suite that all destination connectors must pass. See example(s) in
+`src/test-integration/java/io/airbyte/integrations/sources/elasticsearch/`.
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unit tests:
+```
+./gradlew :airbyte-integrations:connectors:sources-elasticsearch:unitTest
+```
+To run acceptance and custom integration tests:
+```
+./gradlew :airbyte-integrations:connectors:sources-elasticsearch:integrationTest
+```
+
+## Dependency Management
+
+### Publishing a new version of the connector
+You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
+1. Make sure your changes are passing unit and integration tests.
+2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)).
+3. Create a Pull Request.
+4. Pat yourself on the back for being an awesome contributor.
+5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
diff --git a/airbyte-integrations/connectors/source-elasticsearch/acceptance-test-config.yml b/airbyte-integrations/connectors/source-elasticsearch/acceptance-test-config.yml
@@ -0,0 +1,6 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-elasticsearch
+tests:
+  spec:
+    - spec_path: "src/test/resources/expected_spec.json"
diff --git a/airbyte-integrations/connectors/source-elasticsearch/build.gradle b/airbyte-integrations/connectors/source-elasticsearch/build.gradle
@@ -0,0 +1,48 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.source.elasticsearch.ElasticsearchSource'
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+}
+
+dependencies {
+    implementation project(':airbyte-config:config-models')
+    implementation project(':airbyte-protocol:protocol-models')
+    implementation project(':airbyte-integrations:bases:base-java')
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+
+    implementation 'co.elastic.clients:elasticsearch-java:7.15.0'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3'
+
+    // EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+    // https://eclipse-ee4j.github.io/jsonp/
+    implementation 'jakarta.json:jakarta.json-api:2.0.1'
+
+    // Needed even if using Jackson to have an implementation of the Jsonp object model
+    // EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+    // https://github.com/eclipse-ee4j/jsonp
+    implementation 'org.glassfish:jakarta.json:2.0.1'
+
+    //  MIT
+    //  https://www.testcontainers.org/
+    testImplementation libs.connectors.testcontainers.elasticsearch
+    integrationTestJavaImplementation libs.connectors.testcontainers.elasticsearch
+
+    integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
+    integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-elasticsearch')
+}
+
+repositories {
+    maven {
+        name = "ESSnapshots"
+        url = "https://snapshots.elastic.co/maven/"
+    }
+    maven {
+        name = "ESJavaGithubPackages"
+        url = "https://maven.pkg.github.com/elastic/elasticsearch-java"
+    }
+}