From 2c23a5e975ba8596a4e1520129b0eff239fdc076 Mon Sep 17 00:00:00 2001 From: Mercari OSS Bot <92021976+mercari-oss-bot@users.noreply.github.com> Date: Fri, 18 Mar 2022 14:23:51 +0900 Subject: [PATCH] Initial commit --- .github/workflows/publish.yaml | 25 + .github/workflows/release.yaml | 29 ++ .github/workflows/settings.xml | 42 ++ .github/workflows/test.yaml | 19 + .gitignore | 6 + LICENSE.txt | 201 +++++++ Makefile | 15 + README.md | 416 +++++++++++++++ .../pom.xml | 80 +++ .../transforms/kryptonite/CipherField.java | 390 ++++++++++++++ .../transforms/kryptonite/DataKeyConfig.java | 82 +++ .../transforms/kryptonite/FieldConfig.java | 106 ++++ .../kryptonite/FieldPathMatcher.java | 29 ++ .../transforms/kryptonite/RecordHandler.java | 143 +++++ .../transforms/kryptonite/SchemaRewriter.java | 357 +++++++++++++ .../kryptonite/SchemaawareRecordHandler.java | 109 ++++ .../kryptonite/SchemalessRecordHandler.java | 81 +++ .../kryptonite/TypeSchemaMapper.java | 81 +++ .../kryptonite/serdes/KryoInstance.java | 44 ++ .../kryptonite/serdes/KryoSerdeProcessor.java | 194 +++++++ .../kryptonite/serdes/SerdeProcessor.java | 28 + .../kryptonite/util/JsonStringReader.java | 128 +++++ .../kryptonite/util/JsonStringWriter.java | 126 +++++ .../validators/CipherDataKeysValidator.java | 74 +++ .../validators/CipherEncodingValidator.java | 47 ++ .../validators/CipherModeValidator.java | 40 ++ .../validators/CipherNameValidator.java | 49 ++ .../validators/FieldConfigValidator.java | 57 ++ .../validators/FieldModeValidator.java | 40 ++ .../validators/KeySourceValidator.java | 39 ++ .../validators/TimeUnitValidator.java | 23 + .../kryptonite/CipherFieldTest.java | 489 ++++++++++++++++++ .../src/test/resources/logback.xml | 15 + kryptonite/pom.xml | 65 +++ .../hpgrahsl/kryptonite/AesGcmNoPadding.java | 59 +++ .../hpgrahsl/kryptonite/CipherMode.java | 22 + .../hpgrahsl/kryptonite/Cipherable.java | 22 + .../kryptonite/ConfigDataKeyVault.java | 47 ++ .../hpgrahsl/kryptonite/CryptoAlgorithm.java | 24 + .../hpgrahsl/kryptonite/DataException.java | 24 + .../hpgrahsl/kryptonite/FieldMetaData.java | 94 ++++ .../kryptonite/GcpKmsKeyStrategy.java | 30 ++ .../kryptonite/GcpSecretManagerKeyVault.java | 90 ++++ .../hpgrahsl/kryptonite/KeyException.java | 40 ++ .../kryptonite/KeyInvalidException.java | 40 ++ .../kryptonite/KeyNotFoundException.java | 40 ++ .../hpgrahsl/kryptonite/KeyStrategy.java | 31 ++ .../github/hpgrahsl/kryptonite/KeyVault.java | 28 + .../hpgrahsl/kryptonite/Kryptonite.java | 94 ++++ .../hpgrahsl/kryptonite/NoOpKeyStrategy.java | 25 + kryptonite/src/test/resources/logback.xml | 15 + pom.xml | 206 ++++++++ 52 files changed, 4600 insertions(+) create mode 100644 .github/workflows/publish.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/settings.xml create mode 100644 .github/workflows/test.yaml create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 Makefile create mode 100644 README.md create mode 100644 kafka-connect-transform-kryptonite-gcp/pom.xml create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherField.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/DataKeyConfig.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldConfig.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldPathMatcher.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/RecordHandler.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaRewriter.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaawareRecordHandler.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemalessRecordHandler.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/TypeSchemaMapper.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoInstance.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoSerdeProcessor.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/SerdeProcessor.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringReader.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringWriter.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherDataKeysValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherEncodingValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherModeValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherNameValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldConfigValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldModeValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/KeySourceValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/TimeUnitValidator.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/test/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherFieldTest.java create mode 100644 kafka-connect-transform-kryptonite-gcp/src/test/resources/logback.xml create mode 100644 kryptonite/pom.xml create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/AesGcmNoPadding.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CipherMode.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Cipherable.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/ConfigDataKeyVault.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CryptoAlgorithm.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/DataException.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/FieldMetaData.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpKmsKeyStrategy.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpSecretManagerKeyVault.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyException.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyInvalidException.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyNotFoundException.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyStrategy.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyVault.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Kryptonite.java create mode 100644 kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/NoOpKeyStrategy.java create mode 100644 kryptonite/src/test/resources/logback.xml create mode 100644 pom.xml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..38ecbf9 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,25 @@ +name: publish + +on: + push: + branches: + - master + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'adopt' + - name: Publish snapshot package + run: | + VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version|grep -Ev '(^\[|Download\w+:)') + if [[ ${VERSION} == *SNAPSHOT ]]; then + mvn -s ${{ github.workspace }}/.github/workflows/settings.xml --batch-mode deploy + fi + env: + JFROG_USERNAME: ${{ secrets.JFROG_USERNAME }} + JFROG_PASSWORD: ${{ secrets.JFROG_PASSWORD }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..bccc414 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,29 @@ +name: release + +on: + push: + tags: + - "v*" + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'adopt' + - name: Publish package + run: | + mvn -s ${{ github.workspace }}/.github/workflows/settings.xml --batch-mode deploy + rm -rf */target/original-*.jar + env: + JFROG_USERNAME: ${{ secrets.JFROG_USERNAME }} + JFROG_PASSWORD: ${{ secrets.JFROG_PASSWORD }} + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: "*/target/*.jar" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/settings.xml b/.github/workflows/settings.xml new file mode 100644 index 0000000..2918a7e --- /dev/null +++ b/.github/workflows/settings.xml @@ -0,0 +1,42 @@ + + + + + mercari + + + + + mercari + + + central + https://mercari.jfrog.io/mercari/libs-release + + false + + + + snapshots + https://mercari.jfrog.io/mercari/libs-snapshot + + false + + + + + + + + + central + ${env.JFROG_USERNAME} + ${env.JFROG_PASSWORD} + + + snapshots + ${env.JFROG_USERNAME} + ${env.JFROG_PASSWORD} + + + diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..17c269f --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,19 @@ +name: build + +on: + push: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: "8" + distribution: "adopt" + cache: maven + - name: Build with Maven + run: mvn --batch-mode --update-snapshots verify diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bde9135 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.idea/ +target/ +*.iml +dependency-reduced-pom.xml +.gradle +build diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..336b80c --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3cdda17 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +.PHONY: chk +chk: + mvn com.coveo:fmt-maven-plugin:check + +.PHONY: fmt +fmt: + mvn com.coveo:fmt-maven-plugin:format + +.PHONY: test +test: + mvn verify + +.PHONY: jar +jar: + mvn clean package diff --git a/README.md b/README.md new file mode 100644 index 0000000..06f74a6 --- /dev/null +++ b/README.md @@ -0,0 +1,416 @@ +# Kryptonite - An SMT for Kafka Connect + +Kryptonite is a turn-key ready [transformation](https://kafka.apache.org/documentation/#connect_transforms) (SMT) for [Apache Kafka®](https://kafka.apache.org/) +to do field-level encryption/decryption of records with or without schema in data integration scenarios based on [Kafka Connect](https://kafka.apache.org/documentation/#connect). +It uses authenticated encryption with associated data ([AEAD](https://en.wikipedia.org/wiki/Authenticated_encryption)) and in particular applies +[AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) in [GCM](https://en.wikipedia.org/wiki/Galois/Counter_Mode) mode. + +## tl;dr + +### Data Records without Schema + +The following fictional data **record value without schema** - represented in JSON-encoded format - +is used to illustrate a simple encrypt/decrypt scenario: + +```json5 +{ + "id": "1234567890", + "myString": "some foo bla text", + "myInt": 42, + "myBoolean": true, + "mySubDoc1": {"myString":"hello json"}, + "myArray1": ["str_1","str_2","...","str_N"], + "mySubDoc2": {"k1":9,"k2":8,"k3":7} +} +``` + +#### Encryption of selected fields + +Let's assume the fields `"myString"`,`"myArray1"` and `"mySubDoc2"` of the above data record should get encrypted, +the CipherField SMT can be configured as follows: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.cipher_mode": "ENCRYPT", + "transforms.cipher.cipher_data_keys": "[{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\"}]", // key materials of utmost secrecy! + "transforms.cipher.cipher_data_key_name": "my-demo-secret-key", + "transforms.cipher.cipher_data_key_version": "123", + "transforms.cipher.field_config": "[{\"name\":\"myString\"},{\"name\":\"myArray1\"},{\"name\":\"mySubDoc2\"}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +The result after applying this SMT is a record in which all the fields specified in the `field_config` parameter are +**encrypted using the secret key** specified with the `cipher_data_key_name` and `cipher_data_key_version` parameters. + +If you specify `cipher_data_keys`, then apparently, the **configured key materials have to be treated with utmost secrecy**, for leaking any of the secret keys renders encryption useless. +The recommended way of doing this for now is to indirectly reference secret key materials by externalizing them into a separate properties file. +Read a few details about this [here](#externalize-configuration-parameters). +It is also possible to use the GCP Cloud KMS and Secret Manager. Please see [here](#gcp-integrations) for details. + +Since the configuration parameter `field_mode` is set to 'OBJECT', complex field types are processed as a whole instead of element-wise. + +Below is an exemplary JSON-encoded record after the encryption: + +```json5 +{ + "id": "1234567890", + "myString": "123#OtWbJ+VR6P6i1x9DE4FKOmsV43HOHttUjdufCjrt6SIixILy+6Bk9zBdWC4KCgeN9I2z", + "myInt": 42, + "myBoolean": true, + "mySubDoc1": {"myString":"hello json"}, + "myArray1": "123#uWz9MODqJ0hyzXYaraEZ08S1e78ZOC0G4zeL8eZmISUpMiNsfBLDviBlWrCL2cQRbt3qNGlpKUys7/Lio9OIc0A=", + "mySubDoc2": "123#O0AHEZ8pOccnmBHT/5kJj2QQeke3ltf8i/kJzEo/alB2sOqUooFGThBKDZA0HjdC2zz9thvB8zfjw7+fbfts6/4=" +} +``` + +**NOTE:** Encrypted fields are always represented as **Base64-encoded strings**, +with the **ciphertext** of the field's original values and the version number of the secret key appended to the beginning, separated by **#**. + +#### Decryption of selected fields + +Provided that the secret key material used to encrypt the original data record is made available to a specific sink connector, +the CipherField SMT can be configured to decrypt the data like so: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.cipher_mode": "DECRYPT", + "transforms.cipher.cipher_data_keys": "[{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\"}]", // key materials of utmost secrecy! + "transforms.cipher.cipher_data_key_name": "my-demo-secret-key", + "transforms.cipher.cipher_data_key_version": "123", + "transforms.cipher.field_config": "[{\"name\":\"myString\"},{\"name\":\"myArray1\"},{\"name\":\"mySubDoc2\"}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +The result after applying this SMT is a record in which all the fields specified in the `field_config` parameter are **decrypted +using the secret key version that is specified and was used to encrypt the original data**. + +Below is an exemplary JSON-encoded record after the decryption, which is equal to the original record: + +```json5 +{ + "id": "1234567890", + "myString": "some foo bla text", + "myInt": 42, + "myBoolean": true, + "mySubDoc1": {"myString":"hello json"}, + "myArray1": ["str_1","str_2","...","str_N"], + "mySubDoc2": {"k1":9,"k2":8,"k3":7} +} +``` + +### Data Records with Schema + +The following example is based on an **Avro value record** and used to illustrate a simple encrypt/decrypt scenario for data records +with schema. The schema could be defined as: + +```json5 +{ + "type": "record", "fields": [ + { "name": "id", "type": "string" }, + { "name": "myString", "type": "string" }, + { "name": "myInt", "type": "int" }, + { "name": "myBoolean", "type": "boolean" }, + { "name": "mySubDoc1", "type": "record", + "fields": [ + { "name": "myString", "type": "string" } + ] + }, + { "name": "myArray1", "type": { "type": "array", "items": "string"}}, + { "name": "mySubDoc2", "type": { "type": "map", "values": "int"}} + ] +} +``` + +The data of one such fictional record - represented by its `Struct.toString()` output - might look as: + +```text +Struct{ + id=1234567890, + myString=some foo bla text, + myInt=42, + myBoolean=true, + mySubDoc1=Struct{myString=hello json}, + myArray1=[str_1, str_2, ..., str_N], + mySubDoc2={k1=9, k2=8, k3=7} +} +``` + +#### Encryption of selected fields + +Let's assume the fields `"myString"`,`"myArray1"` and `"mySubDoc2"` of the above data record should get encrypted, +the CipherField SMT can be configured as follows: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.cipher_mode": "ENCRYPT", + "transforms.cipher.cipher_data_keys": "[{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\"}]", // key materials of utmost secrecy! + "transforms.cipher.cipher_data_key_name": "my-demo-secret-key", + "transforms.cipher.cipher_data_key_version": "123", + "transforms.cipher.field_config": "[{\"name\":\"myString\"},{\"name\":\"myArray1\"},{\"name\":\"mySubDoc2\"}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +The result after applying this SMT is a record in which all the fields specified in the `field_config` parameter are +**encrypted using the secret key** specified by its id with the `cipher_data_key_name` and `cipher_data_version` parameters. + +If you specify `cipher_data_keys`, then apparently, the **configured key materials have to be treated with utmost secrecy**, for leaking any of the secret keys renders encryption useless. +The recommended way of doing this for now is to indirectly reference secret key materials by externalizing them into a separate properties file. +Read a few details about this [here](#externalize-configuration-parameters). +It is also possible to use the GCP Cloud KMS and Secret Manager. Please see [here](#gcp-integrations) for details. + +Since the configuration parameter `field_mode` is set to 'OBJECT', complex field types are processed as a whole instead of element-wise. + +Below is an exemplary `Struct.toString()` output of the record after the encryption: + +```text +Struct{ + id=1234567890, + myString=123#OtWbJ+VR6P6i1x9DE4FKOmsV43HOHttUjdufCjrt6SIixILy+6Bk9zBdWC4KCgeN9I2z, + myInt=42, + myBoolean=true, + mySubDoc1=Struct{myString=hello json}, + myArray1=123#uWz9MODqJ0hyzXYaraEZ08S1e78ZOC0G4zeL8eZmISUpMiNsfBLDviBlWrCL2cQRbt3qNGlpKUys7/Lio9OIc0A=, + mySubDoc2=123#O0AHEZ8pOccnmBHT/5kJj2QQeke3ltf8i/kJzEo/alB2sOqUooFGThBKDZA0HjdC2zz9thvB8zfjw7+fbfts6/4= +} +``` + +**NOTE 1:** Encrypted fields are always represented as **Base64-encoded strings**, +with the **ciphertext** of the field's original values and the version number of the secret key appended to the beginning, separated by **#**. + +**NOTE 2:** Obviously, in order to support this **the original schema of the data record is automatically redacted such +that any encrypted fields can be stored as strings**, even though the original data types for the fields in question were different ones. + +#### Decryption of selected fields + +Provided that the secret key material used to encrypt the original data record is made available to a specific sink connector, +the CipherField SMT can be configured to decrypt the data like so: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.cipher_mode": "DECRYPT", + "transforms.cipher.cipher_data_keys": "[{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\"}]", // key materials of utmost secrecy! + "transforms.cipher.cipher_data_key_name": "my-demo-secret-key", + "transforms.cipher.cipher_data_key_version": "123", + "transforms.cipher.field_config": "[{\"name\":\"myString\",\"schema\": {\"type\": \"STRING\"}},{\"name\":\"myArray1\",\"schema\": {\"type\": \"ARRAY\",\"valueSchema\": {\"type\": \"STRING\"}}},{\"name\":\"mySubDoc2\",\"schema\": { \"type\": \"MAP\", \"keySchema\": { \"type\": \"STRING\" }, \"valueSchema\": { \"type\": \"INT32\"}}}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +**Take notice of the extended `field_config` parameter settings.** For decryption of schema-aware data, the SMT configuration expects +that for each field to decrypt the original schema information is explicitly specified. +This allows to **redact the encrypted record's schema towards a compatible decrypted record's schema upfront,** +such that the resulting plaintext field values can be stored in accordance with their original data types. + +The result after applying this SMT is a record in which all the fields specified in the `field_config` parameter are +**decrypted using the secret key id that is specified and was used to encrypt the original data**. + +Below is the decrypted data - represented by its `Struct.toString()` output - which is equal to the original record: + +```text +Struct{ + id=1234567890, + myString=some foo bla text, + myInt=42, + myBoolean=true, + mySubDoc1=Struct{myString=hello json}, + myArray1=[str_1, str_2, ..., str_N], + mySubDoc2={k1=9, k2=8, k3=7} +} +``` + +## Configuration Parameters + +| name | Description | Type | Default | Valid values | Importance | +|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------| +| cipher_data_key_name | secret key name to be used as default data encryption key for all fields which don't refer to a field-specific secret key name | string | | non-empty string | high | +| cipher_data_key_version | secret key version to be used as default data encryption key for all fields which don't refer to a field-specific secret key version | string | | non-empty string | high | +| cipher_data_keys | JSON array with data key objects specifying the key name, key version and base64 encoded key bytes used for encryption / decryption. The key material is mandatory if the key_source=CONFIG | password | | JSON array holding at least one valid data key config object, e.g. | medium | +| cipher_data_key_cache_expiry_duration | defines the expiration duration of the secret key cache
To be used if key_source is GCP_SECRET_MANAGER or GCP_SECRET_MANAGER_WITH_KMS | long | 24 | long value | low | +| cipher_data_key_cache_expiry_duration_unit | defines the unit of expiration duration of the private key cache
To be used if key_source is GCP_SECRET_MANAGER or GCP_SECRET_MANAGER_WITH_KMS | string | HOURS | NANOSECONDS, MICROSECONDS, MILLISECONDS, SECONDS, MINUTES, HOURS, DAYS | low | +| cipher_mode | defines whether the data should get encrypted or decrypted | string | | ENCRYPT or DECRYPT | high | +| field_config | JSON array with field config objects specifying which fields together with their settings should get either encrypted / decrypted (nested field names are expected to be separated by '.' per default, or by a custom 'path_delimiter' config | string | | JSON array holding at least one valid field config object, e.g. [{"name": "my-field-abc"},{"name": "my-nested.field-xyz"}] | high | +| key_source | defines the origin of the secret key material (currently supports keys specified in the config or the GCP Secret Manager) | string | CONFIG | CONFIG or GCP_SECRET_MANAGER or GCP_SECRET_MANAGER_WITH_KMS | medium | +| kms_key_name | The GCP Cloud KMS key name for decrypting a data encryption key (DEK), if the DEK is encrypted with a key encryption key (KEK)
To be used if key_source is GCP_SECRET_MANAGER_WITH_KMS | string | | non-empty string e.g. projects/YOUR_PROJECT/locations/LOCATION/keyRings/YOUR_KEY_RING/cryptoKeys/YOUR_KEY | medium | +| field_mode | defines how to process complex field types (maps, lists, structs), either as full objects or element-wise | string | ELEMENT | ELEMENT or OBJECT | medium | +| cipher_algorithm | cipher algorithm used for data encryption (currently supports only one AEAD cipher: AES/GCM/NoPadding) | string | AES/GCM/NoPadding | AES/GCM/NoPadding | low | +| cipher_text_encoding | defines the encoding of the resulting ciphertext bytes (currently only supports 'base64') | string | base64 | base64 | low | +| path_delimiter | path delimiter used as field name separator when referring to nested fields in the input record | string | . | non-empty string | low | + +### Externalize configuration parameters + +The problem with directly specifying configuration parameters which contain sensitive data, such as secret key materials, +is that they are exposed via Kafka Connect's REST API. +This means for connect clusters that are shared among teams the configured secret key materials would leak, which is of course unacceptable. +The way to deal with this for now, is to indirectly reference such configuration parameters from external property files. + +This approach can be used to configure any kind of sensitive data such as KMS-specific client authentication settings, +in case the secret keys aren't sourced from the config directly but rather retrieved from an external KMS such as Azure Key Vault. + +Below is a quick example of how such a configuration would look like: + +1. Before you can make use of configuration parameters from external sources you have to customize your Kafka Connect worker configuration + by adding the following two settings: + +``` +connect.config.providers=file +connect.config.providers.file.class=org.apache.kafka.common.config.provider.FileConfigProvider +``` + +2. Then you create the external properties file e.g. `classified.properties` which contains the secret key materials. + This file needs to be available on all your Kafka Connect workers which you want to run Kryptonite on. + Let's pretend the file is located at path `/secrets/kryptonite/classified.properties` on your worker nodes: + +```properties +cipher_data_keys=[{"name":"my-demo-secret-key","version":"123","material":"YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE="}] +``` + +3. Finally, you simply reference this file and the corresponding key of the property therein, from your SMT configuration like so: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.cipher_mode": "ENCRYPT", + "transforms.cipher.cipher_data_keys": "${file:/secrets/kryptonite/classified.properties:cipher_data_keys}", + "transforms.cipher.cipher_data_key_name": "my-demo-secret-key-123", + "transforms.cipher.cipher_data_key_version": "123", + "transforms.cipher.field_config": "[{\"name\":\"myString\"},{\"name\":\"myArray1\"},{\"name\":\"mySubDoc2\"}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +In case you want to learn more about configuration parameter externalization there is e.g. this nice +[blog post](https://debezium.io/blog/2019/12/13/externalized-secrets/) from the Debezium team showing +how to externalize username and password settings using a docker-compose example. + +### GCP Integrations + +You can use the GCP Secret Manager to manage your secret keys. +The CipherField SMT can be configured as follows: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.key_source": "GCP_SECRET_MANAGER", + "transforms.cipher.cipher_mode": "ENCRYPT", + "transforms.cipher.cipher_data_key_name": "projects/YOUR_PROJECT_NUMBER/secrets/YOUR_SECRET_NAME", + "transforms.cipher.cipher_data_key_version": "3", + "transforms.cipher.field_config": "[{\"name\":\"myString\"},{\"name\":\"myArray1\"},{\"name\":\"mySubDoc2\"}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +Specify `GCP_SECRET_MANAGER` for `key_source`, and specify the secret name and version of the Secret Manager to be used by default for +`cipher_data_key_name` and `cipher_data_key_version`. It is assumed that the Secret Manager stores base64-encoded secret keys. +It retrieves all valid versions of the secret specified for default use at startup and caches them in memory. +Cache expiration can be set with `cipher_data_key_cache_expiry_duration` and `cipher_data_key_cache_expiry_duration_unit`. +The default is 24 hours. When the cache expires, the secret is evicted and automatically cached again the next time it is accessed. +When encrypting, the default secret version is used, or the matching secret version if specified in `field_config`. +When decrypting, the secret key that matches the version prefix of the encrypted data will be used automatically. +If there is no version number prefix, the default or the secret specified in `field_config` will be used. + +Rotating the secret key is simply a matter of registering a new secret version and updating the secret version used by default. +Since the secret version is automatically selected for decryption, data encrypted with an older version of the secret key can be decrypted, +unless the older version of the secret is disabled. + +Secret keys stored in the Secret Manager can also be encrypted with the Cloud KMS. +Use the Cloud KMS for the key encryption key (KEK) and the Secret Manager for the data encryption key (DEK). +See [Envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) for details. +The CipherField SMT can be configured as follows: + +```json5 +{ + //... + "transforms":"cipher", + "transforms.cipher.type":"com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField$Value", + "transforms.cipher.key_source": "GCP_SECRET_MANAGER_WITH_KMS", + "transforms.cipher.cipher_mode": "ENCRYPT", + "transforms.cipher.kms_key_name": "projects/YOUR_PROJECT/locations/YOUR_LOCATION/keyRings/YOUR_KEY_RING/cryptoKeys/YOUR_KEY", + "transforms.cipher.cipher_data_key_name": "projects/YOUR_PROJECT_NUMBER/secrets/YOUR_SECRET_NAME", + "transforms.cipher.cipher_data_key_version": "3", + "transforms.cipher.field_config": "[{\"name\":\"myString\"},{\"name\":\"myArray1\"},{\"name\":\"mySubDoc2\"}]", + "transforms.cipher.field_mode": "OBJECT", + //... +} +``` + +Specify `GCP_SECRET_MANAGER_WITH_KMS` for `key_source`, and specify the name of the Cloud KMS key for `kms_key_name`. +The basic behavior is the same as when `GCP_SECRET_MANAGER` is specified, +but the Cloud KMS will decrypt the key when it retrieves the key stored in the Secret Manager at startup. + +This can be used when you do not want to store the raw secret key in the Secret Manager. +Also, depending on the configuration, it is possible to automate the key encription key (KEK) rotation. + +### Build, installation / deployment + +This project can be built from source via Maven, or you can download the package from the +GitHub [release page](https://github.com/kouzoh/kafka-connect-transform-kryptonite-gcp/releases). + +In order to deploy it you simply put the jar into a _'plugin path'_ that is configured to be scanned by your Kafka Connect worker nodes. + +After that, configure Kryptonite as transformation for any of your source / sink connectors, sit back and relax! +Happy _'binge watching'_ plenty of ciphertexts ;-) + +### Cipher algorithm specifics + +Kryptonite currently provides a single cipher algorithm, namely, AES in GCM mode. +It offers so-called _authenticated encryption with associated data_ (AEAD). + +By design, every application of Kryptonite on a specific record field results in different ciphertexts for one and the same plaintext. +This is in general not only desirable but very important to make attacks harder. +However, in the context of Kafka Connect records this has an unfavorable consequence for source connectors. +**Applying the SMT on a source record's key would result in a 'partition mix-up'** +because records with the same original plaintext key would end up in different topic partitions. +In other words, **do NOT(!) use Kryptonite for +source record keys** at the moment. There are plans in place to do away with this restriction and extend Kryptonite with a deterministic mode. +This could then safely support the encryption of record keys while at the same time keep topic partitioning and record ordering intact. + +## Contribution + +Please read the CLA carefully before submitting your contribution to Mercari. Under any circumstances, by submitting your contribution, you are deemed to accept and agree to be bound by the terms and conditions of the CLA. + +https://www.mercari.com/cla/ + +## License Information + +This project is licensed according to [Apache License Version 2.0](https://www.apache.org/licenses/LICENSE-2.0) + +``` +Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/kafka-connect-transform-kryptonite-gcp/pom.xml b/kafka-connect-transform-kryptonite-gcp/pom.xml new file mode 100644 index 0000000..1836f0c --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/pom.xml @@ -0,0 +1,80 @@ + + + 4.0.0 + + + com.github.hpgrahsl.kafka.connect + kafka-connect-transform-kryptonite-parent + 0.3.1-SNAPSHOT + + kafka-connect-transform-kryptonite-gcp + jar + + kafka-connect-transform-kryptonite-gcp + + + + com.github.hpgrahsl.kafka.connect + kryptonite + ${project.version} + + + org.apache.kafka + connect-api + ${kafka.version} + provided + + + org.apache.kafka + connect-transforms + ${kafka.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.esotericsoftware + kryo + ${kryo.version} + + + ch.qos.logback + logback-core + ${logback.version} + test + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-shade-plugin + + + maven-surefire-plugin + ${surefire.plugin.version} + + + com.coveo + fmt-maven-plugin + + + + + diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherField.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherField.java new file mode 100644 index 0000000..f8e18ee --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherField.java @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import static org.apache.kafka.connect.transforms.util.Requirements.requireMap; +import static org.apache.kafka.connect.transforms.util.Requirements.requireStruct; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.KryoSerdeProcessor; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.SerdeProcessor; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.CipherDataKeysValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.CipherEncodingValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.CipherModeValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.CipherNameValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.FieldConfigValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.FieldModeValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.KeySourceValidator; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators.TimeUnitValidator; +import com.github.hpgrahsl.kryptonite.CipherMode; +import com.github.hpgrahsl.kryptonite.ConfigDataKeyVault; +import com.github.hpgrahsl.kryptonite.GcpSecretManagerKeyVault; +import com.github.hpgrahsl.kryptonite.Kryptonite; +import com.github.hpgrahsl.kryptonite.NoOpKeyStrategy; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.kafka.common.cache.Cache; +import org.apache.kafka.common.cache.LRUCache; +import org.apache.kafka.common.cache.SynchronizedCache; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.NonEmptyString; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.connect.connector.ConnectRecord; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.transforms.Transformation; +import org.apache.kafka.connect.transforms.util.SimpleConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class CipherField> implements Transformation { + + public enum FieldMode { + ELEMENT, + OBJECT + } + + public enum KeySource { + CONFIG, + GCP_SECRET_MANAGER, + GCP_SECRET_MANAGER_WITH_KMS + } + + public static final String OVERVIEW_DOC = + "Encrypt/Decrypt specified record fields with AEAD cipher." + + "

The transformation should currently only be used for the record value (" + + CipherField.Value.class.getName() + + ")." + + "Future versions will support a dedicated 'mode of operation' applicable also to the record key (" + + CipherField.Key.class.getName() + + ") or value ."; + + public static final String FIELD_CONFIG = "field_config"; + public static final String PATH_DELIMITER = "path_delimiter"; + public static final String FIELD_MODE = "field_mode"; + public static final String CIPHER_ALGORITHM = "cipher_algorithm"; + public static final String CIPHER_DATA_KEY_NAME = "cipher_data_key_name"; + public static final String CIPHER_DATA_KEY_VERSION = "cipher_data_key_version"; + public static final String CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION = + "cipher_data_key_cache_expiry_duration"; + public static final String CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_UNIT = + "cipher_data_key_cache_expiry_duration_unit"; + public static final String CIPHER_DATA_KEYS = "cipher_data_keys"; + public static final String CIPHER_TEXT_ENCODING = "cipher_text_encoding"; + public static final String CIPHER_MODE = "cipher_mode"; + public static final String KEY_SOURCE = "key_source"; + public static final String KMS_KEY_NAME = "kms_key_name"; + + private static final String PATH_DELIMITER_DEFAULT = "."; + private static final String FIELD_MODE_DEFAULT = "ELEMENT"; + private static final String CIPHER_ALGORITHM_DEFAULT = "AES/GCM/NoPadding"; + private static final long CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_DEFAULT = 24L; + private static final String CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_UNIT_DEFAULT = "HOURS"; + private static final String CIPHER_TEXT_ENCODING_DEFAULT = "base64"; + private static final String KEY_SOURCE_DEFAULT = "CONFIG"; + private static final String CIPHER_DATA_KEYS_DEFAULT = "[]"; + private static final String KMS_KEY_NAME_DEFAULT = null; + + public static final ConfigDef CONFIG_DEF = + new ConfigDef() + .define( + FIELD_CONFIG, + Type.STRING, + ConfigDef.NO_DEFAULT_VALUE, + new FieldConfigValidator(), + Importance.HIGH, + "JSON array with field config objects specifying which fields together with their settings " + + "should get either encrypted / decrypted (nested field names are expected to be separated by '.' " + + "per default, or by a custom 'path_delimiter' config") + .define( + PATH_DELIMITER, + Type.STRING, + PATH_DELIMITER_DEFAULT, + new NonEmptyString(), + Importance.LOW, + "path delimiter used as field name separator when referring to nested fields " + + "in the input record") + .define( + FIELD_MODE, + Type.STRING, + FIELD_MODE_DEFAULT, + new FieldModeValidator(), + Importance.MEDIUM, + "defines how to process complex field types (maps, lists, structs), either as full objects " + + "or element-wise") + .define( + CIPHER_ALGORITHM, + Type.STRING, + CIPHER_ALGORITHM_DEFAULT, + new CipherNameValidator(), + Importance.LOW, + "cipher algorithm used for data encryption (currently supports only one AEAD cipher: " + + CIPHER_ALGORITHM_DEFAULT + + ")") + .define( + CIPHER_DATA_KEYS, + Type.PASSWORD, + CIPHER_DATA_KEYS_DEFAULT, + new CipherDataKeysValidator(), + Importance.MEDIUM, + "JSON array with data key objects specifying the key name, key version and base64 encoded " + + "key bytes used for encryption / decryption") + .define( + CIPHER_DATA_KEY_NAME, + Type.STRING, + ConfigDef.NO_DEFAULT_VALUE, + new NonEmptyString(), + Importance.HIGH, + "secret key name to be used as default data encryption key for all fields which don't refer to " + + "a field-specific secret key name") + .define( + CIPHER_DATA_KEY_VERSION, + Type.STRING, + ConfigDef.NO_DEFAULT_VALUE, + new NonEmptyString(), + Importance.HIGH, + "secret key version to be used as default data encryption key for all fields which don't refer to " + + "a field-specific secret key version") + .define( + CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION, + Type.LONG, + CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_DEFAULT, + Importance.LOW, + "defines the expiration duration of the secret key cache") + .define( + CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_UNIT, + Type.STRING, + CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_UNIT_DEFAULT, + new TimeUnitValidator(), + Importance.LOW, + "defines the unit of expiration duration of the private key cache") + .define( + CIPHER_TEXT_ENCODING, + Type.STRING, + CIPHER_TEXT_ENCODING_DEFAULT, + new CipherEncodingValidator(), + ConfigDef.Importance.LOW, + "defines the encoding of the resulting ciphertext bytes (currently only supports 'base64')") + .define( + CIPHER_MODE, + Type.STRING, + ConfigDef.NO_DEFAULT_VALUE, + new CipherModeValidator(), + ConfigDef.Importance.HIGH, + "defines whether the data should get encrypted or decrypted") + .define( + KEY_SOURCE, + Type.STRING, + KEY_SOURCE_DEFAULT, + new KeySourceValidator(), + ConfigDef.Importance.HIGH, + "defines the origin of the secret key material (currently supports keys specified in the config or " + + "gcp secret manager)") + .define( + KMS_KEY_NAME, + Type.STRING, + KMS_KEY_NAME_DEFAULT, + Importance.MEDIUM, + "The GCP Cloud KMS key name for decrypting a data encryption key (DEK), " + + "if the DEK is encrypted with a key encryption key (KEK)"); + + private static final String PURPOSE = "(de)cipher record fields"; + + private static final Logger LOGGER = LoggerFactory.getLogger(CipherField.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private RecordHandler recordHandlerWithSchema; + private RecordHandler recordHandlerWithoutSchema; + private SchemaRewriter schemaRewriter; + private Cache schemaCache; + + @Override + public R apply(R record) { + if (operatingSchema(record) == null) { + return processWithoutSchema(record); + } else { + return processWithSchema(record); + } + } + + public R processWithoutSchema(R record) { + LOGGER.debug("processing schemaless data"); + Map valueMap = requireMap(operatingValue(record), PURPOSE); + Map updatedValueMap = new LinkedHashMap<>(valueMap); + recordHandlerWithoutSchema.matchFields(null, valueMap, null, updatedValueMap, ""); + return newRecord(record, null, updatedValueMap); + } + + public R processWithSchema(R record) { + LOGGER.debug("processing schema-aware data"); + Struct valueStruct = requireStruct(operatingValue(record), PURPOSE); + Schema updatedSchema = schemaCache.get(valueStruct.schema()); + if (updatedSchema == null) { + LOGGER.debug("adapting schema because record's schema not present in cache"); + updatedSchema = schemaRewriter.adaptSchema(valueStruct.schema(), ""); + schemaCache.put(valueStruct.schema(), updatedSchema); + } + Struct updatedValueStruct = new Struct(updatedSchema); + recordHandlerWithSchema.matchFields( + valueStruct.schema(), valueStruct, updatedSchema, updatedValueStruct, ""); + return newRecord(record, updatedSchema, updatedValueStruct); + } + + @Override + public ConfigDef config() { + return CONFIG_DEF; + } + + @Override + public void close() {} + + @Override + public void configure(Map props) { + try { + SimpleConfig config = new SimpleConfig(CONFIG_DEF, props); + Map fieldPathMap = + OBJECT_MAPPER + .readValue(config.getString(FIELD_CONFIG), new TypeReference>() {}) + .stream() + .collect(Collectors.toMap(FieldConfig::getName, Function.identity())); + Kryptonite kryptonite = configureKryptonite(config); + SerdeProcessor serdeProcessor = new KryoSerdeProcessor(); + recordHandlerWithSchema = + new SchemaawareRecordHandler( + config, + serdeProcessor, + kryptonite, + CipherMode.valueOf(config.getString(CIPHER_MODE)), + fieldPathMap); + recordHandlerWithoutSchema = + new SchemalessRecordHandler( + config, + serdeProcessor, + kryptonite, + CipherMode.valueOf(config.getString(CIPHER_MODE)), + fieldPathMap); + schemaRewriter = + new SchemaRewriter( + fieldPathMap, + FieldMode.valueOf(config.getString(FIELD_MODE)), + CipherMode.valueOf(config.getString(CIPHER_MODE)), + config.getString(PATH_DELIMITER)); + schemaCache = new SynchronizedCache<>(new LRUCache<>(16)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + throw new ConfigException(e.getMessage()); + } + } + + private Kryptonite configureKryptonite(SimpleConfig config) { + try { + KeySource keySource = KeySource.valueOf(config.getString(KEY_SOURCE)); + switch (keySource) { + case CONFIG: + Set dataKeyConfig = + OBJECT_MAPPER.readValue( + config.getPassword(CIPHER_DATA_KEYS).value(), + new TypeReference>() {}); + Map configKeyMap = + dataKeyConfig.stream() + .collect( + Collectors.toMap(DataKeyConfig::getIdentifier, DataKeyConfig::getKeyBytes)); + return new Kryptonite(new ConfigDataKeyVault(configKeyMap)); + case GCP_SECRET_MANAGER: + return new Kryptonite( + new GcpSecretManagerKeyVault( + config.getString(CIPHER_DATA_KEY_NAME), + new NoOpKeyStrategy(), + config.getLong(CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION), + config.getString(CIPHER_DATA_KEY_CACHE_EXPIRY_DURATION_UNIT))); + case GCP_SECRET_MANAGER_WITH_KMS: + return new Kryptonite( + new GcpSecretManagerKeyVault( + config.getString(CIPHER_DATA_KEY_NAME), config.getString(KMS_KEY_NAME))); + default: + throw new ConfigException( + "failed to configure kryptonite instance due to invalid key source"); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + throw new ConfigException(e.getMessage(), e); + } + } + + protected abstract Schema operatingSchema(R record); + + protected abstract Object operatingValue(R record); + + protected abstract R newRecord(R record, Schema updatedSchema, Object updatedValue); + + public static final class Key> extends CipherField { + + @Override + protected Schema operatingSchema(R record) { + return record.keySchema(); + } + + @Override + protected Object operatingValue(R record) { + return record.key(); + } + + @Override + protected R newRecord(R record, Schema updatedSchema, Object updatedValue) { + return record.newRecord( + record.topic(), + record.kafkaPartition(), + updatedSchema, + updatedValue, + record.valueSchema(), + record.value(), + record.timestamp()); + } + } + + public static final class Value> extends CipherField { + + @Override + protected Schema operatingSchema(R record) { + return record.valueSchema(); + } + + @Override + protected Object operatingValue(R record) { + return record.value(); + } + + @Override + protected R newRecord(R record, Schema updatedSchema, Object updatedValue) { + return record.newRecord( + record.topic(), + record.kafkaPartition(), + record.keySchema(), + record.key(), + updatedSchema, + updatedValue, + record.timestamp()); + } + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/DataKeyConfig.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/DataKeyConfig.java new file mode 100644 index 0000000..e774102 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/DataKeyConfig.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import com.github.hpgrahsl.kryptonite.FieldMetaData; +import java.util.Base64; +import java.util.Objects; + +public class DataKeyConfig { + + private String name; + private String version; + private String material = ""; + + public DataKeyConfig() {} + + public DataKeyConfig(String name, String version, String material) { + this.name = name; + this.version = version; + this.material = material; + } + + public String getName() { + return name; + } + + public String getVersion() { + return version; + } + + public String getMaterial() { + return material; + } + + public String getIdentifier() { + return String.join(FieldMetaData.IDENTIFIER_DELIMITER_DEFAULT, name, version); + } + + public byte[] getKeyBytes() { + return Base64.getDecoder().decode(material); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DataKeyConfig)) { + return false; + } + DataKeyConfig that = (DataKeyConfig) o; + return Objects.equals(getIdentifier(), that.getIdentifier()) + && Objects.equals(material, that.material); + } + + @Override + public int hashCode() { + return Objects.hash(getIdentifier(), material); + } + + @Override + public String toString() { + return "DataKeyConfig{" + + "identifier='" + + getIdentifier() + + "'}"; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldConfig.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldConfig.java new file mode 100644 index 0000000..29e860c --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldConfig.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class FieldConfig { + + private String name; + private String algorithm; + private String keyName; + private String keyVersion; + private Map schema; + + public FieldConfig() {} + + public FieldConfig( + String name, + String algorithm, + String keyName, + String keyVersion, + Map schema) { + this.name = Objects.requireNonNull(name, "field config's name must not be null"); + this.algorithm = algorithm; + this.keyName = keyName; + this.keyVersion = keyVersion; + this.schema = schema; + } + + public String getName() { + return name; + } + + public Optional getAlgorithm() { + return Optional.ofNullable(algorithm); + } + + public Optional getKeyName() { + return Optional.ofNullable(keyName); + } + + public Optional getKeyVersion() { + return Optional.ofNullable(keyVersion); + } + + public Optional> getSchema() { + return Optional.ofNullable(schema); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FieldConfig)) { + return false; + } + FieldConfig that = (FieldConfig) o; + return Objects.equals(name, that.name) + && Objects.equals(algorithm, that.algorithm) + && Objects.equals(keyName, that.keyName) + && Objects.equals(keyVersion, that.keyVersion) + && Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() { + return Objects.hash(name, algorithm, keyName, keyVersion, schema); + } + + @Override + public String toString() { + return "FieldConfig{" + + "name='" + + name + + "'" + + ", algorithm='" + + algorithm + + "'" + + ", keyName='" + + keyName + + "'" + + ", keyVersion='" + + keyVersion + + "'" + + ", schema=" + + schema + + '}'; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldPathMatcher.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldPathMatcher.java new file mode 100644 index 0000000..f624859 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/FieldPathMatcher.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import org.apache.kafka.connect.data.Schema; + +public interface FieldPathMatcher { + + Object matchFields( + Schema schemaOriginal, + Object objectOriginal, + Schema schemaNew, + Object objectNew, + String matchedPath); +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/RecordHandler.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/RecordHandler.java new file mode 100644 index 0000000..9e0a5d1 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/RecordHandler.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.SerdeProcessor; +import com.github.hpgrahsl.kryptonite.CipherMode; +import com.github.hpgrahsl.kryptonite.FieldMetaData; +import com.github.hpgrahsl.kryptonite.Kryptonite; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.connect.errors.DataException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class RecordHandler implements FieldPathMatcher { + + private static final Logger LOGGER = LoggerFactory.getLogger(RecordHandler.class); + + private final AbstractConfig config; + private final SerdeProcessor serdeProcessor; + private final Kryptonite kryptonite; + + protected final String pathDelimiter; + protected final CipherMode cipherMode; + protected final Map fieldConfig; + + public RecordHandler( + AbstractConfig config, + SerdeProcessor serdeProcessor, + Kryptonite kryptonite, + CipherMode cipherMode, + Map fieldConfig) { + this.config = config; + this.serdeProcessor = serdeProcessor; + this.kryptonite = kryptonite; + this.pathDelimiter = config.getString(CipherField.PATH_DELIMITER); + this.cipherMode = cipherMode; + this.fieldConfig = fieldConfig; + } + + public AbstractConfig getConfig() { + return config; + } + + public Kryptonite getKryptonite() { + return kryptonite; + } + + public Object processField(Object object, String matchedPath) { + try { + LOGGER.debug("{} field {}", cipherMode, matchedPath); + FieldMetaData fieldMetaData = determineFieldMetaData(object, matchedPath); + LOGGER.trace("field meta-data for path '{}' {}", matchedPath, fieldMetaData); + if (CipherMode.ENCRYPT == cipherMode) { + byte[] valueBytes = serdeProcessor.objectToBytes(object); + String cipherText = kryptonite.cipher(valueBytes, fieldMetaData); + return cipherText; + } else { + byte[] plainText = kryptonite.decipher((String) object, fieldMetaData); + Object restoredField = serdeProcessor.bytesToObject(plainText); + return restoredField; + } + } catch (Exception e) { + throw new DataException( + "error: " + + cipherMode + + " of field path '" + + matchedPath + + "' failed unexpectedly", + e); + } + } + + public List processListField(List list, String matchedPath) { + return list.stream() + .map( + e -> { + if (e instanceof List) return processListField((List) e, matchedPath); + if (e instanceof Map) return processMapField((Map) e, matchedPath); + return processField(e, matchedPath); + }) + .collect(Collectors.toList()); + } + + public Map processMapField(Map map, String matchedPath) { + return map.entrySet().stream() + .map( + e -> { + String pathUpdate = matchedPath + pathDelimiter + e.getKey(); + if (e.getValue() instanceof List) + return new AbstractMap.SimpleEntry<>( + e.getKey(), processListField((List) e.getValue(), pathUpdate)); + if (e.getValue() instanceof Map) + return new AbstractMap.SimpleEntry<>( + e.getKey(), processMapField((Map) e.getValue(), pathUpdate)); + return new AbstractMap.SimpleEntry<>( + e.getKey(), processField(e.getValue(), pathUpdate)); + }) + .collect( + LinkedHashMap::new, (lhm, e) -> lhm.put(e.getKey(), e.getValue()), HashMap::putAll); + } + + private FieldMetaData determineFieldMetaData(Object object, String fieldPath) { + return Optional.ofNullable(fieldConfig.get(fieldPath)) + .map( + fc -> + new FieldMetaData( + fc.getAlgorithm() + .orElseGet(() -> config.getString(CipherField.CIPHER_ALGORITHM)), + Optional.ofNullable(object).map(o -> o.getClass().getName()).orElse(""), + fc.getKeyName() + .orElseGet(() -> config.getString(CipherField.CIPHER_DATA_KEY_NAME)), + fc.getKeyVersion() + .orElseGet(() -> config.getString(CipherField.CIPHER_DATA_KEY_VERSION)))) + .orElseGet( + () -> + new FieldMetaData( + config.getString(CipherField.CIPHER_ALGORITHM), + Optional.ofNullable(object).map(o -> o.getClass().getName()).orElse(""), + config.getString(CipherField.CIPHER_DATA_KEY_NAME), + config.getString(CipherField.CIPHER_DATA_KEY_VERSION))); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaRewriter.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaRewriter.java new file mode 100644 index 0000000..4380945 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaRewriter.java @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField.FieldMode; +import com.github.hpgrahsl.kryptonite.CipherMode; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import org.apache.kafka.connect.data.Field; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Schema.Type; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.transforms.util.SchemaUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SchemaRewriter { + + public static class DefaultTypeSchemaMapper implements TypeSchemaMapper {} + + private static final Logger LOGGER = LoggerFactory.getLogger(SchemaRewriter.class); + + private final Map fieldConfig; + private final FieldMode fieldMode; + private final CipherMode cipherMode; + private final String pathDelimiter; + private final TypeSchemaMapper typeSchemaMapper; + + public SchemaRewriter( + Map fieldConfig, + FieldMode fieldMode, + CipherMode cipherMode, + String pathDelimiter) { + this.fieldConfig = fieldConfig; + this.fieldMode = fieldMode; + this.cipherMode = cipherMode; + this.pathDelimiter = pathDelimiter; + this.typeSchemaMapper = new DefaultTypeSchemaMapper(); + } + + public SchemaRewriter( + Map fieldConfig, + FieldMode fieldMode, + CipherMode cipherMode, + String pathDelimiter, + TypeSchemaMapper typeSchemaMapper) { + this.fieldConfig = fieldConfig; + this.fieldMode = fieldMode; + this.cipherMode = cipherMode; + this.pathDelimiter = pathDelimiter; + this.typeSchemaMapper = typeSchemaMapper; + } + + public Schema adaptSchema(Schema original, String matchedPath) { + LOGGER.debug("adapting original schema for {} mode", cipherMode); + SchemaBuilder builder = SchemaUtil.copySchemaBasics(original); + for (Field field : original.fields()) { + String updatedPath = + matchedPath.isEmpty() ? field.name() : matchedPath + pathDelimiter + field.name(); + if (fieldConfig.containsKey(updatedPath)) { + LOGGER.debug("adapting schema for matched field '{}'", updatedPath); + adaptField(derivePrimaryType(field, updatedPath), builder, field, updatedPath); + } else { + LOGGER.debug("copying schema for non-matched field '{}'", updatedPath); + builder.field(field.name(), field.schema()); + } + } + return original.isOptional() ? builder.optional().build() : builder.build(); + } + + private void adaptField( + Type decisiveType, SchemaBuilder builder, Field field, String updatedPath) { + LOGGER.trace("adapting to {} field type {}", cipherMode, decisiveType); + switch (decisiveType) { + case ARRAY: + adaptArraySchema(field, builder, updatedPath); + break; + case MAP: + adaptMapSchema(field, builder, updatedPath); + break; + case STRUCT: + adaptStructSchema(field, builder, updatedPath); + break; + default: + builder.field( + field.name(), + typeSchemaMapper.getSchemaForPrimitiveType( + decisiveType, field.schema().isOptional(), cipherMode)); + } + } + + private void adaptArraySchema(Field field, SchemaBuilder builder, String fieldPath) { + try { + if (CipherMode.ENCRYPT == cipherMode) { + LOGGER.trace("creating field schema for type {}", Type.ARRAY); + builder.field( + field.name(), + FieldMode.ELEMENT == fieldMode + ? SchemaBuilder.array( + typeSchemaMapper.getSchemaForPrimitiveType( + field.schema().valueSchema().type(), + field.schema().valueSchema().isOptional(), + cipherMode)) + .build() + : (field.schema().isOptional() + ? Schema.OPTIONAL_STRING_SCHEMA + : Schema.STRING_SCHEMA)); + } else { + // NOTE: whether or not the array itself is optional is specified + // in the config instead of taken from field.schema().isOptional() + LOGGER.trace("rebuilding field schema for type {} from config", Type.ARRAY); + Map fieldSpec = extractFieldSpecFromConfig(fieldPath); + builder.field(field.name(), extractAndAdaptArraySchemaFromConfig(fieldSpec, fieldPath)); + } + } catch (IllegalArgumentException | ClassCastException exc) { + throw new DataException("hit invalid type spec for field path " + fieldPath, exc); + } + } + + private void adaptMapSchema(Field field, SchemaBuilder builder, String fieldPath) { + try { + if (CipherMode.ENCRYPT == cipherMode) { + LOGGER.trace("creating field schema for type {}", Type.MAP); + builder.field( + field.name(), + FieldMode.ELEMENT == fieldMode + ? SchemaBuilder.map( + typeSchemaMapper.getSchemaForPrimitiveType( + field.schema().keySchema().type(), + field.schema().keySchema().isOptional(), + cipherMode), + typeSchemaMapper.getSchemaForPrimitiveType( + field.schema().valueSchema().type(), + field.schema().valueSchema().isOptional(), + cipherMode)) + .build() + : (field.schema().isOptional() + ? Schema.OPTIONAL_STRING_SCHEMA + : Schema.STRING_SCHEMA)); + } else { + // NOTE: whether or not the map itself is optional is specified + // in the config instead of taken from field.schema().isOptional() + LOGGER.trace("rebuilding field schema for type {} from config", Type.MAP); + Map fieldSpec = extractFieldSpecFromConfig(fieldPath); + builder.field(field.name(), extractAndAdaptMapSchemaFromConfig(fieldSpec, fieldPath)); + } + } catch (IllegalArgumentException | ClassCastException exc) { + throw new DataException("hit invalid type spec for field path " + fieldPath, exc); + } + } + + private void adaptStructSchema(Field field, SchemaBuilder builder, String fieldPath) { + if (CipherMode.ENCRYPT == cipherMode) { + LOGGER.trace("creating field schema for type {}", Type.STRUCT); + builder.field( + field.name(), + FieldMode.ELEMENT == fieldMode + ? adaptSchema(field.schema(), fieldPath) + : field.schema().isOptional() ? Schema.OPTIONAL_STRING_SCHEMA : Schema.STRING_SCHEMA); + } else { + // NOTE: whether or not the map itself is optional is specified + // in the config instead of taken from field.schema().isOptional() + LOGGER.trace("rebuilding field schema for type {} from config", Type.STRUCT); + Map fieldSpec = extractFieldSpecFromConfig(fieldPath); + builder.field(field.name(), extractAndAdaptStructSchemaFromConfig(fieldSpec, fieldPath)); + } + } + + private Type derivePrimaryType(Field field, String fieldPath) { + try { + if (CipherMode.ENCRYPT == cipherMode) return field.schema().type(); + FieldConfig fc = fieldConfig.get(fieldPath); + Map fs = + fc.getSchema() + .orElseThrow( + () -> + new DataException( + "schema-aware data needs schema spec for " + + cipherMode + + " but none was given" + + " for field path '" + + fieldPath + + "'")); + return extractTypeFromConfig(fs, fieldPath); + } catch (IllegalArgumentException exc) { + throw new DataException("hit invalid type spec for field path " + fieldPath, exc); + } + } + + private Schema extractAndAdaptArraySchemaFromConfig( + Map fieldSpec, String fieldPath) { + Type arrayType = extractTypeFromConfig(fieldSpec, fieldPath); + if (Type.ARRAY != arrayType) { + throw new DataException( + "expected " + Type.ARRAY.getName() + " but found " + arrayType.getName()); + } + boolean isArrayOptional = extractTypeOptionalFlagFromConfig(fieldSpec); + Type valueType = extractSubTypeFromConfig(fieldSpec, "valueSchema", null, fieldPath); + // TODO: value type for array shall support non-primitive types too + if (!valueType.isPrimitive()) { + throw new DataException( + "expected primitive value type for array elements but found " + valueType.name()); + } + boolean isValueOptional = extractSubTypeOptionalFlagFromConfig(fieldSpec, "valueSchema"); + SchemaBuilder sb = + SchemaBuilder.array( + typeSchemaMapper.getSchemaForPrimitiveType(valueType, isValueOptional, cipherMode)); + return isArrayOptional ? sb.optional().build() : sb.build(); + } + + private Schema extractAndAdaptMapSchemaFromConfig( + Map fieldSpec, String fieldPath) { + Type mapType = extractTypeFromConfig(fieldSpec, fieldPath); + if (Type.MAP != mapType) { + throw new DataException("expected " + Type.MAP.getName() + " but found " + mapType.getName()); + } + boolean isMapOptional = extractTypeOptionalFlagFromConfig(fieldSpec); + Type keyType = extractSubTypeFromConfig(fieldSpec, "keySchema", null, fieldPath); + boolean isKeyOptional = extractSubTypeOptionalFlagFromConfig(fieldSpec, "keySchema"); + Type valueType = extractSubTypeFromConfig(fieldSpec, "valueSchema", null, fieldPath); + boolean isValueOptional = extractSubTypeOptionalFlagFromConfig(fieldSpec, "valueSchema"); + // TODO: key + value types for map shall support non-primitive types too + if (!keyType.isPrimitive() || !valueType.isPrimitive()) { + throw new DataException( + "expected primitive types for both map key and map value but found types: " + + keyType.name() + + " - " + + valueType.name()); + } + SchemaBuilder sb = + SchemaBuilder.map( + typeSchemaMapper.getSchemaForPrimitiveType(keyType, isKeyOptional, cipherMode), + typeSchemaMapper.getSchemaForPrimitiveType(valueType, isValueOptional, cipherMode)); + return isMapOptional ? sb.optional().build() : sb.build(); + } + + @SuppressWarnings("unchecked") + private Schema extractAndAdaptStructSchemaFromConfig( + Map fieldSpec, String fieldPath) { + Type structType = extractTypeFromConfig(fieldSpec, fieldPath); + if (Type.STRUCT != structType) { + throw new DataException( + "expected " + Type.STRUCT.getName() + " but found " + structType.getName()); + } + boolean isStructOptional = extractTypeOptionalFlagFromConfig(fieldSpec); + SchemaBuilder structBuilder = SchemaBuilder.struct(); + List> fields = + Optional.ofNullable((List>) fieldSpec.get("fields")) + .orElseGet( + () -> { + throw new DataException( + Type.STRUCT.getName() + " is missing its mandatory field definitions"); + }); + fields.forEach( + map -> { + String nestedFieldName = extractFieldNameFromConfig(map); + Type nestedFieldType = extractSubTypeFromConfig(map, "schema", null, nestedFieldName); + boolean isNestedFieldOptional = extractSubTypeOptionalFlagFromConfig(map, "schema"); + String updatedPath = fieldPath + pathDelimiter + nestedFieldName; + if (Type.ARRAY == nestedFieldType) { + structBuilder.field( + nestedFieldName, + extractAndAdaptArraySchemaFromConfig( + (Map) map.get("schema"), updatedPath)); + } else if (Type.MAP == nestedFieldType) { + structBuilder.field( + nestedFieldName, + extractAndAdaptMapSchemaFromConfig( + (Map) map.get("schema"), updatedPath)); + } else if (Type.STRUCT == nestedFieldType) { + structBuilder.field( + nestedFieldName, + extractAndAdaptStructSchemaFromConfig( + (Map) map.get("schema"), updatedPath)); + } else { + structBuilder.field( + nestedFieldName, + typeSchemaMapper.getSchemaForPrimitiveType( + nestedFieldType, isNestedFieldOptional, cipherMode)); + } + }); + return isStructOptional ? structBuilder.optional().build() : structBuilder.build(); + } + + private Map extractFieldSpecFromConfig(String fieldName) { + FieldConfig fc = fieldConfig.get(fieldName); + return fc.getSchema() + .orElseThrow( + () -> + new DataException( + "schema-aware data needs schema spec for " + + cipherMode + + " but none was given" + + " for field path '" + + fieldName + + "'")); + } + + private String extractFieldNameFromConfig(Map map) { + return Optional.ofNullable((String) map.get("name")) + .filter(((Predicate) String::isEmpty).negate()) + .orElseThrow(() -> new DataException("missing name for field definition in struct type")); + } + + private Type extractTypeFromConfig(Map schema, String fieldName) { + return Optional.ofNullable((String) schema.get("type")) + .map(Type::valueOf) + .orElseThrow( + () -> new DataException("expected valid schema type for field '" + fieldName + "'")); + } + + private boolean extractTypeOptionalFlagFromConfig(Map schema) { + return Optional.ofNullable((Boolean) schema.get("optional")).orElse(false); + } + + @SuppressWarnings("unchecked") + private Type extractSubTypeFromConfig( + Map schema, String key, Type expected, String fieldName) { + return Optional.ofNullable((Map) schema.get(key)) + .map(m -> (String) m.get("type")) + .map(Type::valueOf) + .filter(t -> expected == null || t == expected) + .orElseThrow( + () -> + new DataException( + "expected valid sub type " + + (expected != null ? expected : "") + + " for field '" + + fieldName + + "'" + + " but either none was present or it was invalid")); + } + + @SuppressWarnings("unchecked") + private boolean extractSubTypeOptionalFlagFromConfig(Map schema, String key) { + return Optional.ofNullable((Map) schema.get(key)) + .map(m -> (Boolean) m.get("optional")) + .orElse(false); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaawareRecordHandler.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaawareRecordHandler.java new file mode 100644 index 0000000..a1a6c79 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemaawareRecordHandler.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField.FieldMode; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.SerdeProcessor; +import com.github.hpgrahsl.kryptonite.CipherMode; +import com.github.hpgrahsl.kryptonite.Kryptonite; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Schema.Type; +import org.apache.kafka.connect.data.Struct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SchemaawareRecordHandler extends RecordHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(SchemaawareRecordHandler.class); + + public SchemaawareRecordHandler( + AbstractConfig config, + SerdeProcessor serdeProcessor, + Kryptonite kryptonite, + CipherMode cipherMode, + Map fieldConfig) { + super(config, serdeProcessor, kryptonite, cipherMode, fieldConfig); + } + + @Override + public Object matchFields( + Schema schemaOriginal, + Object objectOriginal, + Schema schemaNew, + Object objectNew, + String matchedPath) { + Struct dataOriginal = (Struct) objectOriginal; + Struct dataNew = (Struct) objectNew; + schemaOriginal + .fields() + .forEach( + f -> { + String updatedPath = + matchedPath.isEmpty() ? f.name() : matchedPath + pathDelimiter + f.name(); + if (fieldConfig.containsKey(updatedPath)) { + LOGGER.trace("matched field '{}'", updatedPath); + if (FieldMode.ELEMENT + == FieldMode.valueOf(getConfig().getString(CipherField.FIELD_MODE))) { + if (f.schema().type() == Type.ARRAY) { + LOGGER.trace("processing {} field element-wise", Type.ARRAY); + dataNew.put( + schemaNew.field(f.name()), + processListField((List) dataOriginal.get(f.name()), updatedPath)); + } else if (f.schema().type() == Type.MAP) { + LOGGER.trace("processing {} field element-wise", Type.MAP); + dataNew.put( + schemaNew.field(f.name()), + processMapField((Map) dataOriginal.get(f.name()), updatedPath)); + } else if (f.schema().type() == Type.STRUCT) { + if (dataOriginal.get(f.name()) != null) { + LOGGER.trace("processing {} field element-wise", Type.STRUCT); + dataNew.put( + schemaNew.field(f.name()), + matchFields( + f.schema(), + dataOriginal.get(f.name()), + schemaNew.field(f.name()).schema(), + new Struct(schemaNew.field(f.name()).schema()), + updatedPath)); + } else { + LOGGER.trace( + "value of {} field was null -> skip element-wise sub-field matching", + Type.STRUCT); + } + } else { + LOGGER.trace("processing primitive field of type {}", f.schema().type()); + dataNew.put( + schemaNew.field(f.name()), + processField(dataOriginal.get(f.name()), updatedPath)); + } + } else { + LOGGER.trace("processing field of type {}", f.schema().type()); + dataNew.put( + schemaNew.field(f.name()), + processField(dataOriginal.get(f.name()), updatedPath)); + } + } else { + LOGGER.trace("copying non-matched field '{}'", updatedPath); + dataNew.put(schemaNew.field(f.name()), dataOriginal.get(f.name())); + } + }); + return dataNew; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemalessRecordHandler.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemalessRecordHandler.java new file mode 100644 index 0000000..ae45930 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/SchemalessRecordHandler.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField.FieldMode; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.SerdeProcessor; +import com.github.hpgrahsl.kryptonite.CipherMode; +import com.github.hpgrahsl.kryptonite.Kryptonite; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.connect.data.Schema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SchemalessRecordHandler extends RecordHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(SchemalessRecordHandler.class); + + public SchemalessRecordHandler( + AbstractConfig config, + SerdeProcessor serdeProcessor, + Kryptonite kryptonite, + CipherMode cipherMode, + Map fieldConfig) { + super(config, serdeProcessor, kryptonite, cipherMode, fieldConfig); + } + + @SuppressWarnings("unchecked") + @Override + public Object matchFields( + Schema schemaOriginal, + Object objectOriginal, + Schema schemaNew, + Object objectNew, + String matchedPath) { + Map dataOriginal = (Map) objectOriginal; + Map dataNew = (Map) objectNew; + dataOriginal.forEach( + (f, v) -> { + String updatedPath = matchedPath.isEmpty() ? f : matchedPath + pathDelimiter + f; + if (fieldConfig.containsKey(updatedPath)) { + LOGGER.trace("matched field '{}'", updatedPath); + if (FieldMode.ELEMENT + == FieldMode.valueOf(getConfig().getString(CipherField.FIELD_MODE))) { + if (v instanceof List) { + LOGGER.trace("processing {} field element-wise", List.class.getSimpleName()); + dataNew.put(f, processListField((List) dataOriginal.get(f), updatedPath)); + } else if (v instanceof Map) { + LOGGER.trace("processing {} field element-wise", Map.class.getSimpleName()); + dataNew.put(f, processMapField((Map) dataOriginal.get(f), updatedPath)); + } else { + LOGGER.trace("processing primitive field"); + dataNew.put(f, processField(dataOriginal.get(f), updatedPath)); + } + } else { + LOGGER.trace("processing field"); + dataNew.put(f, processField(dataOriginal.get(f), updatedPath)); + } + } else { + LOGGER.trace("copying non-matched field '{}'", updatedPath); + dataNew.put(f, dataOriginal.get(f)); + } + }); + return dataNew; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/TypeSchemaMapper.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/TypeSchemaMapper.java new file mode 100644 index 0000000..a990f95 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/TypeSchemaMapper.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import com.github.hpgrahsl.kryptonite.CipherMode; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Schema.Type; +import org.apache.kafka.connect.data.SchemaBuilder; + +public interface TypeSchemaMapper { + + @SuppressWarnings("serial") + Map> DEFAULT_MAPPINGS_ENCRYPT = + new LinkedHashMap>() { + { + put(Type.BOOLEAN, SchemaBuilder::string); + put(Type.INT8, SchemaBuilder::string); + put(Type.INT16, SchemaBuilder::string); + put(Type.INT32, SchemaBuilder::string); + put(Type.INT64, SchemaBuilder::string); + put(Type.FLOAT32, SchemaBuilder::string); + put(Type.FLOAT64, SchemaBuilder::string); + put(Type.STRING, SchemaBuilder::string); + put(Type.BYTES, SchemaBuilder::string); + } + }; + + @SuppressWarnings("serial") + Map> DEFAULT_MAPPINGS_DECRYPT = + new LinkedHashMap>() { + { + put(Type.BOOLEAN, SchemaBuilder::bool); + put(Type.INT8, SchemaBuilder::int8); + put(Type.INT16, SchemaBuilder::int16); + put(Type.INT32, SchemaBuilder::int32); + put(Type.INT64, SchemaBuilder::int16); + put(Type.FLOAT32, SchemaBuilder::float32); + put(Type.FLOAT64, SchemaBuilder::float64); + put(Type.STRING, SchemaBuilder::string); + put(Type.BYTES, SchemaBuilder::bytes); + } + }; + + default Schema getSchemaForPrimitiveType(Type type, boolean isOptional, CipherMode cipherMode) { + SchemaBuilder builder = + Optional.ofNullable( + CipherMode.ENCRYPT == cipherMode + ? DEFAULT_MAPPINGS_ENCRYPT.get(type) + : DEFAULT_MAPPINGS_DECRYPT.get(type)) + .orElseThrow( + () -> + new NoSuchElementException( + "no default type mapping found for type " + + type + + " (optional " + + isOptional + + ") and cipher mode " + + cipherMode)) + .get(); + return isOptional ? builder.optional().build() : builder.build(); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoInstance.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoInstance.java new file mode 100644 index 0000000..7ca990f --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoInstance.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes; + +import com.esotericsoftware.kryo.Kryo; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.KryoSerdeProcessor.SchemaSerializer; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes.KryoSerdeProcessor.StructSerializer; +import com.github.hpgrahsl.kryptonite.FieldMetaData; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Struct; + +public class KryoInstance { + + private static final Kryo KRYO = new Kryo(); + + static { + KRYO.setRegistrationRequired(false); + // TODO: register all commonly found types for more efficient serialization + // ... + KRYO.register(FieldMetaData.class); + + // NOTE: needed in order to be able to serialize structs with their schemas + KRYO.register(Struct.class).setSerializer(new StructSerializer()); + KRYO.register(Schema.class).setSerializer(new SchemaSerializer()); + } + + public static Kryo get() { + return KRYO; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoSerdeProcessor.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoSerdeProcessor.java new file mode 100644 index 0000000..d58361e --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/KryoSerdeProcessor.java @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; +import org.apache.kafka.connect.data.ConnectSchema; +import org.apache.kafka.connect.data.Field; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Schema.Type; +import org.apache.kafka.connect.data.Struct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KryoSerdeProcessor implements SerdeProcessor { + + private static final Logger LOGGER = LoggerFactory.getLogger(KryoSerdeProcessor.class); + + public KryoSerdeProcessor() {} + + public byte[] objectToBytes(Object object, Class clazz) { + return objectToBytes(object); + } + + public byte[] objectToBytes(Object object) { + Output output = new Output(new ByteArrayOutputStream()); + KryoInstance.get().writeClassAndObject(output, object); + return output.toBytes(); + } + + public Object bytesToObject(byte[] bytes, Class clazz) { + return bytesToObject(bytes); + } + + public Object bytesToObject(byte[] bytes) { + Input input = new Input(bytes); + return KryoInstance.get().readClassAndObject(input); + } + + public static class StructSerializer extends Serializer { + + private final SchemaSerializer schemaSerializer = new SchemaSerializer(); + + public void write(Kryo kryo, Output output, Struct struct) { + LOGGER.info("writing struct's schema"); + kryo.writeObject(output, struct.schema(), schemaSerializer); + writeStructFieldObjects(kryo, output, struct); + } + + private void writeStructFieldObjects(Kryo kryo, Output output, Struct struct) { + LOGGER.info("writing struct objects one by one..."); + struct + .schema() + .fields() + .forEach( + f -> { + LOGGER.info("write full field '{}' of type {}", f.name(), f.schema().type()); + if (f.schema().type() != Type.STRUCT) { + kryo.writeClassAndObject(output, struct.get(f)); + } else { + writeStructFieldObjects(kryo, output, (Struct) struct.get(f)); + } + }); + } + + public Struct read(Kryo kryo, Input input, Class type) { + LOGGER.info("reading struct's schema"); + Schema schema = kryo.readObject(input, Schema.class, schemaSerializer); + return readStructFieldObjects(kryo, input, new Struct(schema)); + } + + private Struct readStructFieldObjects(Kryo kryo, Input input, Struct struct) { + LOGGER.info("reading struct objects one by one..."); + struct + .schema() + .fields() + .forEach( + f -> { + LOGGER.info("read full field '{}' of type {}", f.name(), f.schema().type()); + if (f.schema().type() != Type.STRUCT) { + struct.put(f, kryo.readClassAndObject(input)); + } else { + struct.put(f, readStructFieldObjects(kryo, input, new Struct(f.schema()))); + } + }); + return struct; + } + } + + public static class SchemaSerializer extends Serializer { + + public void write(Kryo kryo, Output output, Schema object) { + LOGGER.trace("writing basic schema info for type {}", object.type()); + kryo.writeClassAndObject(output, object.type()); + output.writeString(object.name()); + output.writeBoolean(object.isOptional()); + Object defaultValue = object.defaultValue(); + kryo.writeObjectOrNull( + output, defaultValue, defaultValue != null ? defaultValue.getClass() : Object.class); + kryo.writeObjectOrNull(output, object.version(), Integer.class); + output.writeString(object.doc()); + + if (Type.STRUCT == object.type()) { + LOGGER.trace("writing struct type schema info"); + output.writeInt(object.fields().size()); + object + .fields() + .forEach( + f -> { + LOGGER.trace( + "writing field name '{}' with index '{}' and schema '{}'", + f.name(), + f.index(), + f.schema().type()); + output.writeString(f.name()); + output.writeInt(f.index()); + write(kryo, output, f.schema()); + }); + } else if (Type.ARRAY == object.type()) { + LOGGER.trace("writing array type schema info"); + write(kryo, output, object.valueSchema()); + } else if (Type.MAP == object.type()) { + LOGGER.trace("writing map type schema info"); + write(kryo, output, object.keySchema()); + write(kryo, output, object.valueSchema()); + } + } + + public Schema read(Kryo kryo, Input input, Class type) { + Type schemaType = (Type) kryo.readClassAndObject(input); + LOGGER.trace("reading basic schema info for type {}", schemaType); + String name = input.readString(); + Boolean isOptional = input.readBoolean(); + Object defaultValue = kryo.readObjectOrNull(input, Object.class); + Integer version = kryo.readObjectOrNull(input, Integer.class); + String doc = input.readString(); + + if (Type.STRUCT == schemaType) { + LOGGER.trace("reading struct type schema info"); + int numFields = input.readInt(); + List fields = new ArrayList<>(); + while (--numFields >= 0) { + String fName = input.readString(); + int fIndex = input.readInt(); + Schema fSchema = read(kryo, input, Schema.class); + LOGGER.trace( + "adding field name '{}' with index '{}' and schema '{}'", + fName, + fIndex, + fSchema.type()); + fields.add(new Field(fName, fIndex, fSchema)); + } + LOGGER.trace("returning struct schema"); + return new ConnectSchema( + schemaType, isOptional, defaultValue, name, version, doc, null, fields, null, null); + } else if (Type.ARRAY == schemaType) { + LOGGER.trace("reading array type schema info"); + Schema vSchema = read(kryo, input, Schema.class); + LOGGER.trace("returning array schema"); + return new ConnectSchema( + schemaType, isOptional, defaultValue, name, version, doc, null, null, null, vSchema); + } else if (Type.MAP == schemaType) { + LOGGER.trace("reading map type schema info"); + Schema kSchema = read(kryo, input, Schema.class); + Schema vSchema = read(kryo, input, Schema.class); + LOGGER.trace("returning map schema"); + return new ConnectSchema( + schemaType, isOptional, defaultValue, name, version, doc, null, null, kSchema, vSchema); + } else { + LOGGER.trace("returning {} schema", schemaType); + return new ConnectSchema(schemaType, isOptional, defaultValue, name, version, doc); + } + } + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/SerdeProcessor.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/SerdeProcessor.java new file mode 100644 index 0000000..3445e82 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/serdes/SerdeProcessor.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.serdes; + +public interface SerdeProcessor { + + byte[] objectToBytes(Object object, Class clazz); + + byte[] objectToBytes(Object object); + + Object bytesToObject(byte[] bytes, Class clazz); + + Object bytesToObject(byte[] bytes); +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringReader.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringReader.java new file mode 100644 index 0000000..7c7e793 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringReader.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.connector.ConnectRecord; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.transforms.Transformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class JsonStringReader> implements Transformation { + + public static final ConfigDef EMPTY_CONFIG_DEF = new ConfigDef(); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger LOGGER = LoggerFactory.getLogger(JsonStringReader.class); + + @Override + public R apply(R r) { + + Object data = operatingValue(r); + + if (data == null) { + LOGGER.warn("data was null -> passing it through without SMT processing"); + return null; + } + + if (!(data instanceof String)) { + LOGGER.error("unexpected data type: {}", data.getClass()); + throw new DataException( + "error: data expected to be of type String but was " + data.getClass()); + } + + try { + Object jsonMap = OBJECT_MAPPER.readValue((String) data, LinkedHashMap.class); + return newRecord(r, jsonMap); + } catch (JsonProcessingException e) { + throw new DataException("error: processing the record value failed", e); + } + } + + @Override + public ConfigDef config() { + return EMPTY_CONFIG_DEF; + } + + @Override + public void close() {} + + @Override + public void configure(Map map) {} + + protected abstract Schema operatingSchema(R record); + + protected abstract Object operatingValue(R record); + + protected abstract R newRecord(R record, Object updatedValue); + + public static class Key> extends JsonStringReader { + + @Override + protected Schema operatingSchema(R record) { + return record.keySchema(); + } + + @Override + protected Object operatingValue(R record) { + return record.key(); + } + + @Override + protected R newRecord(R record, Object updatedValue) { + return record.newRecord( + record.topic(), + record.kafkaPartition(), + null, + updatedValue, + null, + record.value(), + record.timestamp()); + } + } + + public static class Value> extends JsonStringReader { + + @Override + protected Schema operatingSchema(R record) { + return record.valueSchema(); + } + + @Override + protected Object operatingValue(R record) { + return record.value(); + } + + @Override + protected R newRecord(R record, Object updatedValue) { + return record.newRecord( + record.topic(), + record.kafkaPartition(), + null, + record.key(), + null, + updatedValue, + record.timestamp()); + } + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringWriter.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringWriter.java new file mode 100644 index 0000000..dae58b2 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/util/JsonStringWriter.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.connector.ConnectRecord; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.transforms.Transformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class JsonStringWriter> implements Transformation { + + public static final ConfigDef EMPTY_CONFIG_DEF = new ConfigDef(); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger LOGGER = LoggerFactory.getLogger(JsonStringWriter.class); + + @Override + public R apply(R r) { + + Object data = operatingValue(r); + + if (data == null) { + LOGGER.warn("data was null -> passing it through without SMT processing"); + return null; + } + + if (!(data instanceof Map)) { + LOGGER.error("unexpected data type: {}", data.getClass()); + throw new DataException("error: data expected to be of type Map but was " + data.getClass()); + } + + try { + String jsonString = OBJECT_MAPPER.writeValueAsString(data); + return newRecord(r, jsonString); + } catch (JsonProcessingException e) { + throw new DataException("error: processing the record value failed", e); + } + } + + @Override + public ConfigDef config() { + return EMPTY_CONFIG_DEF; + } + + @Override + public void close() {} + + @Override + public void configure(Map map) {} + + protected abstract Schema operatingSchema(R record); + + protected abstract Object operatingValue(R record); + + protected abstract R newRecord(R record, Object updatedValue); + + public static class Key> extends JsonStringWriter { + + @Override + protected Schema operatingSchema(R record) { + return record.keySchema(); + } + + @Override + protected Object operatingValue(R record) { + return record.key(); + } + + @Override + protected R newRecord(R record, Object updatedValue) { + return record.newRecord( + record.topic(), + record.kafkaPartition(), + null, + updatedValue, + null, + record.value(), + record.timestamp()); + } + } + + public static class Value> extends JsonStringWriter { + + @Override + protected Schema operatingSchema(R record) { + return record.valueSchema(); + } + + @Override + protected Object operatingValue(R record) { + return record.value(); + } + + @Override + protected R newRecord(R record, Object updatedValue) { + return record.newRecord( + record.topic(), + record.kafkaPartition(), + null, + record.key(), + null, + updatedValue, + record.timestamp()); + } + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherDataKeysValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherDataKeysValidator.java new file mode 100644 index 0000000..f89deb7 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherDataKeysValidator.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.DataKeyConfig; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Set; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.config.types.Password; + +public class CipherDataKeysValidator implements Validator { + + private static final Set VALID_KEY_LENGTHS = new LinkedHashSet<>(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + static { + VALID_KEY_LENGTHS.addAll(Arrays.asList(16, 24, 32)); + } + + @Override + public void ensureValid(String name, Object o) { + try { + Set dataKeyConfig = + OBJECT_MAPPER.readValue( + ((Password) o).value(), new TypeReference>() {}); + if (!dataKeyConfig.stream() + .filter(dkc -> !dkc.getMaterial().isEmpty()) + .map(DataKeyConfig::getKeyBytes) + .allMatch(bytes -> VALID_KEY_LENGTHS.contains(bytes.length))) { + throw new ConfigException( + name, + o, + "data key specification violation -> invalid key length " + + "(number of bytes must be one of " + + VALID_KEY_LENGTHS + + ")"); + } + } catch (IllegalArgumentException | JsonProcessingException exc) { + throw new ConfigException( + name, + o, + "data key specification violation -> " + + "not properly JSON encoded - " + + exc.getMessage()); + } + } + + @Override + public String toString() { + return "JSON array holding at least one valid data key config object, " + + "e.g. [" + + "{\"name\":\"my-key-id\",\"version\":\"1234\",\"material\":\"dmVyeS1zZWNyZXQta2V5JA==\"}" + + "]"; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherEncodingValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherEncodingValidator.java new file mode 100644 index 0000000..b6e56eb --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherEncodingValidator.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import java.util.LinkedHashSet; +import java.util.Set; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class CipherEncodingValidator implements Validator { + + @SuppressWarnings("serial") + private static final Set VALID_ENCODINGS = + new LinkedHashSet() { + { + add("base64"); + } + }; + + @Override + public void ensureValid(String name, Object o) { + String value = (String) o; + if (!VALID_ENCODINGS.contains(value)) { + throw new ConfigException( + name, o, "Must be one of the following encodings: " + String.join(",", VALID_ENCODINGS)); + } + } + + @Override + public String toString() { + return String.join(",", VALID_ENCODINGS); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherModeValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherModeValidator.java new file mode 100644 index 0000000..87883e6 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherModeValidator.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import com.github.hpgrahsl.kryptonite.CipherMode; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class CipherModeValidator implements Validator { + + private static final String ENCRYPT = CipherMode.ENCRYPT.name(); + private static final String DECRYPT = CipherMode.DECRYPT.name(); + + @Override + public void ensureValid(String name, Object o) { + String value = (String) o; + if (!ENCRYPT.equals(value) && !DECRYPT.equals(value)) { + throw new ConfigException(name, o, "Must be either " + ENCRYPT + " or " + DECRYPT); + } + } + + @Override + public String toString() { + return ENCRYPT + " or " + DECRYPT; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherNameValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherNameValidator.java new file mode 100644 index 0000000..7a8a860 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/CipherNameValidator.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import java.util.LinkedHashSet; +import java.util.Set; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class CipherNameValidator implements Validator { + + @SuppressWarnings("serial") + private static final Set VALID_CIPHERS = + new LinkedHashSet() { + { + add("AES/GCM/NoPadding"); + } + }; + + @Override + public void ensureValid(String name, Object o) { + String value = (String) o; + if (!VALID_CIPHERS.contains(value)) { + throw new ConfigException( + name, + o, + "Must be an AEAD cipher from the following ones: " + String.join(",", VALID_CIPHERS)); + } + } + + @Override + public String toString() { + return String.join(",", VALID_CIPHERS); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldConfigValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldConfigValidator.java new file mode 100644 index 0000000..0278cfc --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldConfigValidator.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.FieldConfig; +import java.util.Set; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class FieldConfigValidator implements Validator { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Override + public void ensureValid(String name, Object o) { + try { + Set fieldPathConfig = + OBJECT_MAPPER.readValue((String) o, new TypeReference>() {}); + if (fieldPathConfig.isEmpty()) { + throw new ConfigException( + name, + o, + "field config specification violation -> " + + " there must be at least 1 valid field path definition entry"); + } + } catch (JsonProcessingException exc) { + throw new ConfigException( + name, + o, + "field config specification violation -> " + + "not properly JSON encoded - " + + exc.getMessage()); + } + } + + @Override + public String toString() { + return "JSON array holding at least one valid field config object, e.g. [{\"name\": \"my-field-abc\"},{\"name\": \"my-nested.field-xyz\"}]"; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldModeValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldModeValidator.java new file mode 100644 index 0000000..304dcf8 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/FieldModeValidator.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField.FieldMode; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class FieldModeValidator implements Validator { + + private static final String ELEMENT = FieldMode.ELEMENT.name(); + private static final String OBJECT = FieldMode.OBJECT.name(); + + @Override + public void ensureValid(String name, Object o) { + String value = (String) o; + if (!ELEMENT.equals(value) && !OBJECT.equals(value)) { + throw new ConfigException(name, o, "Must be either " + ELEMENT + " or " + OBJECT); + } + } + + @Override + public String toString() { + return ELEMENT + " or " + OBJECT; + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/KeySourceValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/KeySourceValidator.java new file mode 100644 index 0000000..db2ebeb --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/KeySourceValidator.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import com.github.hpgrahsl.kafka.connect.transforms.kryptonite.CipherField.KeySource; +import java.util.Arrays; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class KeySourceValidator implements Validator { + + @Override + public void ensureValid(String name, Object o) { + try { + KeySource.valueOf((String) o); + } catch (IllegalArgumentException exc) { + throw new ConfigException(name, o, "Must be one of " + Arrays.toString(KeySource.values())); + } + } + + @Override + public String toString() { + return Arrays.toString(KeySource.values()); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/TimeUnitValidator.java b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/TimeUnitValidator.java new file mode 100644 index 0000000..6498074 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/main/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/validators/TimeUnitValidator.java @@ -0,0 +1,23 @@ +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite.validators; + +import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.common.config.ConfigDef.Validator; +import org.apache.kafka.common.config.ConfigException; + +public class TimeUnitValidator implements Validator { + + @Override + public void ensureValid(String name, Object o) { + try { + TimeUnit.valueOf((String) o); + } catch (IllegalArgumentException exc) { + throw new ConfigException(name, o, "Must be one of " + Arrays.toString(TimeUnit.values())); + } + } + + @Override + public String toString() { + return Arrays.toString(TimeUnit.values()); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/test/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherFieldTest.java b/kafka-connect-transform-kryptonite-gcp/src/test/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherFieldTest.java new file mode 100644 index 0000000..0efb5b4 --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/test/java/com/github/hpgrahsl/kafka/connect/transforms/kryptonite/CipherFieldTest.java @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kafka.connect.transforms.kryptonite; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.sink.SinkRecord; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +public class CipherFieldTest { + + public static Schema OBJ_SCHEMA_1; + public static Struct OBJ_STRUCT_1; + public static Map OBJ_MAP_1; + + @BeforeAll + @SuppressWarnings("serial") + static void initializeTestData() { + + OBJ_SCHEMA_1 = + SchemaBuilder.struct() + .field("id", Schema.STRING_SCHEMA) + .field("myString", Schema.STRING_SCHEMA) + .field("myInt", Schema.INT32_SCHEMA) + .field("myBoolean", Schema.BOOLEAN_SCHEMA) + .field( + "mySubDoc1", SchemaBuilder.struct().field("myString", Schema.STRING_SCHEMA).build()) + .field("myArray1", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + .field( + "mySubDoc2", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA).build()) + .field("myBytes", Schema.BYTES_SCHEMA) + .build(); + + OBJ_STRUCT_1 = + new Struct(OBJ_SCHEMA_1) + .put("id", "1234567890") + .put("myString", "some foo bla text") + .put("myInt", 42) + .put("myBoolean", true) + .put( + "mySubDoc1", + new Struct(OBJ_SCHEMA_1.field("mySubDoc1").schema()).put("myString", "hello json")) + .put("myArray1", Arrays.asList("str_1", "str_2", "...", "str_N")) + .put( + "mySubDoc2", + new HashMap() { + { + put("k1", 9); + put("k2", 8); + put("k3", 7); + } + }) + .put("myBytes", new byte[] {75, 97, 102, 107, 97, 32, 114, 111, 99, 107, 115, 33}); + + OBJ_MAP_1 = new LinkedHashMap<>(); + OBJ_MAP_1.put("id", "1234567890"); + OBJ_MAP_1.put("myString", "some foo bla text"); + OBJ_MAP_1.put("myInt", 42); + OBJ_MAP_1.put("myBoolean", true); + OBJ_MAP_1.put( + "mySubDoc1", + new HashMap() { + { + put("myString", "hello json"); + } + }); + OBJ_MAP_1.put("myArray1", Arrays.asList("str_1", "str_2", "...", "str_N")); + OBJ_MAP_1.put( + "mySubDoc2", + new HashMap() { + { + put("k1", 9); + put("k2", 8); + put("k3", 7); + } + }); + OBJ_MAP_1.put("myBytes", new byte[] {75, 97, 102, 107, 97, 32, 114, 111, 99, 107, 115, 33}); + } + + @Test + @DisplayName( + "apply SMT decrypt(encrypt(plaintext)) = plaintext for schemaless record with object mode") + @SuppressWarnings("unchecked") + void encryptDecryptSchemalessRecordTestWithObjectMode() { + Map encProps = new HashMap<>(); + encProps.put(CipherField.CIPHER_MODE, "ENCRYPT"); + encProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\"}," + + "{\"name\":\"myString\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\"}," + + "{\"name\":\"myBoolean\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\"}," + + "{\"name\":\"myArray1\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\"}," + + "{\"name\":\"myBytes\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + encProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + encProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + encProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + encProps.put(CipherField.FIELD_MODE, "OBJECT"); + + CipherField.Value encryptTransform = new CipherField.Value(); + encryptTransform.configure(encProps); + Map encryptedRecord = + (Map) + encryptTransform + .apply(new SourceRecord(null, null, "some-kafka-topic", 0, null, OBJ_MAP_1)) + .value(); + + assertAll( + () -> assertEquals(String.class, encryptedRecord.get("myArray1").getClass()), + () -> assertEquals(String.class, encryptedRecord.get("mySubDoc2").getClass())); + + Map decProps = new HashMap(); + decProps.put(CipherField.CIPHER_MODE, "DECRYPT"); + decProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\"}," + + "{\"name\":\"myString\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\"}," + + "{\"name\":\"myBoolean\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\"}," + + "{\"name\":\"myArray1\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\"}," + + "{\"name\":\"myBytes\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + decProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + decProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + decProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + decProps.put(CipherField.FIELD_MODE, "OBJECT"); + + CipherField.Value decryptTransform = new CipherField.Value(); + decryptTransform.configure(decProps); + Map decryptedRecord = + (Map) + decryptTransform + .apply(new SinkRecord("some-kafka-topic", 0, null, null, null, encryptedRecord, 0)) + .value(); + + assertAll( + () -> assertEquals(OBJ_MAP_1.get("id"), decryptedRecord.get("id")), + () -> assertEquals(OBJ_MAP_1.get("myString"), decryptedRecord.get("myString")), + () -> assertEquals(OBJ_MAP_1.get("myInt"), decryptedRecord.get("myInt")), + () -> assertEquals(OBJ_MAP_1.get("myBoolean"), decryptedRecord.get("myBoolean")), + () -> assertEquals(OBJ_MAP_1.get("mySubDoc1"), decryptedRecord.get("mySubDoc1")), + () -> assertEquals(OBJ_MAP_1.get("myArray1"), decryptedRecord.get("myArray1")), + () -> assertEquals(OBJ_MAP_1.get("mySubDoc2"), decryptedRecord.get("mySubDoc2")), + () -> + assertArrayEquals( + (byte[]) OBJ_MAP_1.get("myBytes"), (byte[]) decryptedRecord.get("myBytes"))); + } + + @Test + @DisplayName( + "apply SMT decrypt(encrypt(plaintext)) = plaintext for schemaless record with element mode") + @SuppressWarnings("unchecked") + void encryptDecryptSchemalessRecordTestWithElementMode() { + Map encProps = new HashMap(); + encProps.put(CipherField.CIPHER_MODE, "ENCRYPT"); + encProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\"}," + + "{\"name\":\"myString\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\"}," + + "{\"name\":\"myBoolean\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\"}," + + "{\"name\":\"myArray1\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\"}," + + "{\"name\":\"myBytes\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + encProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + encProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + encProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + encProps.put(CipherField.FIELD_MODE, "ELEMENT"); + + CipherField.Value encryptTransform = new CipherField.Value(); + encryptTransform.configure(encProps); + Map encryptedRecord = + (Map) + encryptTransform + .apply(new SourceRecord(null, null, "some-kafka-topic", 0, null, OBJ_MAP_1)) + .value(); + + assertAll( + () -> + assertAll( + () -> assertTrue(encryptedRecord.get("myArray1") instanceof List), + () -> assertEquals(4, ((List) encryptedRecord.get("myArray1")).size())), + () -> + assertAll( + () -> assertTrue(encryptedRecord.get("mySubDoc2") instanceof Map), + () -> assertEquals(3, ((Map) encryptedRecord.get("mySubDoc2")).size()))); + + Map decProps = new HashMap(); + decProps.put(CipherField.CIPHER_MODE, "DECRYPT"); + decProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\"}," + + "{\"name\":\"myString\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\"}," + + "{\"name\":\"myBoolean\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\"}," + + "{\"name\":\"myArray1\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\"}," + + "{\"name\":\"myBytes\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + decProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + decProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + decProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + decProps.put(CipherField.FIELD_MODE, "ELEMENT"); + + CipherField.Value decryptTransform = new CipherField.Value(); + decryptTransform.configure(decProps); + Map decryptedRecord = + (Map) + decryptTransform + .apply(new SinkRecord("some-kafka-topic", 0, null, null, null, encryptedRecord, 0)) + .value(); + + assertAll( + () -> assertEquals(OBJ_MAP_1.get("id"), decryptedRecord.get("id")), + () -> assertEquals(OBJ_MAP_1.get("myString"), decryptedRecord.get("myString")), + () -> assertEquals(OBJ_MAP_1.get("myInt"), decryptedRecord.get("myInt")), + () -> assertEquals(OBJ_MAP_1.get("myBoolean"), decryptedRecord.get("myBoolean")), + () -> assertEquals(OBJ_MAP_1.get("mySubDoc1"), decryptedRecord.get("mySubDoc1")), + () -> assertEquals(OBJ_MAP_1.get("myArray1"), decryptedRecord.get("myArray1")), + () -> assertEquals(OBJ_MAP_1.get("mySubDoc2"), decryptedRecord.get("mySubDoc2")), + () -> + assertArrayEquals( + (byte[]) OBJ_MAP_1.get("myBytes"), (byte[]) decryptedRecord.get("myBytes"))); + } + + @Test + @DisplayName( + "apply SMT decrypt(encrypt(plaintext)) = plaintext for schemaful record with object mode") + void encryptDecryptSchemafulRecordTestWithObjectMode() { + Map encProps = new HashMap(); + encProps.put(CipherField.CIPHER_MODE, "ENCRYPT"); + encProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\"}," + + "{\"name\":\"myString\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\"}," + + "{\"name\":\"myBoolean\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\"}," + + "{\"name\":\"myArray1\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\"}," + + "{\"name\":\"myBytes\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + encProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + encProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + encProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + encProps.put(CipherField.FIELD_MODE, "OBJECT"); + + CipherField.Value encryptTransform = new CipherField.Value(); + encryptTransform.configure(encProps); + Struct encryptedRecord = + (Struct) + encryptTransform + .apply( + new SourceRecord(null, null, "some-kafka-topic", 0, OBJ_SCHEMA_1, OBJ_STRUCT_1)) + .value(); + + assertAll( + () -> assertEquals(String.class, encryptedRecord.get("myArray1").getClass()), + () -> assertEquals(String.class, encryptedRecord.get("mySubDoc2").getClass())); + + Map decProps = new HashMap(); + decProps.put(CipherField.CIPHER_MODE, "DECRYPT"); + decProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\",\"schema\":{\"type\":\"STRING\"}}," + + "{\"name\":\"myString\",\"schema\":{\"type\":\"STRING\"},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\",\"schema\":{\"type\":\"INT32\"}}," + + "{\"name\":\"myBoolean\",\"schema\":{\"type\":\"BOOLEAN\"},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\",\"schema\":{\"type\":\"STRUCT\",\"fields\":[{\"name\":\"myString\",\"schema\":{\"type\":\"STRING\"}}]}}," + + "{\"name\":\"myArray1\",\"schema\":{\"type\":\"ARRAY\",\"valueSchema\":{\"type\":\"STRING\"}},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\",\"schema\":{\"type\":\"MAP\",\"keySchema\":{\"type\":\"STRING\"},\"valueSchema\":{\"type\":\"INT32\"}}}," + + "{\"name\":\"myBytes\",\"schema\":{\"type\":\"BYTES\"},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + decProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + decProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + decProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + decProps.put(CipherField.FIELD_MODE, "OBJECT"); + + CipherField.Value decryptTransform = new CipherField.Value(); + decryptTransform.configure(decProps); + Struct decryptedRecord = + (Struct) + decryptTransform + .apply( + new SinkRecord( + "some-kafka-topic", + 0, + null, + null, + encryptedRecord.schema(), + encryptedRecord, + 0)) + .value(); + + assertAll( + () -> assertEquals(OBJ_SCHEMA_1, decryptedRecord.schema()), + () -> assertEquals(OBJ_STRUCT_1.get("id"), decryptedRecord.get("id")), + () -> assertEquals(OBJ_STRUCT_1.get("myString"), decryptedRecord.get("myString")), + () -> assertEquals(OBJ_STRUCT_1.get("myInt"), decryptedRecord.get("myInt")), + () -> assertEquals(OBJ_STRUCT_1.get("myBoolean"), decryptedRecord.get("myBoolean")), + () -> assertEquals(OBJ_STRUCT_1.get("mySubDoc1"), decryptedRecord.get("mySubDoc1")), + () -> assertEquals(OBJ_STRUCT_1.get("myArray1"), decryptedRecord.get("myArray1")), + () -> assertEquals(OBJ_STRUCT_1.get("mySubDoc2"), decryptedRecord.get("mySubDoc2")), + () -> + assertArrayEquals( + (byte[]) OBJ_STRUCT_1.get("myBytes"), (byte[]) decryptedRecord.get("myBytes"))); + } + + @Test + @DisplayName( + "apply SMT decrypt(encrypt(plaintext)) = plaintext for schemaful record with element mode") + void encryptDecryptSchemafulRecordTestWithElementMode() { + Map encProps = new HashMap(); + encProps.put(CipherField.CIPHER_MODE, "ENCRYPT"); + encProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\"}," + + "{\"name\":\"myString\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\"}," + + "{\"name\":\"myBoolean\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\"}," + + "{\"name\":\"mySubDoc1.myString\"}," + + "{\"name\":\"myArray1\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\"}," + + "{\"name\":\"mySubDoc2.k1\"}," + + "{\"name\":\"mySubDoc2.k2\"}," + + "{\"name\":\"mySubDoc2.k3\"}," + + "{\"name\":\"myBytes\",\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + encProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + encProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + encProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + encProps.put(CipherField.FIELD_MODE, "ELEMENT"); + + CipherField.Value encryptTransform = new CipherField.Value(); + encryptTransform.configure(encProps); + final Struct encryptedRecord = + (Struct) + encryptTransform + .apply( + new SourceRecord(null, null, "some-kafka-topic", 0, OBJ_SCHEMA_1, OBJ_STRUCT_1)) + .value(); + + assertAll( + () -> + assertAll( + () -> assertTrue(encryptedRecord.get("myArray1") instanceof List), + () -> assertEquals(4, ((List) encryptedRecord.get("myArray1")).size())), + () -> + assertAll( + () -> assertTrue(encryptedRecord.get("mySubDoc2") instanceof Map), + () -> assertEquals(3, ((Map) encryptedRecord.get("mySubDoc2")).size()))); + + Map decProps = new HashMap(); + decProps.put(CipherField.CIPHER_MODE, "DECRYPT"); + decProps.put( + CipherField.FIELD_CONFIG, + "[" + + "{\"name\":\"id\",\"schema\":{\"type\":\"STRING\"}}," + + "{\"name\":\"myString\",\"schema\":{\"type\":\"STRING\"},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"myInt\",\"schema\":{\"type\":\"INT32\"}}," + + "{\"name\":\"myBoolean\",\"schema\":{\"type\":\"BOOLEAN\"},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc1\",\"schema\":{\"type\":\"STRUCT\",\"fields\":[{\"name\":\"myString\",\"schema\":{\"type\":\"STRING\"}}]}}," + + "{\"name\":\"mySubDoc1.myString\",\"schema\":{\"type\":\"STRING\"}}," + + "{\"name\":\"myArray1\",\"schema\":{\"type\":\"ARRAY\",\"valueSchema\":{\"type\":\"STRING\"}},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}," + + "{\"name\":\"mySubDoc2\",\"schema\":{\"type\":\"MAP\",\"keySchema\":{\"type\":\"STRING\"},\"valueSchema\":{\"type\":\"INT32\"}}}," + + "{\"name\":\"mySubDoc1.k1\",\"schema\":{\"type\":\"INT32\"}}," + + "{\"name\":\"mySubDoc1.k2\",\"schema\":{\"type\":\"INT32\"}}," + + "{\"name\":\"mySubDoc1.k3\",\"schema\":{\"type\":\"INT32\"}}," + + "{\"name\":\"myBytes\",\"schema\":{\"type\":\"BYTES\"},\"keyName\":\"my-demo-secret-key\",\"keyVersion\":\"987\"}" + + "]"); + decProps.put( + CipherField.CIPHER_DATA_KEYS, + "[" + + "{\"name\":\"my-demo-secret-key\",\"version\":\"123\",\"material\":\"YWFhYWFhYWFhYWFhYWFhYQ==\"}," + + "{\"name\":\"my-demo-secret-key\",\"version\":\"987\",\"material\":\"YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI=\"}" + + "]"); + decProps.put(CipherField.CIPHER_DATA_KEY_NAME, "my-demo-secret-key"); + decProps.put(CipherField.CIPHER_DATA_KEY_VERSION, "123"); + decProps.put(CipherField.FIELD_MODE, "ELEMENT"); + + CipherField.Value decryptTransform = new CipherField.Value(); + decryptTransform.configure(decProps); + Struct decryptedRecord = + (Struct) + decryptTransform + .apply( + new SinkRecord( + "some-kafka-topic", + 0, + null, + null, + encryptedRecord.schema(), + encryptedRecord, + 0)) + .value(); + + assertAll( + () -> assertEquals(OBJ_SCHEMA_1, decryptedRecord.schema()), + () -> assertEquals(OBJ_STRUCT_1.get("id"), decryptedRecord.get("id")), + () -> assertEquals(OBJ_STRUCT_1.get("myString"), decryptedRecord.get("myString")), + () -> assertEquals(OBJ_STRUCT_1.get("myInt"), decryptedRecord.get("myInt")), + () -> assertEquals(OBJ_STRUCT_1.get("myBoolean"), decryptedRecord.get("myBoolean")), + () -> assertEquals(OBJ_STRUCT_1.get("mySubDoc1"), decryptedRecord.get("mySubDoc1")), + () -> assertEquals(OBJ_STRUCT_1.get("myArray1"), decryptedRecord.get("myArray1")), + () -> assertEquals(OBJ_STRUCT_1.get("mySubDoc2"), decryptedRecord.get("mySubDoc2")), + () -> + assertArrayEquals( + (byte[]) OBJ_STRUCT_1.get("myBytes"), (byte[]) decryptedRecord.get("myBytes"))); + } +} diff --git a/kafka-connect-transform-kryptonite-gcp/src/test/resources/logback.xml b/kafka-connect-transform-kryptonite-gcp/src/test/resources/logback.xml new file mode 100644 index 0000000..59a897a --- /dev/null +++ b/kafka-connect-transform-kryptonite-gcp/src/test/resources/logback.xml @@ -0,0 +1,15 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/kryptonite/pom.xml b/kryptonite/pom.xml new file mode 100644 index 0000000..b518d74 --- /dev/null +++ b/kryptonite/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + + com.github.hpgrahsl.kafka.connect + kafka-connect-transform-kryptonite-parent + 0.3.1-SNAPSHOT + + kryptonite + jar + + kryptonite + + + + com.github.ben-manes.caffeine + caffeine + ${caffeine.version} + + + com.google.cloud + google-cloud-kms + + + com.google.cloud + google-cloud-secretmanager + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-shade-plugin + + + maven-surefire-plugin + ${surefire.plugin.version} + + + com.coveo + fmt-maven-plugin + + + + + diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/AesGcmNoPadding.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/AesGcmNoPadding.java new file mode 100644 index 0000000..7422c57 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/AesGcmNoPadding.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +import java.nio.ByteBuffer; +import java.security.SecureRandom; +import java.security.spec.AlgorithmParameterSpec; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +public class AesGcmNoPadding implements CryptoAlgorithm { + + public static final String CIPHER_ALGORITHM = "AES/GCM/NoPadding"; + public static final String KEY_ALGORITHM = "AES"; + public static final int AUTH_TAG_LENGTH = 128; + public static final int IV_LENGTH = 16; + + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + + @Override + public byte[] cipher(byte[] plaintext, byte[] key) throws Exception { + byte[] iv = new byte[IV_LENGTH]; + SECURE_RANDOM.nextBytes(iv); + final Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + GCMParameterSpec parameterSpec = new GCMParameterSpec(AUTH_TAG_LENGTH, iv); + SecretKey secretKey = new SecretKeySpec(key, KEY_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec); + byte[] ciphertext = cipher.doFinal(plaintext); + ByteBuffer byteBuffer = ByteBuffer.allocate(iv.length + ciphertext.length); + byteBuffer.put(iv); + byteBuffer.put(ciphertext); + return byteBuffer.array(); + } + + @Override + public byte[] decipher(byte[] ciphertext, byte[] key) throws Exception { + final Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + AlgorithmParameterSpec gcmIv = new GCMParameterSpec(AUTH_TAG_LENGTH, ciphertext, 0, IV_LENGTH); + SecretKey secretKey = new SecretKeySpec(key, KEY_ALGORITHM); + cipher.init(Cipher.DECRYPT_MODE, secretKey, gcmIv); + return cipher.doFinal(ciphertext, IV_LENGTH, ciphertext.length - IV_LENGTH); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CipherMode.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CipherMode.java new file mode 100644 index 0000000..86fab31 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CipherMode.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +public enum CipherMode { + ENCRYPT, + DECRYPT +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Cipherable.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Cipherable.java new file mode 100644 index 0000000..fa233d2 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Cipherable.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +public interface Cipherable { + + byte[] ciphertext(); +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/ConfigDataKeyVault.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/ConfigDataKeyVault.java new file mode 100644 index 0000000..32d98a5 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/ConfigDataKeyVault.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +import java.util.Map; + +public class ConfigDataKeyVault extends KeyVault { + + private final Map keys; + + public ConfigDataKeyVault(Map keys) { + this(keys, new NoOpKeyStrategy()); + } + + public ConfigDataKeyVault(Map keys, KeyStrategy keyStrategy) { + super(keyStrategy); + this.keys = keys; + } + + @Override + public byte[] readKey(String identifier) { + byte[] keyBytes = keys.get(identifier); + if (keyBytes == null) { + throw new KeyNotFoundException( + "could not find key for identifier '" + + identifier + + "' in " + + ConfigDataKeyVault.class.getName() + + " key vault"); + } + return keyStrategy.processKey(keyBytes, identifier); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CryptoAlgorithm.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CryptoAlgorithm.java new file mode 100644 index 0000000..81d8823 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/CryptoAlgorithm.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +public interface CryptoAlgorithm { + + byte[] cipher(byte[] plaintext, byte[] key) throws Exception; + + byte[] decipher(byte[] ciphertext, byte[] key) throws Exception; +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/DataException.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/DataException.java new file mode 100644 index 0000000..b228acc --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/DataException.java @@ -0,0 +1,24 @@ +package com.github.hpgrahsl.kryptonite; + +@SuppressWarnings("serial") +public class DataException extends RuntimeException { + + public DataException() {} + + public DataException(String message) { + super(message); + } + + public DataException(String message, Throwable cause) { + super(message, cause); + } + + public DataException(Throwable cause) { + super(cause); + } + + public DataException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/FieldMetaData.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/FieldMetaData.java new file mode 100644 index 0000000..d6b8df6 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/FieldMetaData.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +public class FieldMetaData { + + public static final String IDENTIFIER_DELIMITER_DEFAULT = "/versions/"; + + private final String algorithm; + private final String dataType; + private final String keyName; + private final String keyVersion; + private final String delimiter; + + public FieldMetaData(String algorithm, String dataType, String keyName, String keyVersion) { + this(algorithm, dataType, keyName, keyVersion, IDENTIFIER_DELIMITER_DEFAULT); + } + + public FieldMetaData( + String algorithm, String dataType, String keyName, String keyVersion, String delimiter) { + this.algorithm = algorithm; + this.dataType = dataType; + this.keyName = keyName; + this.keyVersion = keyVersion; + this.delimiter = delimiter; + } + + public String getAlgorithm() { + return algorithm; + } + + public String getAlgorithmId() { + return Kryptonite.CIPHERNAME_ID_LUT.get(algorithm); + } + + public String getDataType() { + return dataType; + } + + public String getKeyName() { + return keyName; + } + + public String getKeyVersion() { + return keyVersion; + } + + public String getDelimiter() { + return delimiter; + } + + public String getIdentifier() { + return String.join(delimiter, keyName, keyVersion); + } + + public String getIdentifier(String keyVersion) { + return String.join(delimiter, keyName, keyVersion); + } + + @Override + public String toString() { + return "FieldMetaData{" + + "algorithm='" + + algorithm + + "'" + + ", dataType='" + + dataType + + "'" + + ", keyName='" + + keyName + + "'" + + ", keyVersion='" + + keyVersion + + "'" + + ", delimiter='" + + delimiter + + "'" + + "}"; + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpKmsKeyStrategy.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpKmsKeyStrategy.java new file mode 100644 index 0000000..cc2396b --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpKmsKeyStrategy.java @@ -0,0 +1,30 @@ +package com.github.hpgrahsl.kryptonite; + +import com.google.cloud.kms.v1.CryptoKeyName; +import com.google.cloud.kms.v1.DecryptResponse; +import com.google.cloud.kms.v1.KeyManagementServiceClient; +import com.google.protobuf.ByteString; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GcpKmsKeyStrategy extends KeyStrategy { + + private static final Logger LOGGER = LoggerFactory.getLogger(GcpKmsKeyStrategy.class); + + private final KeyManagementServiceClient client; + private final CryptoKeyName keyName; + + public GcpKmsKeyStrategy(String keyName) throws IOException { + this.client = KeyManagementServiceClient.create(); + this.keyName = CryptoKeyName.parse(keyName); + } + + @Override + byte[] processKey(byte[] origKeyBytes, String identifier) { + LOGGER.info("Process key: " + identifier); + LOGGER.info("KEK name: " + keyName); + DecryptResponse resp = client.decrypt(keyName, ByteString.copyFrom(origKeyBytes)); + return resp.getPlaintext().toByteArray(); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpSecretManagerKeyVault.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpSecretManagerKeyVault.java new file mode 100644 index 0000000..12e9f14 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/GcpSecretManagerKeyVault.java @@ -0,0 +1,90 @@ +package com.github.hpgrahsl.kryptonite; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.RemovalListener; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretVersionsPagedResponse; +import com.google.cloud.secretmanager.v1.SecretName; +import com.google.cloud.secretmanager.v1.SecretVersion.State; +import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.concurrent.TimeUnit; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GcpSecretManagerKeyVault extends KeyVault { + + private static final Logger LOGGER = LoggerFactory.getLogger(GcpSecretManagerKeyVault.class); + + private final SecretManagerServiceClient client; + private final SecretName secretName; + private final LoadingCache secretCache; + + public GcpSecretManagerKeyVault(String secretName, String keyName) throws IOException { + this(secretName, new GcpKmsKeyStrategy(keyName), 24L, TimeUnit.HOURS.name()); + } + + public GcpSecretManagerKeyVault(String secretName, KeyStrategy strategy) throws IOException { + this(secretName, strategy, 24L, TimeUnit.HOURS.name()); + } + + public GcpSecretManagerKeyVault( + String secretName, KeyStrategy strategy, long duration, String unit) throws IOException { + super(strategy); + this.client = SecretManagerServiceClient.create(); + this.secretName = SecretName.parse(secretName); + this.secretCache = + Caffeine.newBuilder() + .expireAfterWrite(duration, TimeUnit.valueOf(unit)) + .evictionListener( + new RemovalListener() { + @Override + public void onRemoval( + @Nullable String s, + byte @Nullable [] bytes, + @NonNull RemovalCause removalCause) { + if (bytes != null) { + Arrays.fill(bytes, (byte) 0); + } + } + }) + .build(key -> accessSecretVersion(key)); + initKeys(); + } + + private void initKeys() { + ListSecretVersionsPagedResponse resp = client.listSecretVersions(secretName); + resp.iterateAll() + .forEach( + version -> { + if (version.getState() == State.ENABLED) { + final String name = version.getName(); + LOGGER.info("Init key: " + name); + secretCache.get(name); + } + }); + } + + private byte[] accessSecretVersion(String identifier) { + AccessSecretVersionResponse resp = client.accessSecretVersion(identifier); + final String key = resp.getPayload().getData().toStringUtf8(); + final byte[] keyBytes = Base64.getDecoder().decode(key); + return keyStrategy.processKey(keyBytes, identifier); + } + + @Override + byte[] readKey(String identifier) { + byte[] keyBytes = secretCache.get(identifier); + if (keyBytes == null) { + LOGGER.info("Read key: " + identifier); + keyBytes = accessSecretVersion(identifier); + } + return keyBytes; + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyException.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyException.java new file mode 100644 index 0000000..0d410e3 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyException.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +@SuppressWarnings("serial") +public class KeyException extends RuntimeException { + + public KeyException() {} + + public KeyException(String message) { + super(message); + } + + public KeyException(String message, Throwable cause) { + super(message, cause); + } + + public KeyException(Throwable cause) { + super(cause); + } + + public KeyException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyInvalidException.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyInvalidException.java new file mode 100644 index 0000000..81ca722 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyInvalidException.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +@SuppressWarnings("serial") +public class KeyInvalidException extends KeyException { + + public KeyInvalidException() {} + + public KeyInvalidException(String message) { + super(message); + } + + public KeyInvalidException(String message, Throwable cause) { + super(message, cause); + } + + public KeyInvalidException(Throwable cause) { + super(cause); + } + + public KeyInvalidException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyNotFoundException.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyNotFoundException.java new file mode 100644 index 0000000..b20e012 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyNotFoundException.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +@SuppressWarnings("serial") +public class KeyNotFoundException extends KeyException { + + public KeyNotFoundException() {} + + public KeyNotFoundException(String message) { + super(message); + } + + public KeyNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public KeyNotFoundException(Throwable cause) { + super(cause); + } + + public KeyNotFoundException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyStrategy.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyStrategy.java new file mode 100644 index 0000000..1c549d3 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyStrategy.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +import java.util.HashMap; +import java.util.Map; + +public abstract class KeyStrategy { + + private final Map keyCache = new HashMap<>(); + + public Map getKeyCache() { + return keyCache; + } + + abstract byte[] processKey(byte[] origKeyBytes, String identifier); +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyVault.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyVault.java new file mode 100644 index 0000000..5a99620 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/KeyVault.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +public abstract class KeyVault { + + KeyStrategy keyStrategy; + + public KeyVault(KeyStrategy keyStrategy) { + this.keyStrategy = keyStrategy; + } + + abstract byte[] readKey(String identifier); +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Kryptonite.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Kryptonite.java new file mode 100644 index 0000000..e9e651c --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/Kryptonite.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.Map; + +public class Kryptonite { + + public static final String VERSION_DELIMITER_DEFAULT = "#"; + + @SuppressWarnings("serial") + public static final Map CIPHERNAME_ID_LUT = + new LinkedHashMap() { + { + put("AES/GCM/NoPadding", "01"); + } + }; + + @SuppressWarnings("serial") + public static final Map ID_CRYPTOALGORITHM_LUT = + new LinkedHashMap() { + { + put("01", new AesGcmNoPadding()); + } + }; + + private final KeyVault keyVault; + private final String delimiter; + + public Kryptonite(KeyVault keyVault) { + this(keyVault, VERSION_DELIMITER_DEFAULT); + } + + public Kryptonite(KeyVault keyVault, String delimiter) { + this.keyVault = keyVault; + this.delimiter = delimiter; + } + + private byte[] cipher(byte[] plainText, String algorithmId, String identifier) { + try { + return ID_CRYPTOALGORITHM_LUT + .get(algorithmId) + .cipher(plainText, keyVault.readKey(identifier)); + } catch (Exception e) { + throw new DataException(e.getMessage(), e); + } + } + + public String cipher(byte[] plainText, FieldMetaData metadata) { + String cipherText = + Base64.getEncoder() + .encodeToString(cipher(plainText, metadata.getAlgorithmId(), metadata.getIdentifier())); + return String.join(delimiter, metadata.getKeyVersion(), cipherText); + } + + private byte[] decipher(byte[] cipherText, String algorithmId, String identifier) { + try { + return ID_CRYPTOALGORITHM_LUT + .get(algorithmId) + .decipher(cipherText, keyVault.readKey(identifier)); + } catch (Exception e) { + throw new DataException(e.getMessage(), e); + } + } + + public byte[] decipher(String cipherText, FieldMetaData metadata) { + String[] splitText = cipherText.split(delimiter); + if (splitText.length == 1) { + byte[] decoded = Base64.getDecoder().decode(splitText[0]); + return decipher(decoded, metadata.getAlgorithmId(), metadata.getIdentifier()); + } else if (splitText.length == 2) { + byte[] decoded = Base64.getDecoder().decode(splitText[1]); + return decipher(decoded, metadata.getAlgorithmId(), metadata.getIdentifier(splitText[0])); + } else { + throw new DataException("Illegal cipher text format."); + } + } +} diff --git a/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/NoOpKeyStrategy.java b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/NoOpKeyStrategy.java new file mode 100644 index 0000000..c071244 --- /dev/null +++ b/kryptonite/src/main/java/com/github/hpgrahsl/kryptonite/NoOpKeyStrategy.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021. Hans-Peter Grahsl (grahslhp@gmail.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.hpgrahsl.kryptonite; + +public class NoOpKeyStrategy extends KeyStrategy { + + @Override + public byte[] processKey(byte[] origKeyBytes, String identifier) { + return origKeyBytes; + } +} diff --git a/kryptonite/src/test/resources/logback.xml b/kryptonite/src/test/resources/logback.xml new file mode 100644 index 0000000..59a897a --- /dev/null +++ b/kryptonite/src/test/resources/logback.xml @@ -0,0 +1,15 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..0a8af65 --- /dev/null +++ b/pom.xml @@ -0,0 +1,206 @@ + + + + 4.0.0 + + com.github.hpgrahsl.kafka.connect + kafka-connect-transform-kryptonite-parent + 0.3.1-SNAPSHOT + pom + + kafka-connect-transform-kryptonite-parent + + Kryptonite: An SMT for Kafka Connect + https://github.com/kouzoh/kafka-connect-smt-kryptonite-gcp + + + The Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0 + repo + + + 2021 + + + Hans-Peter Grahsl + grahslhp@gmail.com + https://github.com/hpgrahsl + + maintainer + + + + + scm:git:git://github.com/kouzoh/kafka-connect-smt-kryptonite-gcp.git + scm:git:ssh://github.com:kouzoh/kafka-connect-smt-kryptonite-gcp.git + https://github.com/kouzoh/kafka-connect-smt-kryptonite-gcp + + + github + https://github.com/kouzoh/kafka-connect-smt-kryptonite-gcp/issues + + + + kafka-connect-transform-kryptonite-gcp + kryptonite + + + + UTF-8 + 1.8 + 1.8 + 2.7.0 + 2.12.6 + 5.0.3 + 2.9.3 + 0.166.0 + 1.43.2 + 1.7.35 + 1.2.8 + 5.7.0 + 3.2.4 + 3.8.0 + 2.22.2 + 2.9 + + + + + central + mercari-releases + https://mercari.jfrog.io/mercari/libs-release-local + + + snapshots + mercari-snapshots + https://mercari.jfrog.io/mercari/libs-snapshot-local + + + + + + + com.google.cloud + google-cloud-bom + ${google.cloud.bom.version} + pom + import + + + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler.plugin.version} + + + -Xlint:all + -Werror + + true + true + ${maven.compiler.source} + ${maven.compiler.target} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${shade.plugin.version} + + + package + + shade + + + + + org.slf4j:slf4j-api:jar: + org.slf4j:slf4j-log4j12:jar: + log4j:log4j:jar: + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + maven-surefire-plugin + ${surefire.plugin.version} + + + com.coveo + fmt-maven-plugin + ${fmt.plugin.version} + + + + format + check + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + +