From 84f42b4e191734d26d387d822723b97603cb90dc Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 6 Aug 2020 17:54:55 -0700 Subject: [PATCH 1/6] wip --- conduit-server/build.gradle | 3 +++ settings.gradle | 1 + 2 files changed, 4 insertions(+) diff --git a/conduit-server/build.gradle b/conduit-server/build.gradle index 3752bbbeb730..85f098d8ea72 100644 --- a/conduit-server/build.gradle +++ b/conduit-server/build.gradle @@ -20,6 +20,9 @@ dependencies { implementation group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: '2.31' implementation group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: '2.31' + compile(group: "com.fasterxml.jackson.core", name: "jackson-databind", version: "2.9.8"); + compile(group: "com.networknt", name: "json-schema-validator", version: "1.0.42"); + implementation project(':conduit-api') implementation project(':conduit-commons') } diff --git a/settings.gradle b/settings.gradle index 03909be23fd7..a926d73c5cbd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -3,3 +3,4 @@ rootProject.name = 'conduit' include 'conduit-api' include 'conduit-server' include 'conduit-commons' +include 'conduit-config-persistence' From dce7ab712c82e49fcc0542f0adfed620cf8900cc Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 6 Aug 2020 19:05:12 -0700 Subject: [PATCH 2/6] add jsonschema to java generation --- conduit-config/build.gradle | 36 +++++++++++++ .../json/StandardConnectionStatus.json | 17 ++++++ .../resources/json/StandardDataSchema.json | 52 +++++++++++++++++++ .../json/StandardDiscoveryOutput.json | 14 +++++ .../json/StandardScheduleConfiguration.json | 15 ++++++ .../json/StandardSyncConfiguration.json | 17 ++++++ .../resources/json/StandardSyncSummary.json | 47 +++++++++++++++++ settings.gradle | 3 +- 8 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 conduit-config/build.gradle create mode 100644 conduit-config/src/main/resources/json/StandardConnectionStatus.json create mode 100644 conduit-config/src/main/resources/json/StandardDataSchema.json create mode 100644 conduit-config/src/main/resources/json/StandardDiscoveryOutput.json create mode 100644 conduit-config/src/main/resources/json/StandardScheduleConfiguration.json create mode 100644 conduit-config/src/main/resources/json/StandardSyncConfiguration.json create mode 100644 conduit-config/src/main/resources/json/StandardSyncSummary.json diff --git a/conduit-config/build.gradle b/conduit-config/build.gradle new file mode 100644 index 000000000000..4dfa872e1b52 --- /dev/null +++ b/conduit-config/build.gradle @@ -0,0 +1,36 @@ +group 'io.dataline.conduit' +version '0.1.0' + +//https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin +apply plugin: 'jsonschema2pojo' + + +buildscript { + repositories { + mavenCentral() + } + + dependencies { +// classpath 'org.jsonschema2pojo:jsonschema2pojo-gradle-plugin:${js2p.version}' + classpath 'org.jsonschema2pojo:jsonschema2pojo-gradle-plugin:1.0.2' + } +} + + +repositories { + mavenCentral() +} + +dependencies { + testCompile group: 'junit', name: 'junit', version: '4.12' + + // Required if generating JSR-303 annotations + compile 'javax.validation:validation-api:1.1.0.CR2' + // Required if generating Jackson 2 annotations + compile 'com.fasterxml.jackson.core:jackson-databind:2.9.7' + // Required if generating JodaTime data types + compile 'joda-time:joda-time:2.2' +} + +jsonSchema2Pojo { +} diff --git a/conduit-config/src/main/resources/json/StandardConnectionStatus.json b/conduit-config/src/main/resources/json/StandardConnectionStatus.json new file mode 100644 index 000000000000..401bc569e1a0 --- /dev/null +++ b/conduit-config/src/main/resources/json/StandardConnectionStatus.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "StandardConnectionStatus", + "title": "StandardConnectionStatus", + "description": "describes the result of a 'test connection' action.", + "type": "object", + "required": ["status"], + "properties": { + "status": { + "type": "string", + "enum": ["success", "failure"] + }, + "message": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/conduit-config/src/main/resources/json/StandardDataSchema.json b/conduit-config/src/main/resources/json/StandardDataSchema.json new file mode 100644 index 000000000000..04f725c50055 --- /dev/null +++ b/conduit-config/src/main/resources/json/StandardDataSchema.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "StandardDataSchema", + "title": "StandardDataSchema", + "type": "object", + "definitions": { + "schema": { + "description": "describes the available schema.", + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "columns" + ], + "properties": { + "name": { + "type": "string" + }, + "columns": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "dataType" + ], + "properties": { + "name": { + "type": "string" + }, + "dataType": { + "type": "string", + "enum": [ + "string", + "number", + "boolean" + ] + } + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json b/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json new file mode 100644 index 000000000000..5972fe7a5e3f --- /dev/null +++ b/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "StandardDiscoveryOutput", + "title": "StandardDiscoveryOutput", + "description": "describes the standard output for any discovery run.", + "type": "object", + "required": ["schema"], + "properties": { + "schema": { + "description": "describes the available schema.", + "$ref": "StandardDataSchema.json#/definitions/schema" + } + } +} \ No newline at end of file diff --git a/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json b/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json new file mode 100644 index 000000000000..aaf7866b245f --- /dev/null +++ b/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "StandardScheduleConfiguration", + "title": "StandardScheduleConfiguration", + "type": "object", + "properties": { + "timeUnit": { + "type": "string", + "enum": ["minutes", "hours", "days", "weeks"] + }, + "units": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/conduit-config/src/main/resources/json/StandardSyncConfiguration.json b/conduit-config/src/main/resources/json/StandardSyncConfiguration.json new file mode 100644 index 000000000000..2f578cd810e4 --- /dev/null +++ b/conduit-config/src/main/resources/json/StandardSyncConfiguration.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "StandardSyncConfiguration", + "title": "StandardSyncConfiguration", + "description": "configuration required for sync for ALL taps", + "type": "object", + "properties": { + "syncMode": { + "type": "string", + "enum": ["full_refresh", "append"] + }, + "schema": { + "description": "describes the elements of the schema that will be synced.", + "$ref": "StandardDataSchema.json#/definitions/schema" + } + } +} \ No newline at end of file diff --git a/conduit-config/src/main/resources/json/StandardSyncSummary.json b/conduit-config/src/main/resources/json/StandardSyncSummary.json new file mode 100644 index 000000000000..8be9d283346e --- /dev/null +++ b/conduit-config/src/main/resources/json/StandardSyncSummary.json @@ -0,0 +1,47 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "StandardSyncSummary", + "title": "StandardSyncSummary", + "description": "standard information output by ALL taps for a sync step (our version of state.json)", + "type": "object", + "properties": { + "attemptId": { + "type": "string", + "format": "uuid" + }, + "status": { + "type": "string", + "enum": ["pending", "in_progress","completed", "failed", "cancelled"] + }, + "recordsSynced": { + "type": "integer", + "minValue": 0 + }, + "version": { + "type": "integer" + }, + "tables": { + "type": "array", + "items": { + "type": "object", + "properties": { + "lastRecord": { + "description": "blob of the last record", + "type": "object" + }, + "version": { + "type": "integer" + } + } + } + }, + "startTime": { + "type": "integer" + }, + "endTime": { + "type": "integer" + }, + "logs": { + } + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index a926d73c5cbd..1dffbfa8109c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -3,4 +3,5 @@ rootProject.name = 'conduit' include 'conduit-api' include 'conduit-server' include 'conduit-commons' -include 'conduit-config-persistence' +include 'conduit-config' + From 5a6ace5eb48a3be2ef81c7c861c917436297e2c1 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 6 Aug 2020 19:09:37 -0700 Subject: [PATCH 3/6] remove comment --- conduit-config/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/conduit-config/build.gradle b/conduit-config/build.gradle index 4dfa872e1b52..ece8a46d1473 100644 --- a/conduit-config/build.gradle +++ b/conduit-config/build.gradle @@ -11,7 +11,6 @@ buildscript { } dependencies { -// classpath 'org.jsonschema2pojo:jsonschema2pojo-gradle-plugin:${js2p.version}' classpath 'org.jsonschema2pojo:jsonschema2pojo-gradle-plugin:1.0.2' } } From be479782bfd811fb86fefb76425b33a92d2b24e6 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 6 Aug 2020 19:11:03 -0700 Subject: [PATCH 4/6] whitespace --- .../src/main/resources/json/StandardConnectionStatus.json | 2 +- conduit-config/src/main/resources/json/StandardDataSchema.json | 2 +- .../src/main/resources/json/StandardDiscoveryOutput.json | 2 +- .../src/main/resources/json/StandardScheduleConfiguration.json | 2 +- .../src/main/resources/json/StandardSyncConfiguration.json | 2 +- conduit-config/src/main/resources/json/StandardSyncSummary.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conduit-config/src/main/resources/json/StandardConnectionStatus.json b/conduit-config/src/main/resources/json/StandardConnectionStatus.json index 401bc569e1a0..a1fcb8682ad8 100644 --- a/conduit-config/src/main/resources/json/StandardConnectionStatus.json +++ b/conduit-config/src/main/resources/json/StandardConnectionStatus.json @@ -14,4 +14,4 @@ "type": "string" } } -} \ No newline at end of file +} diff --git a/conduit-config/src/main/resources/json/StandardDataSchema.json b/conduit-config/src/main/resources/json/StandardDataSchema.json index 04f725c50055..dc55ff16051b 100644 --- a/conduit-config/src/main/resources/json/StandardDataSchema.json +++ b/conduit-config/src/main/resources/json/StandardDataSchema.json @@ -49,4 +49,4 @@ } } } -} \ No newline at end of file +} diff --git a/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json b/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json index 5972fe7a5e3f..1c37d293a0df 100644 --- a/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json +++ b/conduit-config/src/main/resources/json/StandardDiscoveryOutput.json @@ -11,4 +11,4 @@ "$ref": "StandardDataSchema.json#/definitions/schema" } } -} \ No newline at end of file +} diff --git a/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json b/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json index aaf7866b245f..747e079f43e1 100644 --- a/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json +++ b/conduit-config/src/main/resources/json/StandardScheduleConfiguration.json @@ -12,4 +12,4 @@ "type": "integer" } } -} \ No newline at end of file +} diff --git a/conduit-config/src/main/resources/json/StandardSyncConfiguration.json b/conduit-config/src/main/resources/json/StandardSyncConfiguration.json index 2f578cd810e4..00b29496d362 100644 --- a/conduit-config/src/main/resources/json/StandardSyncConfiguration.json +++ b/conduit-config/src/main/resources/json/StandardSyncConfiguration.json @@ -14,4 +14,4 @@ "$ref": "StandardDataSchema.json#/definitions/schema" } } -} \ No newline at end of file +} diff --git a/conduit-config/src/main/resources/json/StandardSyncSummary.json b/conduit-config/src/main/resources/json/StandardSyncSummary.json index 8be9d283346e..4a4c5bdce9a3 100644 --- a/conduit-config/src/main/resources/json/StandardSyncSummary.json +++ b/conduit-config/src/main/resources/json/StandardSyncSummary.json @@ -44,4 +44,4 @@ "logs": { } } -} \ No newline at end of file +} From 2068cf4ab543bd294846e434c402e4b1ebf5cddc Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 6 Aug 2020 19:11:45 -0700 Subject: [PATCH 5/6] remove imports not relevant to this pr --- conduit-server/build.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/conduit-server/build.gradle b/conduit-server/build.gradle index 85f098d8ea72..3752bbbeb730 100644 --- a/conduit-server/build.gradle +++ b/conduit-server/build.gradle @@ -20,9 +20,6 @@ dependencies { implementation group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: '2.31' implementation group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: '2.31' - compile(group: "com.fasterxml.jackson.core", name: "jackson-databind", version: "2.9.8"); - compile(group: "com.networknt", name: "json-schema-validator", version: "1.0.42"); - implementation project(':conduit-api') implementation project(':conduit-commons') } From 6ca0030af361315f4ec73fc7093832809a527a54 Mon Sep 17 00:00:00 2001 From: cgardens Date: Sat, 8 Aug 2020 14:42:41 -0700 Subject: [PATCH 6/6] add todos --- conduit-config/build.gradle | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conduit-config/build.gradle b/conduit-config/build.gradle index ece8a46d1473..83e2759f08e7 100644 --- a/conduit-config/build.gradle +++ b/conduit-config/build.gradle @@ -1,7 +1,8 @@ group 'io.dataline.conduit' version '0.1.0' -//https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin +// todo: make sure we use the non-deprecated version of this. +// https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin apply plugin: 'jsonschema2pojo' @@ -27,8 +28,6 @@ dependencies { compile 'javax.validation:validation-api:1.1.0.CR2' // Required if generating Jackson 2 annotations compile 'com.fasterxml.jackson.core:jackson-databind:2.9.7' - // Required if generating JodaTime data types - compile 'joda-time:joda-time:2.2' } jsonSchema2Pojo {