From 159f19f71037f317e8acb20ed07096691f91e9c2 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Mon, 5 Dec 2022 17:41:17 -0800 Subject: [PATCH] Revert "RecordSchemaValidator can resolve $ref schemas (#19625)" This reverts commit 56bfdaab6a1edddde15dc2f46e2e12d92ff21210. --- .../validation/json/JsonSchemaValidator.java | 69 ++----------------- .../json/JsonSchemaValidatorTest.java | 39 ----------- airbyte-workers/Dockerfile | 3 - airbyte-workers/build.gradle | 15 ---- build.gradle | 2 - 5 files changed, 5 insertions(+), 123 deletions(-) diff --git a/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java b/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java index 5c9507bee977..04fb601221d9 100644 --- a/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java +++ b/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java @@ -5,19 +5,14 @@ package io.airbyte.validation.json; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.networknt.schema.JsonMetaSchema; -import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SchemaValidatorsConfig; import com.networknt.schema.SpecVersion; -import com.networknt.schema.ValidationContext; import com.networknt.schema.ValidationMessage; import io.airbyte.commons.string.Strings; import java.io.File; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -30,37 +25,13 @@ public class JsonSchemaValidator { private static final Logger LOGGER = LoggerFactory.getLogger(JsonSchemaValidator.class); - // This URI just needs to point at any path in the same directory as /app/WellKnownTypes.json - // It's required for the JsonSchema#validate method to resolve $ref correctly. - private static final URI DEFAULT_BASE_URI; - - static { - try { - DEFAULT_BASE_URI = new URI("file:///app/nonexistent_file.json"); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + private final SchemaValidatorsConfig schemaValidatorsConfig; private final JsonSchemaFactory jsonSchemaFactory; - private final URI baseUri; public JsonSchemaValidator() { - this(DEFAULT_BASE_URI); - } - - /** - * The public constructor hardcodes a URL with access to WellKnownTypes.json. This method allows - * tests to override that URI - * - * Required to resolve $ref schemas using WellKnownTypes.json - * - * @param baseUri The base URI for schema resolution - */ - @VisibleForTesting - protected JsonSchemaValidator(URI baseUri) { + this.schemaValidatorsConfig = new SchemaValidatorsConfig(); this.jsonSchemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7); - this.baseUri = baseUri; } public Set validate(final JsonNode schemaJson, final JsonNode objectJson) { @@ -89,38 +60,8 @@ private Set validateInternal(final JsonNode schemaJson, final Preconditions.checkNotNull(schemaJson); Preconditions.checkNotNull(objectJson); - // Default to draft-07, but have handling for the other metaschemas that networknt supports - JsonMetaSchema metaschema; - JsonNode metaschemaNode = schemaJson.get("$schema"); - if (metaschemaNode == null || metaschemaNode.asText() == null || metaschemaNode.asText().isEmpty()) { - metaschema = JsonMetaSchema.getV7(); - } else { - String metaschemaString = metaschemaNode.asText(); - // We're not using "http://....".equals(), because we want to avoid weirdness with https, etc. - if (metaschemaString.contains("json-schema.org/draft-04")) { - metaschema = JsonMetaSchema.getV4(); - } else if (metaschemaString.contains("json-schema.org/draft-06")) { - metaschema = JsonMetaSchema.getV6(); - } else if (metaschemaString.contains("json-schema.org/draft/2019-09")) { - metaschema = JsonMetaSchema.getV201909(); - } else if (metaschemaString.contains("json-schema.org/draft/2020-12")) { - metaschema = JsonMetaSchema.getV202012(); - } else { - metaschema = JsonMetaSchema.getV7(); - } - } - - ValidationContext context = new ValidationContext( - jsonSchemaFactory.getUriFactory(), - null, - metaschema, - jsonSchemaFactory, - null); - JsonSchema schema = new JsonSchema( - context, - baseUri, - schemaJson); - return schema.validate(objectJson); + return jsonSchemaFactory.getSchema(schemaJson, schemaValidatorsConfig) + .validate(objectJson); } public boolean test(final JsonNode schemaJson, final JsonNode objectJson) { diff --git a/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java b/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java index a1a0b511e40c..6dcf3c554b29 100644 --- a/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java +++ b/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java @@ -5,7 +5,6 @@ package io.airbyte.validation.json; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -16,10 +15,7 @@ import io.airbyte.commons.json.Jsons; import java.io.File; import java.io.IOException; -import java.net.URI; import java.nio.file.Files; -import java.util.Set; -import lombok.SneakyThrows; import org.junit.jupiter.api.Test; class JsonSchemaValidatorTest { @@ -106,39 +102,4 @@ void test() throws IOException { assertNull(JsonSchemaValidator.getSchema(schemaFile, "NonExistentObject")); } - @SneakyThrows - @Test - void testResolveReferences() throws IOException { - String referencableSchemas = """ - { - "definitions": { - "ref1": {"type": "string"}, - "ref2": {"type": "boolean"} - } - } - """; - final File schemaFile = IOs.writeFile(Files.createTempDirectory("test"), "WellKnownTypes.json", referencableSchemas).toFile(); - JsonSchemaValidator jsonSchemaValidator = - new JsonSchemaValidator(new URI("file://" + schemaFile.getParentFile().getAbsolutePath() + "/foo.json")); - - Set validationResult = jsonSchemaValidator.validate( - Jsons.deserialize(""" - { - "type": "object", - "properties": { - "prop1": {"$ref": "WellKnownTypes.json#/definitions/ref1"}, - "prop2": {"$ref": "WellKnownTypes.json#/definitions/ref2"} - } - } - """), - Jsons.deserialize(""" - { - "prop1": "foo", - "prop2": "false" - } - """)); - - assertEquals(Set.of("$.prop2: string found, boolean expected"), validationResult); - } - } diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index caf2e27bdb06..474df37796be 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -20,8 +20,5 @@ WORKDIR /app # Move worker app ADD bin/${APPLICATION}-${VERSION}.tar /app -# Grab well-known types file -COPY WellKnownTypes.json /app - # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"] diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 8c36042c85f1..681fe2f30933 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -1,6 +1,3 @@ -import groovy.yaml.YamlSlurper -import groovy.json.JsonBuilder - plugins { id 'application' id 'airbyte-integration-test-java' @@ -120,20 +117,8 @@ task cloudStorageIntegrationTest(type: Test) { } } -task generateWellKnownTypes() { - doLast { - def wellKnownTypesYaml = project(':airbyte-protocol').file('protocol-models/src/main/resources/airbyte_protocol/well_known_types.yaml').text - def parsedYaml = new YamlSlurper().parseText(wellKnownTypesYaml) - def wellKnownTypesJson = new JsonBuilder(parsedYaml).toPrettyString() - def targetFile = project.file("${buildDir}/docker/WellKnownTypes.json") - targetFile.getParentFile().mkdirs() - targetFile.text = wellKnownTypesJson - } -} - tasks.named("buildDockerImage") { dependsOn copyGeneratedTar - dependsOn generateWellKnownTypes } Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/build.gradle b/build.gradle index 5a062ee276b0..821fb7dde5a3 100644 --- a/build.gradle +++ b/build.gradle @@ -20,8 +20,6 @@ buildscript { // The alternative is to import the openapi plugin for all modules. // This might need to be updated when we change openapi plugin versions. classpath 'com.fasterxml.jackson.core:jackson-core:2.13.0' - - classpath 'org.codehaus.groovy:groovy-yaml:3.0.3' } }