From aaaf997b78c681b30b40ad33c3369c3f109eaf7e Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Sat, 19 Oct 2019 19:26:29 +0700 Subject: [PATCH 1/5] Clean up the Maven build Three main themes: - Fix core module dependency on ingestion, so that Maven resolves it as an inter-project dep without requiring it to be installed to (local) repository and mucking with the version. - Use dependencyManagement in parent POM for common dependencies and families -- version drift between modules on things like gRPC and Google Cloud deps will bring more pain in the long run than any short- term convenience of letting them differ between modules. This makes it easy to control that consistently. - Avoid (re)defining properties in submodule POMs -- similar reasoning as dependencies: drift leads to suffering. If you build only a subset of modules at a time, resolved values may differ based on how Maven orders the build. Also avoid repeating things that inherit from the parent POM, like module groupId. --- core/pom.xml | 98 +------ .../core/job/dataflow/DataflowJobManager.java | 13 +- .../job/direct/DirectRunnerJobManager.java | 11 +- .../job/dataflow/DataflowJobManagerTest.java | 12 +- .../direct/DirectRunnerJobManagerTest.java | 12 +- ingestion/pom.xml | 85 +----- .../feast/store/TextFileDynamicIOTest.java | 1 - ingestion/src/test/resources/logback-test.xml | 6 +- pom.xml | 262 +++++++++++++++++- serving/pom.xml | 63 +---- 10 files changed, 301 insertions(+), 262 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 12574876d7..5c6ce72472 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -19,14 +19,15 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 + feast feast-parent ${revision} - feast-core - jar + Feast Core + feast-core @@ -78,46 +79,26 @@ - - - jcenter - https://jcenter.bintray.com/ - - - jitpack.io - https://jitpack.io - - - - com.github.gojek.feast + feast feast-ingestion - 9428b230a5 + ${project.version} org.springframework.boot spring-boot-starter-web - ${springBootVersion} - - - org.springframework.boot - spring-boot-starter-logging - - org.springframework.boot spring-boot-starter-log4j2 - ${springBootVersion} org.lognet grpc-spring-boot-starter - 2.4.1 @@ -129,14 +110,12 @@ org.springframework.boot spring-boot-starter-actuator - ${springBootVersion} org.springframework.boot spring-boot-configuration-processor - ${springBootVersion} @@ -144,32 +123,27 @@ io.grpc grpc-netty - ${grpcVersion} io.grpc grpc-services - ${grpcVersion} io.grpc grpc-stub - ${grpcVersion} com.google.protobuf protobuf-java-util - ${protobufVersion} com.google.guava guava - 26.0-jre @@ -181,13 +155,11 @@ commons-codec commons-codec - 1.10 joda-time joda-time - 2.9.9 com.google.api-client @@ -234,12 +206,10 @@ com.google.cloud google-cloud-bigquery - 1.48.0 com.google.cloud google-cloud-nio - 0.83.0-alpha org.apache.flink @@ -247,70 +217,26 @@ 1.5.5 - - - com.fasterxml.jackson.core - jackson-databind - 2.9.9.1 - - - com.fasterxml.jackson.core - jackson-core - 2.9.9 - - - com.fasterxml.jackson.core - jackson-annotations - 2.9.9 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - 2.9.9 - - com.github.spullara.mustache.java compiler 0.9.5 - - com.hubspot.jinjava - jinjava - 2.4.12 - - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - + io.micrometer micrometer-core - 1.0.7 io.micrometer micrometer-registry-statsd - 1.0.7 com.datadoghq java-dogstatsd-client - 2.6.1 @@ -330,23 +256,17 @@ org.projectlombok lombok - 1.18.2 - provided io.grpc grpc-testing - ${grpcVersion} - test org.hamcrest hamcrest-all - 1.3 - test @@ -372,20 +292,14 @@ org.springframework spring-test - 5.0.8.RELEASE - test org.springframework.boot spring-boot-test - ${springBootVersion} - test org.springframework.boot spring-boot-test-autoconfigure - ${springBootVersion} - test com.h2database diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java index 13767a20ea..d94ed6118d 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java +++ b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java @@ -17,8 +17,6 @@ package feast.core.job.dataflow; -import static com.google.common.base.Preconditions.checkNotNull; - import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.Job; import com.google.common.base.Strings; @@ -35,7 +33,7 @@ import feast.core.model.JobInfo; import feast.core.util.TypeConversion; import feast.ingestion.ImportJob; -import feast.ingestion.options.ImportJobPipelineOptions; +import feast.ingestion.options.ImportOptions; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -128,7 +126,7 @@ public void abortJob(String dataflowJobId) { private String submitDataflowJob(String jobName, List featureSets, Store sink, boolean update) { try { - ImportJobPipelineOptions pipelineOptions = getPipelineOptions(jobName, featureSets, sink, update); + ImportOptions pipelineOptions = getPipelineOptions(jobName, featureSets, sink, update); DataflowPipelineJob pipelineResult = runPipeline(pipelineOptions); String jobId = waitForJobToRun(pipelineResult); return jobId; @@ -138,11 +136,10 @@ private String submitDataflowJob(String jobName, List featureSet } } - private ImportJobPipelineOptions getPipelineOptions(String jobName, List featureSets, Store sink, + private ImportOptions getPipelineOptions(String jobName, List featureSets, Store sink, boolean update) throws InvalidProtocolBufferException { String[] args = TypeConversion.convertMapToArgs(defaultOptions); - ImportJobPipelineOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args) - .as(ImportJobPipelineOptions.class); + ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class); Printer printer = JsonFormat.printer(); List featureSetsJson = new ArrayList<>(); for (FeatureSetSpec featureSet : featureSets) { @@ -164,7 +161,7 @@ private ImportJobPipelineOptions getPipelineOptions(String jobName, List featureSetSpecs, StoreProto.Store sinkSpec) { try { - ImportJobPipelineOptions pipelineOptions = getPipelineOptions(featureSetSpecs, sinkSpec); + ImportOptions pipelineOptions = getPipelineOptions(featureSetSpecs, sinkSpec); PipelineResult pipelineResult = runPipeline(pipelineOptions); DirectJob directJob = new DirectJob(name, pipelineResult); jobs.add(directJob); @@ -85,12 +85,11 @@ public String startJob(String name, List featureSetSpecs, } } - private ImportJobPipelineOptions getPipelineOptions(List featureSetSpecs, + private ImportOptions getPipelineOptions(List featureSetSpecs, StoreProto.Store sink) throws InvalidProtocolBufferException { String[] args = TypeConversion.convertMapToArgs(defaultOptions); - ImportJobPipelineOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args) - .as(ImportJobPipelineOptions.class); + ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class); Printer printer = JsonFormat.printer(); List featureSetsJson = new ArrayList<>(); for (FeatureSetSpec featureSetSpec : featureSetSpecs) { @@ -136,7 +135,7 @@ public void abortJob(String extId) { jobs.remove(extId); } - public PipelineResult runPipeline(ImportJobPipelineOptions pipelineOptions) + public PipelineResult runPipeline(ImportOptions pipelineOptions) throws IOException { return ImportJob.runPipeline(pipelineOptions); } diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java index 4aad1de0c2..ba017aa2fd 100644 --- a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java +++ b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java @@ -37,7 +37,7 @@ import feast.core.StoreProto.Store.StoreType; import feast.core.config.FeastProperties.MetricsProperties; import feast.core.exception.JobExecutionException; -import feast.ingestion.options.ImportJobPipelineOptions; +import feast.ingestion.options.ImportOptions; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -93,8 +93,8 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException { String expectedExtJobId = "feast-job-0"; String jobName = "job"; - ImportJobPipelineOptions expectedPipelineOptions = PipelineOptionsFactory.fromArgs("") - .as(ImportJobPipelineOptions.class); + ImportOptions expectedPipelineOptions = PipelineOptionsFactory.fromArgs("") + .as(ImportOptions.class); expectedPipelineOptions.setRunner(DataflowRunner.class); expectedPipelineOptions.setProject("project"); expectedPipelineOptions.setRegion("region"); @@ -105,8 +105,8 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException { expectedPipelineOptions .setFeatureSetSpecJson(Lists.newArrayList(printer.print(featureSetSpec))); - ArgumentCaptor captor = ArgumentCaptor - .forClass(ImportJobPipelineOptions.class); + ArgumentCaptor captor = ArgumentCaptor + .forClass(ImportOptions.class); DataflowPipelineJob mockPipelineResult = Mockito.mock(DataflowPipelineJob.class); when(mockPipelineResult.getState()).thenReturn(State.RUNNING); @@ -116,7 +116,7 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException { String jobId = dfJobManager.startJob(jobName, Lists.newArrayList(featureSetSpec), store); verify(dfJobManager, times(1)).runPipeline(captor.capture()); - ImportJobPipelineOptions actualPipelineOptions = captor.getValue(); + ImportOptions actualPipelineOptions = captor.getValue(); expectedPipelineOptions.setOptionsId(actualPipelineOptions.getOptionsId()); // avoid comparing this value diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java index c89f758b9e..93f2519c38 100644 --- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java +++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java @@ -17,7 +17,7 @@ import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; import feast.core.config.FeastProperties.MetricsProperties; -import feast.ingestion.options.ImportJobPipelineOptions; +import feast.ingestion.options.ImportOptions; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -68,8 +68,8 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { Printer printer = JsonFormat.printer(); - ImportJobPipelineOptions expectedPipelineOptions = PipelineOptionsFactory.fromArgs("") - .as(ImportJobPipelineOptions.class); + ImportOptions expectedPipelineOptions = PipelineOptionsFactory.fromArgs("") + .as(ImportOptions.class); expectedPipelineOptions.setAppName("DirectRunnerJobManager"); expectedPipelineOptions.setRunner(DirectRunner.class); expectedPipelineOptions.setBlockOnRun(false); @@ -78,8 +78,8 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { .setFeatureSetSpecJson(Lists.newArrayList(printer.print(featureSetSpec))); String expectedJobId = "feast-job-0"; - ArgumentCaptor pipelineOptionsCaptor = ArgumentCaptor - .forClass(ImportJobPipelineOptions.class); + ArgumentCaptor pipelineOptionsCaptor = ArgumentCaptor + .forClass(ImportOptions.class); ArgumentCaptor directJobCaptor = ArgumentCaptor .forClass(DirectJob.class); @@ -90,7 +90,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { verify(drJobManager, times(1)).runPipeline(pipelineOptionsCaptor.capture()); verify(directJobRegistry, times(1)).add(directJobCaptor.capture()); - ImportJobPipelineOptions actualPipelineOptions = pipelineOptionsCaptor.getValue(); + ImportOptions actualPipelineOptions = pipelineOptionsCaptor.getValue(); DirectJob jobStarted = directJobCaptor.getValue(); expectedPipelineOptions.setOptionsId(actualPipelineOptions.getOptionsId()); // avoid comparing this value diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 8bfbec7dc7..9516d8c33b 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -19,63 +19,21 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - feast - ${revision} - feast-ingestion - jar - Feast Ingestion - - 0.3.0-SNAPSHOT - UTF-8 + + feast + feast-parent + ${revision} + - 2.16.0 - 1.91.0 - 1.21.1 - 3.6.1 - 3.6.1 - + Feast Ingestion + feast-ingestion - - - kr.motd.maven - os-maven-plugin - 1.6.0 - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.7.0 - - 1.8 - 1.8 - - org.xolstice.maven.plugins protobuf-maven-plugin - 0.5.1 - - - com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier} - - grpc-java - - io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier} - - - - - - compile - compile-custom - test-compile - - - org.apache.maven.plugins @@ -161,6 +119,12 @@ 6.0.13.Final + + + io.opencensus + opencensus-contrib-http-util + + com.google.auto.value auto-value-annotations @@ -173,54 +137,43 @@ provided - com.google.cloud google-cloud-nio - 0.83.0-alpha io.grpc grpc-stub - 1.21.1 com.google.cloud google-cloud-storage - ${com.google.cloud.version} com.google.cloud google-cloud-bigquery - ${com.google.cloud.version} - + org.hamcrest hamcrest-library - 1.3 - test org.mockito mockito-core - 2.28.2 - test com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.9.9 com.fasterxml.jackson.module jackson-module-jsonSchema - 2.9.9 @@ -239,7 +192,6 @@ joda-time joda-time - 2.9.9 @@ -269,7 +221,6 @@ redis.clients jedis - 3.1.0 io.prometheus @@ -279,8 +230,6 @@ org.projectlombok lombok - 1.16.22 - provided @@ -295,24 +244,18 @@ com.fasterxml.jackson.dataformat jackson-dataformat-csv - 2.9.6 test com.github.kstyrc embedded-redis - 0.6 - test com.google.guava guava - 26.0-jre - compile - diff --git a/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java b/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java index 0a52b4bc78..5f0eedb994 100644 --- a/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java +++ b/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java @@ -77,7 +77,6 @@ List getAllLines(Path path) throws IOException { List files = Files.walk(path).collect(Collectors.toList()); List lines = Lists.newArrayList(); for (Path file : files) { - System.out.println(file); if (file.toFile().isFile()) { lines.addAll(Files.readAllLines(file)); } diff --git a/ingestion/src/test/resources/logback-test.xml b/ingestion/src/test/resources/logback-test.xml index 9a301f3bc6..429ae39aec 100644 --- a/ingestion/src/test/resources/logback-test.xml +++ b/ingestion/src/test/resources/logback-test.xml @@ -16,13 +16,13 @@ --> - + %d{HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n - + - \ No newline at end of file + diff --git a/pom.xml b/pom.xml index a8377fe951..3c4e74bcc5 100644 --- a/pom.xml +++ b/pom.xml @@ -17,19 +17,34 @@ 4.0.0 + + Feast Parent feast feast-parent ${revision} pom - Feast Parent + + + ingestion + core + serving + - 0.3.0 + 0.3.0-SNAPSHOT UTF-8 + UTF-8 + + 1.17.1 3.6.1 - 1.14.0 3.6.1 2.0.9.RELEASE + 2.16.0 + 1.91.0 + 1.3 + 1.0.7 + + 0.21.0 @@ -43,11 +58,236 @@ - - ingestion - core - serving - + + + + + com.google.cloud + google-cloud-bigquery + ${com.google.cloud.version} + + + com.google.cloud + google-cloud-storage + ${com.google.cloud.version} + + + + + com.google.cloud + google-cloud-nio + 0.83.0-alpha + + + + io.opencensus + opencensus-api + ${opencensus.version} + + + io.opencensus + opencensus-contrib-grpc-util + ${opencensus.version} + + + io.opencensus + opencensus-contrib-http-util + ${opencensus.version} + + + + + io.grpc + grpc-netty + ${grpcVersion} + + + io.grpc + grpc-services + ${grpcVersion} + + + io.grpc + grpc-stub + ${grpcVersion} + + + io.grpc + grpc-testing + ${grpcVersion} + test + + + + + com.fasterxml.jackson.core + jackson-annotations + 2.9.9 + + + com.fasterxml.jackson.core + jackson-core + 2.9.9 + + + com.fasterxml.jackson.core + jackson-databind + 2.9.9.1 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + 2.9.6 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.9.9 + + + com.fasterxml.jackson.module + jackson-module-jsonSchema + 2.9.9 + + + + + org.springframework.boot + spring-boot-configuration-processor + ${springBootVersion} + + + org.springframework.boot + spring-boot-devtools + ${springBootVersion} + + + org.springframework.boot + spring-boot-starter-actuator + ${springBootVersion} + + + org.springframework.boot + spring-boot-starter-log4j2 + ${springBootVersion} + + + org.springframework.boot + spring-boot-starter-test + ${springBootVersion} + test + + + org.springframework.boot + spring-boot-starter-web + ${springBootVersion} + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-test + ${springBootVersion} + test + + + org.springframework.boot + spring-boot-test-autoconfigure + ${springBootVersion} + test + + + org.springframework + spring-test + 5.0.8.RELEASE + test + + + + org.lognet + grpc-spring-boot-starter + 2.4.1 + + + + + commons-codec + commons-codec + 1.10 + + + com.datadoghq + java-dogstatsd-client + 2.6.1 + + + com.google.guava + guava + 26.0-jre + + + com.google.protobuf + protobuf-java-util + ${protobufVersion} + + + io.micrometer + micrometer-core + ${micrometer.version} + + + io.micrometer + micrometer-registry-statsd + ${micrometer.version} + + + joda-time + joda-time + 2.9.9 + + + org.projectlombok + lombok + 1.18.2 + provided + + + redis.clients + jedis + 3.1.0 + + + + + com.github.kstyrc + embedded-redis + 0.6 + test + + + org.hamcrest + hamcrest-all + ${hamcrest.version} + test + + + org.hamcrest + hamcrest-library + ${hamcrest.version} + test + + + org.mockito + mockito-core + 2.28.2 + test + + + @@ -57,6 +297,7 @@ 1.6.0 + org.apache.maven.plugins @@ -82,6 +323,11 @@ + + io.fabric8 + docker-maven-plugin + 0.20.1 + org.codehaus.mojo exec-maven-plugin diff --git a/serving/pom.xml b/serving/pom.xml index 4ed3d91115..9a066facfc 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -20,15 +20,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - - 0.3.0 - UTF-8 - 3.6.1 - 1.21.0 - 3.6.1 - 2.0.9.RELEASE - - feast feast-parent @@ -37,17 +28,8 @@ feast-serving Feast Serving - jar - - - false - - central - bintray - https://jcenter.bintray.com - spring-plugins Spring Plugins @@ -91,7 +73,6 @@ org.springframework.boot spring-boot-configuration-processor - ${springBootVersion} true @@ -99,25 +80,16 @@ org.springframework.boot spring-boot-starter-web - ${springBootVersion} - - - org.springframework.boot - spring-boot-starter-logging - - org.springframework.boot spring-boot-starter-log4j2 - ${springBootVersion} org.springframework.boot spring-boot-devtools - ${springBootVersion} true @@ -131,38 +103,32 @@ org.lognet grpc-spring-boot-starter - 2.4.1 org.springframework.boot spring-boot-starter-actuator - ${springBootVersion} io.grpc grpc-netty - ${grpcVersion} io.grpc grpc-services - ${grpcVersion} io.grpc grpc-stub - ${grpcVersion} com.google.protobuf protobuf-java-util - ${protobufVersion} @@ -175,25 +141,21 @@ redis.clients jedis - 2.9.0 com.google.guava guava - 26.0-jre commons-codec commons-codec - 1.10 joda-time joda-time - 2.9.9 @@ -217,70 +179,54 @@ io.micrometer micrometer-core - 1.0.7 io.micrometer micrometer-registry-statsd - 1.0.7 com.datadoghq java-dogstatsd-client - 2.6.1 com.google.cloud google-cloud-bigquery - 1.91.0 com.google.cloud google-cloud-storage - 1.91.0 io.opencensus - opencensus-impl - 0.24.0 - runtime + opencensus-api - org.projectlombok lombok - 1.18.2 - provided io.grpc grpc-testing - ${grpcVersion} - test org.springframework.boot spring-boot-starter-test - ${springBootVersion} - test com.github.kstyrc embedded-redis - 0.6 - test org.mockito @@ -292,7 +238,7 @@ org.junit.jupiter junit-jupiter - RELEASE + 5.5.2 test @@ -319,11 +265,8 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.9.9 - compile - @@ -352,7 +295,6 @@ io.fabric8 docker-maven-plugin - 0.20.1 prepare-bigtable-emulator @@ -408,7 +350,6 @@ io.fabric8 docker-maven-plugin - 0.20.1 prepare-bigtable-emulator From 88b3fcdddc4bddf914d0b28ebec048ff9af0dec7 Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Sat, 19 Oct 2019 20:09:29 +0700 Subject: [PATCH 2/5] Enable -Xlint warnings, fix trivial deprecations --- ingestion/pom.xml | 4 ---- .../main/java/feast/ingestion/transform/ReadFromSource.java | 3 +-- pom.xml | 6 ++++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 9516d8c33b..99242f6ea5 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -96,10 +96,6 @@ jar - - - none - diff --git a/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java b/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java index 3c6e9c8f41..b950903844 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java @@ -3,7 +3,6 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.TextFormat; import feast.core.SourceProto.Source; import feast.core.SourceProto.SourceType; import feast.ingestion.values.FailedElement; @@ -148,7 +147,7 @@ public void processElement(ProcessContext context) { FailedElement.newBuilder() .setTransformName("KafkaRecordToFeatureRow") .setJobName(context.getPipelineOptions().getJobName()) - .setPayload(TextFormat.printToString(featureRow)) + .setPayload(featureRow.toString()) .setErrorMessage(error) .build()); } else { diff --git a/pom.xml b/pom.xml index 3c4e74bcc5..c5413264e1 100644 --- a/pom.xml +++ b/pom.xml @@ -306,6 +306,12 @@ 1.8 1.8 + + -Xlint:all + + + -Xdoclint:-syntax + From d4a318e07303eb263ef4e7e2df7caa75b0f2856c Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Sat, 19 Oct 2019 22:24:50 +0700 Subject: [PATCH 3/5] Make sdk/java a child module of the main build Updating the protobuf/protoc version everywhere to what the SDK used, because it requires new APIs. --- ingestion/pom.xml | 1 - pom.xml | 31 +++++++++++++++--- protos/feast/types/Value.proto | 2 -- sdk/java/pom.xml | 59 ++++++---------------------------- 4 files changed, 36 insertions(+), 57 deletions(-) diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 99242f6ea5..f97ab1923b 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -89,7 +89,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.0.1 attach-javadocs diff --git a/pom.xml b/pom.xml index c5413264e1..e44975d386 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ ingestion core serving + sdk/java @@ -36,8 +37,8 @@ UTF-8 1.17.1 - 3.6.1 - 3.6.1 + 3.10.0 + 3.10.0 2.0.9.RELEASE 2.16.0 1.91.0 @@ -101,6 +102,16 @@ grpc-netty ${grpcVersion} + + io.grpc + grpc-netty-shaded + ${grpcVersion} + + + io.grpc + grpc-protobuf + ${grpcVersion} + io.grpc grpc-services @@ -229,6 +240,11 @@ guava 26.0-jre + + com.google.protobuf + protobuf-java + ${protobufVersion} + com.google.protobuf protobuf-java-util @@ -294,7 +310,7 @@ kr.motd.maven os-maven-plugin - 1.6.0 + 1.6.2 @@ -302,7 +318,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.7.0 + 3.8.1 1.8 1.8 @@ -334,6 +350,11 @@ docker-maven-plugin 0.20.1 + + org.apache.maven.plugins + maven-javadoc-plugin + 3.1.0 + org.codehaus.mojo exec-maven-plugin @@ -357,7 +378,7 @@ org.xolstice.maven.plugins protobuf-maven-plugin - 0.5.1 + 0.6.1 com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier} diff --git a/protos/feast/types/Value.proto b/protos/feast/types/Value.proto index 2cdb2c924e..065497f30a 100644 --- a/protos/feast/types/Value.proto +++ b/protos/feast/types/Value.proto @@ -16,8 +16,6 @@ syntax = "proto3"; -import "google/protobuf/timestamp.proto"; - package feast.types; option java_package = "feast.types"; diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml index e9690cff54..6a94a54fcf 100644 --- a/sdk/java/pom.xml +++ b/sdk/java/pom.xml @@ -4,13 +4,18 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.gojek.feast + Feast SDK for Java feast-client - 0.3-SNAPSHOT + + + feast + feast-parent + ${revision} + ../.. + - 1.24.0 - 3.10.0 + 5.5.2 @@ -19,27 +24,22 @@ io.grpc grpc-netty-shaded - ${grpc.version} io.grpc grpc-protobuf - ${grpc.version} io.grpc grpc-stub - ${grpc.version} com.google.protobuf protobuf-java-util - ${protobuf.version} com.google.protobuf protobuf-java - ${protobuf.version} @@ -66,54 +66,15 @@ - - - kr.motd.maven - os-maven-plugin - 1.6.2 - - org.xolstice.maven.plugins protobuf-maven-plugin - 0.6.1 - - com.google.protobuf:protoc:3.10.0:exe:${os.detected.classifier} - - grpc-java - io.grpc:protoc-gen-grpc-java:1.24.0:exe:${os.detected.classifier} - - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 8 - 8 - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.0 org.apache.maven.plugins maven-javadoc-plugin - 3.1.0 true @@ -121,4 +82,4 @@ - \ No newline at end of file + From e97bf59e8a97d09afc7f099466c41f1d2ec56d68 Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Sun, 20 Oct 2019 01:09:50 +0700 Subject: [PATCH 4/5] Don't recompile protos when they haven't changed Checks mtime versus target/ directory. Saves considerable build time. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index e44975d386..4a0734eb15 100644 --- a/pom.xml +++ b/pom.xml @@ -380,6 +380,7 @@ protobuf-maven-plugin 0.6.1 + true com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier} From 5e65a077320104954370e634e422d2609f312ecd Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Sun, 20 Oct 2019 14:00:46 +0700 Subject: [PATCH 5/5] Remove unused dependencies According to mvn dependency:analyze -DignoreNonCompile and running tests. Also added some undeclared direct dependencies, though not covering them comprehensively yet. --- core/pom.xml | 110 +----------------- ingestion/pom.xml | 63 +++++----- .../feast/store/FileStoreOptionsTest.java | 1 - pom.xml | 42 +++++-- sdk/java/pom.xml | 1 - serving/pom.xml | 78 ++----------- .../serving/service/CachedSpecService.java | 2 - .../service/CachedSpecServiceTest.java | 7 -- 8 files changed, 78 insertions(+), 226 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 5c6ce72472..e4c63bf845 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -85,6 +85,12 @@ feast-ingestion ${project.version} + + + javax.inject + javax.inject + 1 + org.springframework.boot @@ -119,11 +125,6 @@ - - - io.grpc - grpc-netty - io.grpc @@ -151,16 +152,6 @@ gson 2.8.5 - - - commons-codec - commons-codec - - - - joda-time - joda-time - com.google.api-client google-api-client @@ -172,84 +163,16 @@ v1b3-rev266-1.25.0 - - - org.jdbi - jdbi3 - 3.0.0-beta2 - - - - org.jdbi - jdbi3-sqlobject - 3.4.0 - - - - org.postgresql - postgresql - 42.2.5 - org.hibernate hibernate-core 5.3.6.Final - - - com.google.cloud.bigtable - bigtable-hbase-2.x-shaded - 1.5.0 - - - - com.google.cloud - google-cloud-bigquery - - - com.google.cloud - google-cloud-nio - - - org.apache.flink - flink-clients_2.11 - 1.5.5 - - - - - com.github.spullara.mustache.java - compiler - 0.9.5 - - - - - io.micrometer - micrometer-core - - - - io.micrometer - micrometer-registry-statsd - - - com.datadoghq - java-dogstatsd-client - org.apache.kafka kafka-clients - 2.3.0 - - - - - org.apache.commons - commons-exec - 1.3 @@ -258,11 +181,6 @@ lombok - - - io.grpc - grpc-testing - org.hamcrest @@ -282,17 +200,7 @@ 2.23.0 test - - com.squareup.okhttp3 - mockwebserver - 3.11.0 - test - - - org.springframework - spring-test - org.springframework.boot spring-boot-test @@ -301,11 +209,5 @@ org.springframework.boot spring-boot-test-autoconfigure - - com.h2database - h2 - 1.4.198 - test - diff --git a/ingestion/pom.xml b/ingestion/pom.xml index f97ab1923b..d79637ecbd 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -109,15 +109,15 @@ - org.hibernate.validator - hibernate-validator - 6.0.13.Final + javax.validation + validation-api + 2.0.1.Final - - io.opencensus - opencensus-contrib-http-util + org.hibernate.validator + hibernate-validator + 6.0.13.Final @@ -132,11 +132,6 @@ provided - - com.google.cloud - google-cloud-nio - - io.grpc grpc-stub @@ -151,7 +146,10 @@ google-cloud-bigquery - + + org.hamcrest + hamcrest-core + org.hamcrest hamcrest-library @@ -162,6 +160,18 @@ mockito-core + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -174,7 +184,10 @@ com.google.protobuf protobuf-java - ${protobufVersion} + + + com.google.protobuf + protobuf-java-util @@ -207,12 +220,6 @@ ${org.apache.beam.version} - - org.apache.kafka - kafka_2.12 - 2.3.0 - - redis.clients jedis @@ -227,24 +234,18 @@ lombok + + org.slf4j + slf4j-api + + ch.qos.logback logback-classic 1.2.3 - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - test - - - - com.github.kstyrc - embedded-redis + runtime diff --git a/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java b/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java index dc6629bba0..562944b493 100644 --- a/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertEquals; import com.google.common.collect.ImmutableMap; -import feast.store.FileStoreOptions; import org.joda.time.Duration; import org.junit.Test; import feast.options.OptionsParser; diff --git a/pom.xml b/pom.xml index 4a0734eb15..f3735945d2 100644 --- a/pom.xml +++ b/pom.xml @@ -224,6 +224,13 @@ 2.4.1 + + + org.slf4j + slf4j-api + 1.7.25 + + commons-codec @@ -265,6 +272,11 @@ joda-time 2.9.9 + + org.apache.kafka + kafka-clients + 2.3.0 + org.projectlombok lombok @@ -285,22 +297,28 @@ test - org.hamcrest - hamcrest-all - ${hamcrest.version} - test + org.hamcrest + hamcrest-all + ${hamcrest.version} + test - org.hamcrest - hamcrest-library - ${hamcrest.version} - test + org.hamcrest + hamcrest-core + ${hamcrest.version} + test - org.mockito - mockito-core - 2.28.2 - test + org.hamcrest + hamcrest-library + ${hamcrest.version} + test + + + org.mockito + mockito-core + 2.28.2 + test diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml index 6a94a54fcf..43465ad911 100644 --- a/sdk/java/pom.xml +++ b/sdk/java/pom.xml @@ -46,7 +46,6 @@ org.slf4j slf4j-api - 1.8.0-beta4 diff --git a/serving/pom.xml b/serving/pom.xml index 9a066facfc..fd24bcafcf 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -70,6 +70,12 @@ + + + org.slf4j + slf4j-api + + org.springframework.boot spring-boot-configuration-processor @@ -93,12 +99,6 @@ true - - org.apache.logging.log4j - log4j-web - 2.12.0 - - org.lognet @@ -110,11 +110,6 @@ spring-boot-starter-actuator - - - io.grpc - grpc-netty - io.grpc @@ -147,11 +142,6 @@ com.google.guava guava - - - commons-codec - commons-codec - joda-time @@ -169,25 +159,9 @@ 0.31.0 - io.opentracing.contrib - opentracing-concurrent - 0.2.0 - - - - - - io.micrometer - micrometer-core - - - - io.micrometer - micrometer-registry-statsd - - - com.datadoghq - java-dogstatsd-client + io.opentracing + opentracing-noop + 0.31.0 @@ -201,12 +175,6 @@ google-cloud-storage - - - io.opencensus - opencensus-api - - org.projectlombok @@ -223,11 +191,6 @@ org.springframework.boot spring-boot-starter-test - - - com.github.kstyrc - embedded-redis - org.mockito mockito-core @@ -235,20 +198,6 @@ test - - org.junit.jupiter - junit-jupiter - 5.5.2 - test - - - - com.google.guava - guava-testlib - 26.0-jre - test - - org.hibernate @@ -257,19 +206,12 @@ - - org.apache.commons - commons-collections4 - 4.4 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - profile-local @@ -335,9 +277,9 @@ - + profile-ci diff --git a/serving/src/main/java/feast/serving/service/CachedSpecService.java b/serving/src/main/java/feast/serving/service/CachedSpecService.java index dfb62eca57..4fdf3e147a 100644 --- a/serving/src/main/java/feast/serving/service/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/service/CachedSpecService.java @@ -9,8 +9,6 @@ import feast.core.CoreServiceProto.GetFeatureSetsRequest; import feast.core.CoreServiceProto.GetFeatureSetsRequest.Filter; import feast.core.CoreServiceProto.GetFeatureSetsResponse; -import feast.core.CoreServiceProto.GetStoresRequest; -import feast.core.CoreServiceProto.GetStoresResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.FeatureSetProto.FeatureSetSpec; diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index 509e24689d..6b27f1a50e 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -1,8 +1,6 @@ package feast.serving.service; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.*; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -10,9 +8,6 @@ import com.google.common.collect.Lists; import feast.core.CoreServiceProto.GetFeatureSetsRequest; import feast.core.CoreServiceProto.GetFeatureSetsResponse; -import feast.core.CoreServiceProto.GetStoresRequest; -import feast.core.CoreServiceProto.GetStoresRequest.Filter; -import feast.core.CoreServiceProto.GetStoresResponse; import feast.core.CoreServiceProto.UpdateStoreRequest; import feast.core.CoreServiceProto.UpdateStoreResponse; import feast.core.FeatureSetProto.FeatureSetSpec; @@ -24,7 +19,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -34,7 +28,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mock; -import redis.embedded.Redis; public class CachedSpecServiceTest {