Merge branch '0.3-dev' into 0.3-dev-serving-api-change

Makefile

@@ -14,9 +14,6 @@
 #  limitations under the License.
 # 
 
-VERSION_FILE                = VERSION
-FEAST_VERSION               = `cat $(VERSION_FILE)`
-
 test:
 	mvn test
 
@@ -32,13 +29,13 @@ build-cli:
 	$(MAKE) -C cli build-all
 
 build-java:
-	mvn clean verify -Drevision=$(FEAST_VERSION)
+	mvn clean verify -Drevision=$(VERSION)
 
 build-docker:
-	docker build -t $(registry)/feast-core:$(version) -f docker/core/Dockerfile .
-	docker build -t $(registry)/feast-serving:$(version) -f docker/serving/Dockerfile .
+	docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
+	docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
 
 build-push-docker:
-	@$(MAKE) build-docker registry=$(registry) version=$(version)
-	docker push $(registry)/feast-core:$(version)
-	docker push $(registry)/feast-serving:$(version)
+	@$(MAKE) build-docker registry=$(REGISTRY) version=$(VERSION)
+	docker push $(REGISTRY)/feast-core:$(VERSION)
+	docker push $(REGISTRY)/feast-serving:$(VERSION)

infra/charts/feast/requirements.yaml

@@ -9,5 +9,5 @@ dependencies:
   condition: redis.provision
 - name: kafka
   version: 0.17.0
-  repository: "@stable"
-  condition: kafka.provision
+  repository: "@incubator"
+  condition: kafka.provision

infra/charts/feast/templates/core-config.yaml

@@ -11,4 +11,52 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
   config: |
-{{ toYaml core.config | indent 4 }}
+
+    # Please see gojek/feast/core/src/main/resources/application.yml for an explanation of config
+
+    grpc:
+      port: {{ .Values.core.service.grpc.port }}
+      enable-reflection: {{ .Values.core.enableReflection }}
+
+    feast:
+      store:
+        serving-type: {{ .Values.core.store.serving.type }}
+        serving-options:
+          host: {{ .Values.core.store.serving.options.redis.host }}
+          port: {{ .Values.core.store.serving.options.redis.port }}
+          subscriptions: ".*:>0"
+        warehouse-type: ""
+      jobs:
+        runner: {{ .Values.core.jobs.runner }}
+        options: {{ .Values.core.jobs.options }}
+      stream:
+        # Feature stream type. Only kafka is supported.
+        type: {{ .Values.stream.type }}
+        # Feature stream options.
+        options:
+          bootstrapServers: {{ .Values.stream.options.bootstrapServers }}
+          replicationFactor: {{ .Values.stream.options.replicationFactor }}
+          partitions: {{ .Values.stream.options.partitions }}
+      statsd:
+        host: {{ .Values.statsd.host }}
+        port: {{ .Values.statsd.port }}
+
+    spring:
+      jpa:
+        properties.hibernate.format_sql: true
+        hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
+        hibernate.ddl-auto: update
+      datasource:
+        url: jdbc:postgresql://{{ .Values.core.postgres.host }}:{{ .Values.core.postgres.port }}/{{ .Values.core.postgres.database }}
+        username: {{ .Values.core.postgres.username }}
+        password: {{ .Values.core.postgres.password }}
+
+    management:
+      metrics:
+        export:
+          simple:
+            enabled: false
+          statsd:
+            enabled: true
+            host: {{ .Values.statsd.host }}
+            port: {{ .Values.statsd.port }}

infra/charts/feast/templates/core-deploy.yaml

@@ -61,9 +61,11 @@ spec:
           requests:
             cpu: {{ .Values.core.resources.requests.cpu }}
             memory: {{ .Values.core.resources.requests.memory }}
+          {{- if .Values.core.resources.limits }}
           limits:
             cpu: {{ .Values.core.resources.limits.cpu }}
             memory: {{ .Values.core.resources.limits.memory }}
+          {{- end }}
         {{- if .Values.serviceAccount }}
         volumeMounts:
           - name: "{{ .Values.serviceAccount.name }}"

infra/charts/feast/templates/serving-deploy.yaml

@@ -62,22 +62,24 @@ spec:
           requests:
             cpu: "{{ .Values.serving.resources.requests.cpu }}"
             memory: "{{ .Values.serving.resources.requests.memory }}"
+          {{- if .Values.serving.resources.limits }}
           limits:
             cpu: "{{ .Values.serving.resources.limits.cpu }}"
             memory: "{{ .Values.serving.resources.limits.memory }}"
+          {{- end }}
         env:
           - name: FEAST_SERVING_HTTP_PORT
             value: "{{ .Values.serving.service.http.targetPort }}"
           - name: FEAST_SERVING_GRPC_PORT
             value: "{{ .Values.serving.service.grpc.targetPort }}"
           - name: FEAST_CORE_HOST
-            value: "{{ printf "%s.%s.svc.cluster.local" (include "feast.core.name" .) .Release.Namespace }}"
+            value: '{{ printf "%s.%s.svc.cluster.local" (include "feast.core.name" .) .Release.Namespace }}'
           - name: FEAST_CORE_GRPC_PORT
             value: "{{ .Values.core.service.grpc.port }}"
           - name: STORE_SERVING_TYPE
-            value: {{ .Values.store.serving.type }}
+            value: '{{ .Values.core.store.serving.type }}'
           - name: STORE_SERVING_OPTIONS
-            value: {{ .Values.store.serving.options | toJson }}
+            value: '{{ .Values.core.store.serving.options | toJson }}'
           - name: FEAST_MAX_NB_THREAD
             value: "{{ .Values.serving.config.maxNumberOfThread }}"
           - name: FEAST_MAX_ENTITY_PER_BATCH

infra/charts/feast/values.yaml

@@ -6,12 +6,9 @@ core:
     pullPolicy: IfNotPresent
     registry: gcr.io/kf-feast
     repository: feast-core
-    tag: "0.1.1"
+    tag: "0.3.0"
   replicaCount: 1
-  resources: 
-    limits: 
-      cpu: 4
-      memory: 6G
+  resources:
     requests: 
       cpu: 1
       memory: 2G
@@ -29,52 +26,59 @@ core:
     # loadBalancerSourceRanges sets the accepted IP ranges for firewall ingress rule
     # this firewall rule is usually created when the service type is "LoadBalancer"
     # loadBalancerSourceRanges: ["10.0.0.0/8"]
+  postgres:
+    host: 127.0.0.1
+    port: 5432
+    database: postgres
+    username: postgres
+    password: password
   jobs:
-    workspace: "/tmp"
     # runner specifies the Beam pipeline runner, use either DirectRunner (for development) or DataflowRunner (for production)
     runner: DirectRunner
     options: "{}"
     monitoring:
       period: 5000
       initialDelay: 60000
-  trainingDatasetPrefix: "fs"
-  # logType: JSON
   livenessProbe:
     initialDelaySeconds: 120
     failureThreshold: 3
   readinessProbe:
     initialDelaySeconds: 60
     failureThreshold: 1
+  enableReflection: true
+  store:
+    errors:
+      type: "STDOUT"
+    warehouse:
+      type: "BIGQUERY"
+    serving:
+      type: "REDIS"
+      options:
+        redis:
+          host: localhost
+          port: 6379
 
-# dataflow configuration is required when core.jobs.runner=DataflowRunner
-# dataflow:
-#   projectID: ${GCP_PROJECT}
-#   location: ${GCP_REGION}
+#  dataflow configuration is required when core.jobs.runner=DataflowRunner
+#  dataflow:
+#    projectID: ${GCP_PROJECT}
+#    location: ${GCP_REGION}
 
 ###### Feature stream ###### 
 stream:
   type: kafka
-  options: '{"bootstrapServers":""}'
+  options:
+    bootstrapServers: ""
+    replicationFactor: 1
+    partitions: 1
 
 kafka:
   provision: false
 
-###### Stores/Sinks ######
-store:
- errors:
-   type: "STDOUT"
- warehouse:
-   type: "BIGQUERY"
-   # options: '{"project": "gcp-project-id", "dataset": "feast"}'
- serving:
-   type: "REDIS"
-   # options: '{"host": "redis-master", "port": "6379"}'
-
-postgresql: 
+postgresql:
   provision: true
   persistence:
     enabled: true
-    
+
 redis:
   provision: false
   cluster:
@@ -93,12 +97,9 @@ serving:
     pullPolicy: IfNotPresent
     registry: gcr.io/kf-feast
     repository: feast-serving
-    tag: "0.1.1"
+    tag: "0.3.0"
   replicaCount: 1
-  resources: 
-    limits: 
-      cpu: 2
-      memory: 4G
+  resources:
     requests: 
       cpu: 1
       memory: 1G

infra/Dockerfiles/core/Dockerfile → infra/docker/core/Dockerfile

@@ -1,12 +1,20 @@
 FROM maven:3.6-jdk-8-slim as builder
 ARG REVISION=dev
+
+WORKDIR /build
+COPY pom.xml .
+COPY core/pom.xml core/pom.xml
+COPY ingestion/pom.xml ingestion/pom.xml
+COPY serving/pom.xml serving/pom.xml
+
+RUN mvn dependency:go-offline -B
+
 COPY . /build
 WORKDIR /build
-ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
 RUN mvn --projects core,ingestion -Drevision=$REVISION -DskipTests=true --batch-mode package
 
 FROM openjdk:8-jre as production
 ARG REVISION=dev
 COPY --from=builder /build/core/target/feast-core-$REVISION.jar /usr/share/feast/feast-core.jar
-COPY --from=builder /build/infra/Dockerfiles/core/bootstrap.sh /usr/share/feast/bootstrap.sh
+COPY --from=builder /build/infra/docker/core/bootstrap.sh /usr/share/feast/bootstrap.sh
 ENTRYPOINT ["/usr/share/feast/boostrap.sh"]

infra/Dockerfiles/core/bootstrap.sh → infra/docker/core/bootstrap.sh

@@ -2,7 +2,7 @@
 
 CONFIG_PATH_FLAG=""
 if [ -z "$CONFIG_PATH" ]; then
-CONFIG_PATH_FLAG="-Dspring.config.additional-location=${CONFIG_PATH}"
+CONFIG_PATH_FLAG="-Dspring.config.location=${CONFIG_PATH}"
 fi
 
 java $CONFIG_PATH_FLAG \

sdk/python/cli.py

@@ -26,6 +26,17 @@
 
 _logger = logging.getLogger(__name__)
 
+_common_options = [
+    click.option("--core-url", help="Set Feast core URL to connect to"),
+    click.option("--serving-url", help="Set Feast serving URL to connect to"),
+]
+
+
+def common_options(func):
+    for option in reversed(_common_options):
+        func = option(func)
+    return func
+
 
 @click.group()
 def cli():
@@ -36,7 +47,8 @@ def cli():
 @click.option(
     "--client-only", "-c", is_flag=True, help="Print only the version of the CLI"
 )
-def version(client_only: bool):
+@common_options
+def version(client_only: bool, **kwargs):
     """
     Displays version and connectivity information
     """
@@ -48,8 +60,12 @@ def version(client_only: bool):
 
         if not client_only:
             feast_client = Client(
-                core_url=feast_config.get_config_property_or_fail("core_url"),
-                serving_url=feast_config.get_config_property_or_fail("serving_url"),
+                core_url=feast_config.get_config_property_or_fail(
+                    "core_url", cli_config=kwargs
+                ),
+                serving_url=feast_config.get_config_property_or_fail(
+                    "serving_url", cli_config=kwargs
+                ),
             )
             feast_versions_dict.update(feast_client.version())