diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index fa373fea23c..caf31ab47fc 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -7,6 +7,9 @@ on: - synchronize - labeled +permissions: + pull-requests: read + jobs: lint-java: # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. @@ -23,6 +26,7 @@ jobs: # code from the PR. ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive + persist-credentials: false - name: Lint java run: make lint-java @@ -42,6 +46,7 @@ jobs: # code from the PR. ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive + persist-credentials: false - name: Set up JDK 11 uses: actions/setup-java@v1 with: @@ -84,6 +89,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'true' + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 id: setup-python @@ -120,6 +126,7 @@ jobs: # code from the PR. ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive + persist-credentials: false - name: Set up JDK 11 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml index d1aa7d16a3e..81732258455 100644 --- a/.github/workflows/lint_pr.yml +++ b/.github/workflows/lint_pr.yml @@ -1,20 +1,16 @@ name: lint-pr on: - pull_request_target: + pull_request: types: - opened - edited - synchronize -permissions: - # read-only perms specified due to use of pull_request_target in lieu of security label check - pull-requests: read - jobs: validate-title: if: - github.repository == 'feast-dev/feast' + github.event.pull_request.base.repo.full_name == 'feast-dev/feast' name: Validate PR title runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index f4a9132d292..59de3ce9585 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -11,6 +11,9 @@ on: #concurrency: # group: pr-integration-tests-${{ github.event.pull_request.number }} # cancel-in-progress: true +permissions: + actions: write + pull-requests: read jobs: integration-test-python: @@ -46,6 +49,7 @@ jobs: # code from the PR. ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 id: setup-python diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 3de72621931..6515d411f01 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -2,7 +2,7 @@ name: pr-local-integration-tests # This runs local tests with containerized stubs of online stores. This is the main dev workflow on: - pull_request_target: + pull_request: types: - opened - synchronize @@ -10,11 +10,10 @@ on: jobs: integration-test-python-local: - # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && - github.repository == 'feast-dev/feast' + github.event.pull_request.base.repo.full_name == 'feast-dev/feast' runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -27,10 +26,9 @@ jobs: steps: - uses: actions/checkout@v4 with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge + repository: ${{ github.event.repository.full_name }} # Uses the full repository name + ref: ${{ github.ref }} # Uses the ref from the event + token: ${{ secrets.GITHUB_TOKEN }} # Automatically provided token submodules: recursive - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 03429433134..c2a5bd21ee4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: needs: [get-version, publish-python-sdk] strategy: matrix: - component: [feature-server, feature-server-java, feature-transformation-server, feast-operator] + component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev diff --git a/.releaserc.js b/.releaserc.js index ee9c62a04df..60084dc24f2 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -66,7 +66,7 @@ module.exports = { "CHANGELOG.md", "java/pom.xml", "infra/charts/**/*.*", - "infra/feast-operator/**/*", + "infra/feast-helm-operator/**/*", "ui/package.json", "sdk/python/feast/ui/package.json", "sdk/python/feast/ui/yarn.lock" diff --git a/Makefile b/Makefile index 6831a583379..10187b8ebc0 100644 --- a/Makefile +++ b/Makefile @@ -430,15 +430,15 @@ build-feature-server-java-docker: -t $(REGISTRY)/feature-server-java:$(VERSION) \ -f java/infra/docker/feature-server/Dockerfile --load . -push-feast-operator-docker: - cd infra/feast-operator && \ - IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ +push-feast-helm-operator-docker: + cd infra/feast-helm-operator && \ + IMAGE_TAG_BASE=$(REGISTRY)/feast-helm-operator \ VERSION=$(VERSION) \ $(MAKE) docker-push -build-feast-operator-docker: - cd infra/feast-operator && \ - IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ +build-feast-helm-operator-docker: + cd infra/feast-helm-operator && \ + IMAGE_TAG_BASE=$(REGISTRY)/feast-helm-operator \ VERSION=$(VERSION) \ $(MAKE) docker-build diff --git a/docs/assets/feast_model_inference_architecture.png b/docs/assets/feast_model_inference_architecture.png new file mode 100644 index 00000000000..3ea4fba4d05 Binary files /dev/null and b/docs/assets/feast_model_inference_architecture.png differ diff --git a/docs/getting-started/architecture/model-inference.md b/docs/getting-started/architecture/model-inference.md index 3a061603c19..582657dbc43 100644 --- a/docs/getting-started/architecture/model-inference.md +++ b/docs/getting-started/architecture/model-inference.md @@ -1,5 +1,13 @@ # Feature Serving and Model Inference +![](../../assets/feast_model_inference_architecture.png) + + +{% hint style="info" %} +**Note:** this ML Infrastructure diagram highlights an orchestration pattern that is driven by a client application. +This is not the only approach that can be taken and different patterns will result in different trade-offs. +{% endhint %} + Production machine learning systems can choose from four approaches to serving machine learning predictions (the output of model inference): 1. Online model inference with online features diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index dc8b87e34f2..021a10ac1ca 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -225,7 +225,7 @@ helm install feast-release feast-charts/feast-feature-server \ This will deploy a single service. The service must have read access to the registry file on cloud storage and to the online store (e.g. via [podAnnotations](https://kubernetes-on-aws.readthedocs.io/en/latest/user-guide/iam-roles.html)). It will keep a copy of the registry in their memory and periodically refresh it, so expect some delays in update propagation in exchange for better performance. -> Alternatively, deploy the same helm chart with a [Kubernetes Operator](/infra/feast-operator). +> Alternatively, deploy the same helm chart with a [Kubernetes Operator](/infra/feast-helm-operator). ## 5. Using environment variables in your yaml configuration diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index b9ce7f408a0..06909bd5654 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -40,7 +40,7 @@ registry: path: postgresql://@localhost:5432/feast online_store: type: postgres - pgvector_enabled: true + vector_enabled: true vector_len: 384 host: 127.0.0.1 port: 5432 diff --git a/docs/reference/alpha-web-ui.md b/docs/reference/alpha-web-ui.md index 2caeed9e2ad..02dd107f1b4 100644 --- a/docs/reference/alpha-web-ui.md +++ b/docs/reference/alpha-web-ui.md @@ -45,11 +45,14 @@ Start with bootstrapping a React app with `create-react-app` npx create-react-app your-feast-ui ``` -Then, in your app folder, install Feast UI and its peer dependencies. Assuming you use yarn +Then, in your app folder, install Feast UI and optionally its peer dependencies. Assuming you use yarn ``` yarn add @feast-dev/feast-ui -yarn add @elastic/eui @elastic/datemath @emotion/react moment prop-types inter-ui react-query react-router-dom use-query-params zod typescript query-string d3 @types/d3 +# For custom UI using the Elastic UI Framework (optional): +yarn add @elastic/eui +# For general custom styling (optional): +yarn add @emotion/react ``` Edit `index.js` in the React app to use Feast UI. diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 77a9408d2bd..e4e2173ccd7 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -30,7 +30,7 @@ online_store: sslkey_path: /path/to/client-key.pem sslcert_path: /path/to/client-cert.pem sslrootcert_path: /path/to/server-ca.pem - pgvector_enabled: false + vector_enabled: false vector_len: 512 ``` {% endcode %} @@ -65,7 +65,7 @@ To compare this set of functionality against other online stores, please see the ## PGVector The Postgres online store supports the use of [PGVector](https://github.com/pgvector/pgvector) for storing feature values. -To enable PGVector, set `pgvector_enabled: true` in the online store configuration. +To enable PGVector, set `vector_enabled: true` in the online store configuration. The `vector_len` parameter can be used to specify the length of the vector. The default value is 512. diff --git a/infra/feast-operator/.gitignore b/infra/feast-helm-operator/.gitignore similarity index 100% rename from infra/feast-operator/.gitignore rename to infra/feast-helm-operator/.gitignore diff --git a/infra/feast-operator/Dockerfile b/infra/feast-helm-operator/Dockerfile similarity index 100% rename from infra/feast-operator/Dockerfile rename to infra/feast-helm-operator/Dockerfile diff --git a/infra/feast-operator/Makefile b/infra/feast-helm-operator/Makefile similarity index 98% rename from infra/feast-operator/Makefile rename to infra/feast-helm-operator/Makefile index f52911f4321..6712a37b4af 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-helm-operator/Makefile @@ -28,8 +28,8 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# feastdev/feast-operator-bundle:$VERSION and feastdev/feast-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= feastdev/feast-operator +# feastdev/feast-helm-operator-bundle:$VERSION and feastdev/feast-helm-operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= feastdev/feast-helm-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) diff --git a/infra/feast-operator/PROJECT b/infra/feast-helm-operator/PROJECT similarity index 94% rename from infra/feast-operator/PROJECT rename to infra/feast-helm-operator/PROJECT index 56b2532d859..fefe00db1ec 100644 --- a/infra/feast-operator/PROJECT +++ b/infra/feast-helm-operator/PROJECT @@ -8,7 +8,7 @@ layout: plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} -projectName: feast-operator +projectName: feast-helm-operator resources: - api: crdVersion: v1 diff --git a/infra/feast-operator/README.md b/infra/feast-helm-operator/README.md similarity index 100% rename from infra/feast-operator/README.md rename to infra/feast-helm-operator/README.md diff --git a/infra/feast-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml b/infra/feast-helm-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml similarity index 100% rename from infra/feast-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml rename to infra/feast-helm-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml diff --git a/infra/feast-operator/config/crd/kustomization.yaml b/infra/feast-helm-operator/config/crd/kustomization.yaml similarity index 100% rename from infra/feast-operator/config/crd/kustomization.yaml rename to infra/feast-helm-operator/config/crd/kustomization.yaml diff --git a/infra/feast-operator/config/default/kustomization.yaml b/infra/feast-helm-operator/config/default/kustomization.yaml similarity index 85% rename from infra/feast-operator/config/default/kustomization.yaml rename to infra/feast-helm-operator/config/default/kustomization.yaml index 6cd524d5199..3252e11de8d 100644 --- a/infra/feast-operator/config/default/kustomization.yaml +++ b/infra/feast-helm-operator/config/default/kustomization.yaml @@ -1,12 +1,12 @@ # Adds namespace to all resources. -namespace: feast-operator-system +namespace: feast-helm-operator-system # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: feast-operator- +namePrefix: feast-helm-operator- # Labels to add to all resources and selectors. #labels: diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-helm-operator/config/manager/kustomization.yaml similarity index 77% rename from infra/feast-operator/config/manager/kustomization.yaml rename to infra/feast-helm-operator/config/manager/kustomization.yaml index aba3224be6a..0d27c2149dd 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-helm-operator/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: feastdev/feast-operator + newName: feastdev/feast-helm-operator newTag: 0.40.0 diff --git a/infra/feast-operator/config/manager/manager.yaml b/infra/feast-helm-operator/config/manager/manager.yaml similarity index 92% rename from infra/feast-operator/config/manager/manager.yaml rename to infra/feast-helm-operator/config/manager/manager.yaml index d65e8a78902..03f1fe173d8 100644 --- a/infra/feast-operator/config/manager/manager.yaml +++ b/infra/feast-helm-operator/config/manager/manager.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/name: namespace app.kubernetes.io/instance: system app.kubernetes.io/component: manager - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: system --- @@ -21,8 +21,8 @@ metadata: app.kubernetes.io/name: deployment app.kubernetes.io/instance: controller-manager app.kubernetes.io/component: manager - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize spec: selector: @@ -68,7 +68,7 @@ spec: containers: - args: - --leader-elect - - --leader-election-id=feast-operator + - --leader-election-id=feast-helm-operator image: controller:latest name: manager securityContext: diff --git a/infra/feast-operator/config/manifests/kustomization.yaml b/infra/feast-helm-operator/config/manifests/kustomization.yaml similarity index 76% rename from infra/feast-operator/config/manifests/kustomization.yaml rename to infra/feast-helm-operator/config/manifests/kustomization.yaml index 392c30f6b6f..90dccb22c3b 100644 --- a/infra/feast-operator/config/manifests/kustomization.yaml +++ b/infra/feast-helm-operator/config/manifests/kustomization.yaml @@ -1,7 +1,7 @@ # These resources constitute the fully configured set of manifests # used to generate the 'manifests/' directory in a bundle. resources: -- bases/feast-operator.clusterserviceversion.yaml +- bases/feast-helm-operator.clusterserviceversion.yaml - ../default - ../samples - ../scorecard diff --git a/infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml b/infra/feast-helm-operator/config/rbac/feastfeatureserver_editor_role.yaml similarity index 87% rename from infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml rename to infra/feast-helm-operator/config/rbac/feastfeatureserver_editor_role.yaml index f03ac20fddc..4749204cf75 100644 --- a/infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml +++ b/infra/feast-helm-operator/config/rbac/feastfeatureserver_editor_role.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: feastfeatureserver-editor-role app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: feastfeatureserver-editor-role rules: diff --git a/infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml b/infra/feast-helm-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml similarity index 82% rename from infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml rename to infra/feast-helm-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml index 054eb5a1a20..66c985fd00d 100644 --- a/infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml +++ b/infra/feast-helm-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml @@ -5,8 +5,8 @@ metadata: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: feastfeatureserver-editor-role app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: feastfeatureserver-editor-rolebinding roleRef: diff --git a/infra/feast-operator/config/rbac/kustomization.yaml b/infra/feast-helm-operator/config/rbac/kustomization.yaml similarity index 100% rename from infra/feast-operator/config/rbac/kustomization.yaml rename to infra/feast-helm-operator/config/rbac/kustomization.yaml diff --git a/infra/feast-operator/config/rbac/leader_election_role.yaml b/infra/feast-helm-operator/config/rbac/leader_election_role.yaml similarity index 86% rename from infra/feast-operator/config/rbac/leader_election_role.yaml rename to infra/feast-helm-operator/config/rbac/leader_election_role.yaml index 0adc316dd39..f2523c129ff 100644 --- a/infra/feast-operator/config/rbac/leader_election_role.yaml +++ b/infra/feast-helm-operator/config/rbac/leader_election_role.yaml @@ -6,8 +6,8 @@ metadata: app.kubernetes.io/name: role app.kubernetes.io/instance: leader-election-role app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: leader-election-role rules: diff --git a/infra/feast-operator/config/rbac/leader_election_role_binding.yaml b/infra/feast-helm-operator/config/rbac/leader_election_role_binding.yaml similarity index 81% rename from infra/feast-operator/config/rbac/leader_election_role_binding.yaml rename to infra/feast-helm-operator/config/rbac/leader_election_role_binding.yaml index f745675c0e7..b56c4101678 100644 --- a/infra/feast-operator/config/rbac/leader_election_role_binding.yaml +++ b/infra/feast-helm-operator/config/rbac/leader_election_role_binding.yaml @@ -5,8 +5,8 @@ metadata: app.kubernetes.io/name: rolebinding app.kubernetes.io/instance: leader-election-rolebinding app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: leader-election-rolebinding roleRef: diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-helm-operator/config/rbac/role.yaml similarity index 82% rename from infra/feast-operator/config/rbac/role.yaml rename to infra/feast-helm-operator/config/rbac/role.yaml index 2469689484e..ffc82274830 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-helm-operator/config/rbac/role.yaml @@ -5,8 +5,8 @@ metadata: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: manager-role app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: manager-role rules: diff --git a/infra/feast-operator/config/rbac/role_binding.yaml b/infra/feast-helm-operator/config/rbac/role_binding.yaml similarity index 81% rename from infra/feast-operator/config/rbac/role_binding.yaml rename to infra/feast-helm-operator/config/rbac/role_binding.yaml index 3359e911695..9aa7f4264f3 100644 --- a/infra/feast-operator/config/rbac/role_binding.yaml +++ b/infra/feast-helm-operator/config/rbac/role_binding.yaml @@ -5,8 +5,8 @@ metadata: app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/instance: manager-rolebinding app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: manager-rolebinding roleRef: diff --git a/infra/feast-operator/config/rbac/service_account.yaml b/infra/feast-helm-operator/config/rbac/service_account.yaml similarity index 72% rename from infra/feast-operator/config/rbac/service_account.yaml rename to infra/feast-helm-operator/config/rbac/service_account.yaml index 7ba6f27c603..b2ab5d5e917 100644 --- a/infra/feast-operator/config/rbac/service_account.yaml +++ b/infra/feast-helm-operator/config/rbac/service_account.yaml @@ -5,8 +5,8 @@ metadata: app.kubernetes.io/name: serviceaccount app.kubernetes.io/instance: controller-manager-sa app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: feast-operator - app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/created-by: feast-helm-operator + app.kubernetes.io/part-of: feast-helm-operator app.kubernetes.io/managed-by: kustomize name: controller-manager namespace: system diff --git a/infra/feast-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml b/infra/feast-helm-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml similarity index 100% rename from infra/feast-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml rename to infra/feast-helm-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml diff --git a/infra/feast-operator/config/samples/kustomization.yaml b/infra/feast-helm-operator/config/samples/kustomization.yaml similarity index 100% rename from infra/feast-operator/config/samples/kustomization.yaml rename to infra/feast-helm-operator/config/samples/kustomization.yaml diff --git a/infra/feast-operator/config/scorecard/bases/config.yaml b/infra/feast-helm-operator/config/scorecard/bases/config.yaml similarity index 100% rename from infra/feast-operator/config/scorecard/bases/config.yaml rename to infra/feast-helm-operator/config/scorecard/bases/config.yaml diff --git a/infra/feast-operator/config/scorecard/kustomization.yaml b/infra/feast-helm-operator/config/scorecard/kustomization.yaml similarity index 100% rename from infra/feast-operator/config/scorecard/kustomization.yaml rename to infra/feast-helm-operator/config/scorecard/kustomization.yaml diff --git a/infra/feast-operator/config/scorecard/patches/basic.config.yaml b/infra/feast-helm-operator/config/scorecard/patches/basic.config.yaml similarity index 100% rename from infra/feast-operator/config/scorecard/patches/basic.config.yaml rename to infra/feast-helm-operator/config/scorecard/patches/basic.config.yaml diff --git a/infra/feast-operator/config/scorecard/patches/olm.config.yaml b/infra/feast-helm-operator/config/scorecard/patches/olm.config.yaml similarity index 100% rename from infra/feast-operator/config/scorecard/patches/olm.config.yaml rename to infra/feast-helm-operator/config/scorecard/patches/olm.config.yaml diff --git a/infra/feast-operator/helm-charts/feast-feature-server b/infra/feast-helm-operator/helm-charts/feast-feature-server similarity index 100% rename from infra/feast-operator/helm-charts/feast-feature-server rename to infra/feast-helm-operator/helm-charts/feast-feature-server diff --git a/infra/feast-operator/watches.yaml b/infra/feast-helm-operator/watches.yaml similarity index 100% rename from infra/feast-operator/watches.yaml rename to infra/feast-helm-operator/watches.yaml diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 505ef87b243..4b3967b23e9 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -10,7 +10,7 @@ infra/charts/feast/README.md 11 68 69 infra/charts/feast-feature-server/Chart.yaml 5 infra/charts/feast-feature-server/README.md 3 infra/charts/feast-feature-server/values.yaml 12 -infra/feast-operator/Makefile 6 -infra/feast-operator/config/manager/kustomization.yaml 8 +infra/feast-helm-operator/Makefile 6 +infra/feast-helm-operator/config/manager/kustomization.yaml 8 java/pom.xml 38 ui/package.json 3 diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 3114f03f523..9084eb4bcec 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,8 +1,9 @@ -FROM python:3.11 - +FROM debian:11-slim RUN apt update && \ apt install -y \ jq \ + python3 \ + python3-pip \ python3-dev \ build-essential @@ -18,4 +19,4 @@ RUN apt update RUN apt -y install libarrow-dev # modify permissions to support running with a random uid RUN mkdir -m 775 /.cache -RUN chmod g+w $(python -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json +RUN chmod g+w $(python3 -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 49e70839a9a..3be42056f0c 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,8 +1,10 @@ -FROM python:3.11 +FROM debian:11-slim RUN apt update && \ apt install -y \ jq \ + python3 \ + python3-pip \ python3-dev \ build-essential @@ -19,4 +21,4 @@ RUN apt update RUN apt -y install libarrow-dev # modify permissions to support running with a random uid RUN mkdir -m 775 /.cache -RUN chmod g+w $(python -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json +RUN chmod g+w $(python3 -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/materialization/kubernetes/Dockerfile b/sdk/python/feast/infra/materialization/kubernetes/Dockerfile index 38d4f5f1880..b6ecb490afd 100644 --- a/sdk/python/feast/infra/materialization/kubernetes/Dockerfile +++ b/sdk/python/feast/infra/materialization/kubernetes/Dockerfile @@ -1,7 +1,7 @@ -FROM python:3.11-slim-bullseye AS build +FROM debian:11-slim AS build RUN apt-get update && \ - apt-get install --no-install-suggests --no-install-recommends --yes git + apt-get install --no-install-suggests --no-install-recommends --yes git python3 python3-pip WORKDIR /app diff --git a/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py b/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py index a0c25b931a9..0152ca330c9 100644 --- a/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py +++ b/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py @@ -14,13 +14,14 @@ serialize_entity_key, ) from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel from feast.utils import _build_retrieve_online_document_record, to_naive_utc -class ElasticSearchOnlineStoreConfig(FeastConfigBaseModel): +class ElasticSearchOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """ Configuration for the ElasticSearch online store. NOTE: The class *must* end with the `OnlineStoreConfig` suffix. @@ -38,13 +39,6 @@ class ElasticSearchOnlineStoreConfig(FeastConfigBaseModel): # The number of rows to write in a single batch write_batch_size: Optional[int] = 40 - # The length of the vector value - vector_len: Optional[int] = 512 - - # The vector similarity metric to use in KNN search - # more details: https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html - similarity: Optional[str] = "cosine" - class ElasticSearchOnlineStore(OnlineStore): _client: Optional[Elasticsearch] = None diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 8125da33be0..7c099c80ecc 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -23,10 +23,9 @@ from feast import Entity from feast.feature_view import FeatureView from feast.infra.key_encoding_utils import get_list_val_str, serialize_entity_key -from feast.infra.online_stores.contrib.singlestore_online_store.singlestore import ( - _to_naive_utc, -) +from feast.infra.online_stores.helpers import _to_naive_utc from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.infra.utils.postgres.connection_utils import ( _get_conn, _get_conn_async, @@ -47,15 +46,9 @@ } -class PostgreSQLOnlineStoreConfig(PostgreSQLConfig): +class PostgreSQLOnlineStoreConfig(PostgreSQLConfig, VectorStoreConfig): type: Literal["postgres"] = "postgres" - # Whether to enable the pgvector extension for vector similarity search - pgvector_enabled: Optional[bool] = False - - # If pgvector is enabled, the length of the vector field - vector_len: Optional[int] = 512 - class PostgreSQLOnlineStore(OnlineStore): _conn: Optional[Connection] = None @@ -120,7 +113,7 @@ def online_write_batch( for feature_name, val in values.items(): vector_val = None - if config.online_store.pgvector_enabled: + if config.online_store.vector_enabled: vector_val = get_list_val_str(val) insert_values.append( ( @@ -304,7 +297,7 @@ def update( for table in tables_to_keep: table_name = _table_id(project, table) - if config.online_store.pgvector_enabled: + if config.online_store.vector_enabled: vector_value_type = f"vector({config.online_store.vector_len})" else: # keep the vector_value_type as BYTEA if pgvector is not enabled, to maintain compatibility @@ -382,7 +375,7 @@ def retrieve_online_documents( """ project = config.project - if not config.online_store.pgvector_enabled: + if not config.online_store.vector_enabled: raise ValueError( "pgvector is not enabled in the online store configuration" ) diff --git a/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py b/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py index 3e921afceaa..d78289c8671 100644 --- a/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py +++ b/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from collections import defaultdict -from datetime import datetime, timezone +from datetime import datetime from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import singlestoredb @@ -11,6 +11,7 @@ from feast import Entity, FeatureView, RepoConfig from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.helpers import _to_naive_utc from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -225,10 +226,3 @@ def _drop_table_and_index(cur: Cursor, project: str, table: FeatureView) -> None def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" - - -def _to_naive_utc(ts: datetime) -> datetime: - if ts.tzinfo is None: - return ts - else: - return ts.astimezone(tz=timezone.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 0e2fdb35007..409a1eb2126 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -1,4 +1,5 @@ import struct +from datetime import datetime, timezone from typing import Any, List import mmh3 @@ -62,3 +63,10 @@ def compute_entity_id( entity_key_serialization_version=entity_key_serialization_version, ) ).hex() + + +def _to_naive_utc(ts: datetime) -> datetime: + if ts.tzinfo is None: + return ts + else: + return ts.astimezone(tz=timezone.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 061a766b8c8..1b79b1a94ba 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -29,6 +29,7 @@ from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject from feast.infra.key_encoding_utils import serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto @@ -38,7 +39,7 @@ from feast.utils import _build_retrieve_online_document_record, to_naive_utc -class SqliteOnlineStoreConfig(FeastConfigBaseModel): +class SqliteOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """Online store config for local (SQLite-based) store""" type: Literal["sqlite", "feast.infra.online_stores.sqlite.SqliteOnlineStore"] = ( @@ -49,12 +50,6 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ - vec_enabled: Optional[bool] = False - """ (optional) Enable or disable sqlite-vss for vector search""" - - vector_len: Optional[int] = 512 - """ (optional) Length of the vector to be stored in the database""" - class SqliteOnlineStore(OnlineStore): """ @@ -83,7 +78,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if sys.version_info[0:2] == (3, 10) and config.online_store.vec_enabled: + if sys.version_info[0:2] == (3, 10) and config.online_store.vector_enabled: import sqlite_vec # noqa: F401 self._conn.enable_load_extension(True) # type: ignore @@ -121,7 +116,7 @@ def online_write_batch( table_name = _table_id(project, table) for feature_name, val in values.items(): - if config.online_store.vec_enabled: + if config.online_store.vector_enabled: vector_bin = serialize_f32( val.float_list_val.val, config.online_store.vector_len ) # type: ignore @@ -321,7 +316,7 @@ def retrieve_online_documents( """ project = config.project - if not config.online_store.vec_enabled: + if not config.online_store.vector_enabled: raise ValueError("sqlite-vss is not enabled in the online store config") conn = self._get_conn(config) diff --git a/sdk/python/feast/infra/online_stores/vector_store.py b/sdk/python/feast/infra/online_stores/vector_store.py new file mode 100644 index 00000000000..051f9bcaedd --- /dev/null +++ b/sdk/python/feast/infra/online_stores/vector_store.py @@ -0,0 +1,16 @@ +from typing import Optional + + +class VectorStoreConfig: + # Whether to enable the online store for vector similarity search, + # This is only applicable for online store. + vector_enabled: Optional[bool] = False + + # If vector is enabled, the length of the vector field + vector_len: Optional[int] = 512 + + # The vector similarity metric to use in KNN search + # It is helpful for vector database that does not support config at retrieval runtime + # E.g. Elasticsearch dense_vector field at + # https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html + similarity: Optional[str] = "cosine" diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py b/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py index 622ee99e144..7ff72a48a37 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py @@ -67,7 +67,7 @@ def create_online_store(self) -> Dict[str, Any]: "user": "root", "password": "test!@#$%", "database": "test", - "pgvector_enabled": True, + "vector_enabled": True, "vector_len": 2, "port": self.container.get_exposed_port(5432), } diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 48adb6da7d9..83184643f35 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -441,7 +441,7 @@ def test_sqlite_get_online_documents() -> None: with runner.local_repo( get_example_repo("example_feature_repo_1.py"), "file" ) as store: - store.config.online_store.vec_enabled = True + store.config.online_store.vector_enabled = True store.config.online_store.vector_len = vector_length # Write some data to two tables document_embeddings_fv = store.get_feature_view(name="document_embeddings") @@ -548,7 +548,6 @@ def test_sqlite_vec_import() -> None: sqlite_version, vec_version = db.execute( "select sqlite_version(), vec_version()" ).fetchone() - assert vec_version == "v0.1.1" print(f"sqlite_version={sqlite_version}, vec_version={vec_version}") result = db.execute(""" diff --git a/sdk/python/tests/unit/online_store/test_online_writes.py b/sdk/python/tests/unit/online_store/test_online_writes.py index 0f7547a93b5..7927b564c67 100644 --- a/sdk/python/tests/unit/online_store/test_online_writes.py +++ b/sdk/python/tests/unit/online_store/test_online_writes.py @@ -75,6 +75,8 @@ def setUp(self): online=True, source=driver_stats_source, ) + # Before apply() join_keys is empty + assert driver_stats_fv.join_keys == [] @on_demand_feature_view( sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], @@ -100,6 +102,8 @@ def test_view(inputs: dict[str, Any]) -> dict[str, Any]: test_view, ] ) + # after apply() join_keys is [driver] + assert driver_stats_fv.join_keys == [driver.join_key] self.store.write_to_online_store( feature_view_name="driver_hourly_stats", df=driver_df ) diff --git a/ui/README.md b/ui/README.md index 852bddc2960..a2326e1a9ef 100644 --- a/ui/README.md +++ b/ui/README.md @@ -21,11 +21,14 @@ Start with bootstrapping a React app with `create-react-app` npx create-react-app your-feast-ui ``` -Then, in your app folder, install Feast UI and its peer dependencies. Assuming you use yarn +Then, in your app folder, install Feast UI and optionally its peer dependencies. Assuming you use yarn ``` yarn add @feast-dev/feast-ui -yarn add @elastic/eui @elastic/datemath @emotion/react moment prop-types inter-ui react-query react-router-dom use-query-params zod typescript query-string d3 @types/d3 +# For custom UI using the Elastic UI Framework (optional): +yarn add @elastic/eui +# For general custom styling (optional): +yarn add @emotion/react ``` Edit `index.js` in the React app to use Feast UI. diff --git a/ui/package.json b/ui/package.json index bc2a71378a1..9524944db5f 100644 --- a/ui/package.json +++ b/ui/package.json @@ -9,21 +9,18 @@ "types": "./dist/FeastUI.d.ts", "module": "./dist/feast-ui.module.js", "peerDependencies": { - "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.7.1", - "d3": "^7.3.0", - "inter-ui": "^3.19.3", - "moment": "^2.29.1", - "prop-types": "^15.8.1", - "query-string": "^7.1.1", "react": "^17.0.2", - "react-dom": "^17.0.2", - "react-query": "^3.34.12", - "react-router-dom": "<6.4.0", - "react-scripts": "^5.0.0", - "use-query-params": "^1.2.3", - "zod": "^3.11.6" + "react-dom": "^17.0.2" + }, + "peerDependenciesMeta": { + "@elastic/eui": { + "optional": true + }, + "@emotion/react": { + "optional": true + } }, "dependencies": { "@elastic/datemath": "^5.0.3", diff --git a/ui/yarn.lock b/ui/yarn.lock index 1f36143b67f..40a4d355eb8 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -9919,17 +9919,10 @@ rollup-pluginutils@^1.3.1: estree-walker "^0.2.1" minimatch "^3.0.2" -rollup@^2.43.1: - version "2.66.1" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.66.1.tgz#366b0404de353c4331d538c3ad2963934fcb4937" - integrity sha512-crSgLhSkLMnKr4s9iZ/1qJCplgAgrRY+igWv8KhG/AjKOJ0YX/WpmANyn8oxrw+zenF3BXWDLa7Xl/QZISH+7w== - optionalDependencies: - fsevents "~2.3.2" - -rollup@^2.68.0: - version "2.68.0" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.68.0.tgz#6ccabfd649447f8f21d62bf41662e5caece3bd66" - integrity sha512-XrMKOYK7oQcTio4wyTz466mucnd8LzkiZLozZ4Rz0zQD+HeX4nUK4B8GrTX/2EvN2/vBF/i2WnaXboPxo0JylA== +rollup@^2.43.1, rollup@^2.68.0: + version "2.79.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.2.tgz#f150e4a5db4b121a21a747d762f701e5e9f49090" + integrity sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ== optionalDependencies: fsevents "~2.3.2"