diff --git a/examples/kind-quickstart/01-Install.ipynb b/examples/kind-quickstart/01-Install.ipynb new file mode 100644 index 0000000000..e5ece97fc2 --- /dev/null +++ b/examples/kind-quickstart/01-Install.ipynb @@ -0,0 +1,932 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: feast==0.40.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (0.40.1)\n", + "Requirement already satisfied: click<9.0.0,>=7.0.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (8.1.7)\n", + "Requirement already satisfied: colorama<1,>=0.3.9 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (0.4.6)\n", + "Requirement already satisfied: dill~=0.3.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (0.3.8)\n", + "Requirement already satisfied: mypy-protobuf>=3.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (3.6.0)\n", + "Requirement already satisfied: Jinja2<4,>=2 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (3.1.4)\n", + "Requirement already satisfied: jsonschema in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (4.22.0)\n", + "Requirement already satisfied: mmh3 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (4.1.0)\n", + "Requirement already satisfied: numpy<2,>=1.22 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (1.26.4)\n", + "Requirement already satisfied: pandas<3,>=1.4.3 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (2.2.2)\n", + "Requirement already satisfied: protobuf<5.0.0,>=4.24.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (4.25.4)\n", + "Requirement already satisfied: pyarrow>=4 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (15.0.2)\n", + "Requirement already satisfied: pydantic>=2.0.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (2.7.4)\n", + "Requirement already satisfied: pygments<3,>=2.12.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (2.18.0)\n", + "Requirement already satisfied: PyYAML<7,>=5.4.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (6.0.1)\n", + "Requirement already satisfied: requests in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (2.32.3)\n", + "Requirement already satisfied: SQLAlchemy>1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from SQLAlchemy[mypy]>1->feast==0.40.1) (2.0.34)\n", + "Requirement already satisfied: tabulate<1,>=0.8.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (0.9.0)\n", + "Requirement already satisfied: tenacity<9,>=7 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (8.5.0)\n", + "Requirement already satisfied: toml<1,>=0.10.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (0.10.2)\n", + "Requirement already satisfied: tqdm<5,>=4 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (4.66.4)\n", + "Requirement already satisfied: typeguard>=4.0.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (4.3.0)\n", + "Requirement already satisfied: fastapi>=0.68.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (0.114.2)\n", + "Requirement already satisfied: uvicorn<1,>=0.14.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.30.6)\n", + "Requirement already satisfied: dask>=2024.2.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask[dataframe]>=2024.2.1->feast==0.40.1) (2024.6.2)\n", + "Requirement already satisfied: gunicorn in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from feast==0.40.1) (23.0.0)\n", + "Requirement already satisfied: cloudpickle>=1.5.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (3.0.0)\n", + "Requirement already satisfied: fsspec>=2021.09.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (2023.12.2)\n", + "Requirement already satisfied: packaging>=20.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (24.1)\n", + "Requirement already satisfied: partd>=1.2.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (1.4.2)\n", + "Requirement already satisfied: toolz>=0.10.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (0.12.1)\n", + "Requirement already satisfied: importlib-metadata>=4.13.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (8.0.0)\n", + "Requirement already satisfied: dask-expr<1.2,>=1.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from dask[dataframe]>=2024.2.1->feast==0.40.1) (1.1.6)\n", + "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (0.38.5)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (4.12.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from Jinja2<4,>=2->feast==0.40.1) (2.1.5)\n", + "Requirement already satisfied: types-protobuf>=4.24 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from mypy-protobuf>=3.1->feast==0.40.1) (5.27.0.20240626)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from pandas<3,>=1.4.3->feast==0.40.1) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from pandas<3,>=1.4.3->feast==0.40.1) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from pandas<3,>=1.4.3->feast==0.40.1) (2024.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from pydantic>=2.0.0->feast==0.40.1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.18.4 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from pydantic>=2.0.0->feast==0.40.1) (2.18.4)\n", + "Requirement already satisfied: mypy>=0.910 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from SQLAlchemy[mypy]>1->feast==0.40.1) (1.10.1)\n", + "Requirement already satisfied: h11>=0.8 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn<1,>=0.14.0->uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.14.0)\n", + "Requirement already satisfied: httptools>=0.5.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.6.1)\n", + "Requirement already satisfied: python-dotenv>=0.13 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (1.0.1)\n", + "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.19.0)\n", + "Requirement already satisfied: watchfiles>=0.13 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.22.0)\n", + "Requirement already satisfied: websockets>=10.4 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (12.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (23.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (0.18.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from requests->feast==0.40.1) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from requests->feast==0.40.1) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from requests->feast==0.40.1) (1.26.19)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from requests->feast==0.40.1) (2024.7.4)\n", + "Requirement already satisfied: zipp>=0.5 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from importlib-metadata>=4.13.0->dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (3.19.1)\n", + "Requirement already satisfied: mypy-extensions>=1.0.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from mypy>=0.910->SQLAlchemy[mypy]>1->feast==0.40.1) (1.0.0)\n", + "Requirement already satisfied: locket in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from partd>=1.2.0->dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (1.0.0)\n", + "Requirement already satisfied: six>=1.5 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas<3,>=1.4.3->feast==0.40.1) (1.16.0)\n", + "Requirement already satisfied: anyio<5,>=3.4.0 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from starlette<0.39.0,>=0.37.2->fastapi>=0.68.0->feast==0.40.1) (4.4.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /Users/dmartino/.pyenv/versions/3.11.9/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.39.0,>=0.37.2->fastapi>=0.68.0->feast==0.40.1) (1.3.1)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "# WE MUST ENSURE PYTHON CONSISTENCY BETWEEN NOTEBOOK AND FEAST SERVERS\n", + "# LAUNCH THIS NOTEBOOK FROM A CLEAN PYTHON ENVIRONMENT >3.9\n", + "%pip install feast==0.40.1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install Feast on Kind\n", + "## Objective\n", + "\n", + "Provide a reference implementation of a runbook to deploy a Feast development environment on a Kubernets cluster using [Kind](https://kind.sigs.k8s.io/docs/user/quick-start).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "* [Kind](https://kind.sigs.k8s.io/) cluster and a Docker runtime container\n", + "* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) Kubernetes CLI tool.\n", + "* [Helm](https://helm.sh/) Kubernetes package manager.\n", + "* [yq](https://github.com/mikefarah/yq) YAML processor." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Prerequisites\n", + "The following commands install and configure all the prerequisites on MacOS environment. You can find the\n", + "equivalent instructions on the offical documentation pages:\n", + "* Install Kind and Docker runtime (e.g. [Colima](https://github.com/abiosoft/colima)).\n", + "* Create Kind cluster named `feast`.\n", + "* Install and setup the `kubectl` context.\n", + "* `Helm`.\n", + "* `yq`.\n", + "```bash\n", + "brew install colima\n", + "colima start\n", + "brew install kind\n", + "kind create cluster --name feast\n", + "kind start\n", + "brew install helm\n", + "brew install kubectl\n", + "kubectl config use-context kind-feast\n", + "brew install yq\n", + "```\n", + "\n", + "Additionally, we create a `feast` namespace and use it as the default for the `kubectl` CLI:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast created\n", + "Context \"kind-feast\" modified.\n" + ] + } + ], + "source": [ + "!kubectl create ns feast\n", + "!kubectl config set-context --current --namespace feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Validate the cluster setup:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\n", + "default Active 26h\n", + "feast Active 3s\n", + "kube-node-lease Active 26h\n", + "kube-public Active 26h\n", + "kube-system Active 26h\n", + "local-path-storage Active 26h\n" + ] + } + ], + "source": [ + "!kubectl get ns" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deployment Architecture\n", + "The primary objective of this runbook is to guide the deployment of Feast services on a Kubernetes Kind cluster, using the default `postgres` template to set up a basic feature store.\n", + "\n", + "> 🚀 We will also add instructions to repeat the example with a custom project, for a personalized experience.\n", + "\n", + "In this notebook, we will deploy a distributed topology of Feast services, which includes:\n", + "\n", + "* `Registry Server`: Exposes endpoints at the [default port 6570](https://github.com/feast-dev/feast/blob/89bc5512572130510dd18690309b5a392aaf73b1/sdk/python/feast/constants.py#L39) and handles metadata storage for feature definitions.\n", + "* `Online Store Server`: Exposes endpoints at the [default port 6566](https://github.com/feast-dev/feast/blob/4a6b663f80bc91d6de35ed2ec428d34811d17a18/sdk/python/feast/cli.py#L871-L872). This service uses the `Registry Server` to query metadata and is responsible for low-latency serving of features.\n", + "* `Offline Store Server`: Exposes endpoints at the [default port 8815](https://github.com/feast-dev/feast/blob/89bc5512572130510dd18690309b5a392aaf73b1/sdk/python/feast/constants.py#L42). It uses the `Registry Server` to query metadata and provides access to batch data for historical feature retrieval.\n", + "\n", + "Each service is backed by a `PostgreSQL` database, which is also deployed within the same Kind cluster.\n", + "\n", + "Finally, port forwarding will be configured to expose these Feast services locally. This will allow a local client, implemented in the accompanying client notebook, to interact with the deployed services." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install PostgreSQL\n", + "Install the [reference deployment](./postgres/postgres.yaml) to install and configure a simple PostgreSQL database." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgres-secret created\n", + "persistentvolume/postgres-volume created\n", + "persistentvolumeclaim/postgres-volume-claim created\n", + "deployment.apps/postgres created\n", + "service/postgres created\n", + "deployment.apps/postgres condition met\n" + ] + } + ], + "source": [ + "!kubectl apply -f postgres/postgres.yaml\n", + "!kubectl wait --for=condition=available deployment/postgres --timeout=2m" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "postgres-76c8d94d6-pngvm 1/1 Running 0 8s\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "postgres NodePort 10.96.231.4 5432:30565/TCP 8s\n" + ] + } + ], + "source": [ + "!kubectl get pods\n", + "!kubectl get svc" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create the feature store project\n", + "Use the `feast init` command to create the default project.\n", + "\n", + "We also start port forwarding for the `postgres` service to populate the tables with default data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> 🚀 If you want to use a custom configuration, replace it under the sample/feature_repo folder and skip this section" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Port-forwarding postgres with process ID: 9611\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Forwarding from 127.0.0.1:5432 -> 5432\n", + "Forwarding from [::1]:5432 -> 5432\n" + ] + } + ], + "source": [ + "from src.utils import port_forward\n", + "psql_process = port_forward(\"postgres\", 5432, 5432)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are going to emulate the `feast init -t postgres sample` command using Python code. This is needed to mock the request of additional\n", + "parameters to configure the DB connection and also request the upload of example data to Postgres tables." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Handling connection for 5432\n", + "Handling connection for 5432\n", + "\n", + "Creating a new Feast repository in \u001b[1m\u001b[32m/Users/dmartino/projects/AI/feast/feast/examples/kind-quickstart/sample\u001b[0m.\n", + "\n" + ] + } + ], + "source": [ + "from feast.repo_operations import init_repo\n", + "from unittest import mock\n", + "from feast.templates.postgres.bootstrap import bootstrap\n", + "\n", + "project_directory = \"sample\"\n", + "template = \"postgres\"\n", + "\n", + "with mock.patch(\"click.prompt\", side_effect=[\"localhost\", \"5432\", \"feast\", \"public\", \"feast\", \"feast\"]):\n", + " with mock.patch(\"click.confirm\", side_effect=[True]):\n", + " init_repo(project_directory, template)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify that the DB includes the expected tables with pre-populated data." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " List of relations\n", + " Schema | Name | Type | Owner \n", + "--------+---------------------------+-------+-------\n", + " public | feast_driver_hourly_stats | table | feast\n", + "(1 row)\n", + "\n", + " count \n", + "-------\n", + " 1807\n", + "(1 row)\n", + "\n" + ] + } + ], + "source": [ + "!PSQL_POD=$(kubectl get pods -l app=postgres -oname) && kubectl exec $PSQL_POD -- psql -h localhost -U feast feast -c '\\dt'\n", + "!PSQL_POD=$(kubectl get pods -l app=postgres -oname) && kubectl exec $PSQL_POD -- psql -h localhost -U feast feast -c 'select count(*) from feast_driver_hourly_stats'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let's stop port forwarding." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 501 10392 6947 0 1:12PM ttys051 0:00.12 /bin/zsh -c ps -ef | grep port-forward\n", + " 501 10394 10392 0 1:12PM ttys051 0:00.00 grep port-forward\n" + ] + } + ], + "source": [ + "psql_process.terminate()\n", + "!ps -ef | grep port-forward" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate server configurations\n", + "Each server has its own configuration that we generate from the one initialized before.\n", + "\n", + "We use `yq` to manipulate the original configuration and generate the server specifics.\n", + "\n", + "Note: from now on, we assume that the Feast service names will be as follows:\n", + "* For `Registry Server`: `registry-server`\n", + "* For `Online Store`: `online-server`\n", + "* For `Offline Store`: `offline-server`\n", + "\n", + "> 🚀 If you used different service names, replace the `host` parameter in the following `yq` commands." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: FEATURE_REPO_DIR=sample/feature_repo\n", + "project: sample\n", + "provider: local\n", + "registry:\n", + " registry_type: sql\n", + " path: postgresql://feast:feast@postgres:5432/feast\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "online_store:\n", + " type: postgres\n", + " host: postgres\n", + " port: 5432\n", + " database: feast\n", + " db_schema: public\n", + " user: feast\n", + " password: feast\n", + "offline_store:\n", + " type: postgres\n", + " host: postgres\n", + " port: 5432\n", + " database: feast\n", + " db_schema: public\n", + " user: feast\n", + " password: feast\n", + "entity_key_serialization_version: 2\n" + ] + } + ], + "source": [ + "%env FEATURE_REPO_DIR=sample/feature_repo\n", + "# Adjust the database host to match the postgres service\n", + "!yq -i '.registry.path=\"postgresql://feast:feast@postgres:5432/feast\"' $FEATURE_REPO_DIR/feature_store.yaml\n", + "!yq -i '.online_store.host=\"postgres\"' $FEATURE_REPO_DIR/feature_store.yaml\n", + "!yq -i '.offline_store.host=\"postgres\"' $FEATURE_REPO_DIR/feature_store.yaml\n", + "!cat $FEATURE_REPO_DIR/feature_store.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "registry:\n", + " registry_type: sql\n", + " path: postgresql://feast:feast@postgres:5432/feast\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "provider: local\n", + "entity_key_serialization_version: 2\n" + ] + } + ], + "source": [ + "# Registry server has only `registry` section\n", + "!cat $FEATURE_REPO_DIR/feature_store.yaml | yq '.project | {key: .}, .registry | {key: .}, .provider | {key: .}, .entity_key_serialization_version | {key: .}' > registry_feature_store.yaml\n", + "! cat registry_feature_store.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "provider: local\n", + "online_store:\n", + " type: postgres\n", + " host: postgres\n", + " port: 5432\n", + " database: feast\n", + " db_schema: public\n", + " user: feast\n", + " password: feast\n", + "entity_key_serialization_version: 2\n", + "registry:\n", + " path: registry-server:80\n", + " registry_type: remote\n", + "offline_store:\n", + " type: remote\n", + " host: offline-server\n", + " port: 80\n" + ] + } + ], + "source": [ + "# Online server has `online_store` section, a remote `registry` and a remote `offline_store`\n", + "!cat $FEATURE_REPO_DIR/feature_store.yaml | yq '.project | {key: .}, .provider | {key: .}, .online_store | {key: .}, .entity_key_serialization_version | {key: .}' > online_feature_store.yaml\n", + "!yq -i '.registry.path=\"registry-server:80\"' online_feature_store.yaml\n", + "!yq -i '.registry.registry_type=\"remote\"' online_feature_store.yaml\n", + "!yq -i '.offline_store.type=\"remote\"' online_feature_store.yaml\n", + "!yq -i '.offline_store.host=\"offline-server\"' online_feature_store.yaml\n", + "!yq -i '.offline_store.port=80' online_feature_store.yaml\n", + "\n", + "!cat online_feature_store.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "provider: local\n", + "offline_store:\n", + " type: postgres\n", + " host: postgres\n", + " port: 5432\n", + " database: feast\n", + " db_schema: public\n", + " user: feast\n", + " password: feast\n", + "entity_key_serialization_version: 2\n", + "registry:\n", + " path: registry-server:80\n", + " registry_type: remote\n" + ] + } + ], + "source": [ + "# Offline server has `offline_store` section and a remote `registry`\n", + "!cat $FEATURE_REPO_DIR/feature_store.yaml | yq '.project | {key: .}, .provider | {key: .}, .offline_store | {key: .}, .entity_key_serialization_version | {key: .}' > offline_feature_store.yaml\n", + "!yq -i '.registry.path=\"registry-server:80\"' offline_feature_store.yaml\n", + "!yq -i '.registry.registry_type=\"remote\"' offline_feature_store.yaml\n", + "!cat offline_feature_store.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Encode configuration files\n", + "Next step is to encode in base64 the configuration files for each server. We'll store the output in environment variables." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "def base64_file(file):\n", + " import base64\n", + "\n", + " with open(file, 'rb') as file:\n", + " yaml_content = file.read()\n", + " return base64.b64encode(yaml_content).decode('utf-8')\n", + "\n", + "os.environ['REGISTRY_CONFIG_BASE64'] = base64_file('registry_feature_store.yaml')\n", + "os.environ['ONLINE_CONFIG_BASE64'] = base64_file('online_feature_store.yaml')\n", + "os.environ['OFFLINE_CONFIG_BASE64'] = base64_file('offline_feature_store.yaml')" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "REGISTRY_CONFIG_BASE64=cHJvamVjdDogc2FtcGxlCnJlZ2lzdHJ5OgogIHJlZ2lzdHJ5X3R5cGU6IHNxbAogIHBhdGg6IHBvc3RncmVzcWw6Ly9mZWFzdDpmZWFzdEBwb3N0Z3Jlczo1NDMyL2ZlYXN0CiAgY2FjaGVfdHRsX3NlY29uZHM6IDYwCiAgc3FsYWxjaGVteV9jb25maWdfa3dhcmdzOgogICAgZWNobzogZmFsc2UKICAgIHBvb2xfcHJlX3Bpbmc6IHRydWUKcHJvdmlkZXI6IGxvY2FsCmVudGl0eV9rZXlfc2VyaWFsaXphdGlvbl92ZXJzaW9uOiAyCg==\n", + "ONLINE_CONFIG_BASE64=cHJvamVjdDogc2FtcGxlCnByb3ZpZGVyOiBsb2NhbApvbmxpbmVfc3RvcmU6CiAgdHlwZTogcG9zdGdyZXMKICBob3N0OiBwb3N0Z3JlcwogIHBvcnQ6IDU0MzIKICBkYXRhYmFzZTogZmVhc3QKICBkYl9zY2hlbWE6IHB1YmxpYwogIHVzZXI6IGZlYXN0CiAgcGFzc3dvcmQ6IGZlYXN0CmVudGl0eV9rZXlfc2VyaWFsaXphdGlvbl92ZXJzaW9uOiAyCnJlZ2lzdHJ5OgogIHBhdGg6IHJlZ2lzdHJ5LXNlcnZlcjo4MAogIHJlZ2lzdHJ5X3R5cGU6IHJlbW90ZQpvZmZsaW5lX3N0b3JlOgogIHR5cGU6IHJlbW90ZQogIGhvc3Q6IG9mZmxpbmUtc2VydmVyCiAgcG9ydDogODAK\n", + "OFFLINE_CONFIG_BASE64=cHJvamVjdDogc2FtcGxlCnByb3ZpZGVyOiBsb2NhbApvZmZsaW5lX3N0b3JlOgogIHR5cGU6IHBvc3RncmVzCiAgaG9zdDogcG9zdGdyZXMKICBwb3J0OiA1NDMyCiAgZGF0YWJhc2U6IGZlYXN0CiAgZGJfc2NoZW1hOiBwdWJsaWMKICB1c2VyOiBmZWFzdAogIHBhc3N3b3JkOiBmZWFzdAplbnRpdHlfa2V5X3NlcmlhbGl6YXRpb25fdmVyc2lvbjogMgpyZWdpc3RyeToKICBwYXRoOiByZWdpc3RyeS1zZXJ2ZXI6ODAKICByZWdpc3RyeV90eXBlOiByZW1vdGUK\n" + ] + } + ], + "source": [ + "!env | grep BASE64" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install servers\n", + "We'll use the charts defined in this local repository to install the servers.\n", + "\n", + "The installation order reflects the dependency between the deployments:\n", + "* `Registry Server` starts first because it has no dependencies\n", + "* Then `Offline Server` as it depends only on the `Registry Server`\n", + "* Last the `Online Server` that depends on both the other servers" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: FEAST_IMAGE_REPO=feastdev/feature-server\n", + "env: FEAST_IMAGE_VERSION=0.40.1\n" + ] + } + ], + "source": [ + "%env FEAST_IMAGE_REPO=feastdev/feature-server\n", + "%env FEAST_IMAGE_VERSION=0.40.1" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"feast-registry\" does not exist. Installing it now.\n", + "NAME: feast-registry\n", + "LAST DEPLOYED: Tue Sep 17 13:14:05 2024\n", + "NAMESPACE: feast\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n", + "deployment.apps/registry-server condition met\n" + ] + } + ], + "source": [ + "# Registry\n", + "!helm upgrade --install feast-registry ../../infra/charts/feast-feature-server \\\n", + "--set fullnameOverride=registry-server --set feast_mode=registry \\\n", + "--set image.repository=${FEAST_IMAGE_REPO} --set image.tag=${FEAST_IMAGE_VERSION} \\\n", + "--set feature_store_yaml_base64=$REGISTRY_CONFIG_BASE64\n", + "\n", + "!kubectl wait --for=condition=available deployment/registry-server --timeout=2m" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"feast-offline\" does not exist. Installing it now.\n", + "NAME: feast-offline\n", + "LAST DEPLOYED: Tue Sep 17 13:14:33 2024\n", + "NAMESPACE: feast\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n", + "deployment.apps/offline-server condition met\n" + ] + } + ], + "source": [ + "# Offline\n", + "!helm upgrade --install feast-offline ../../infra/charts/feast-feature-server \\\n", + "--set fullnameOverride=offline-server --set feast_mode=offline \\\n", + "--set image.repository=${FEAST_IMAGE_REPO} --set image.tag=${FEAST_IMAGE_VERSION} \\\n", + "--set feature_store_yaml_base64=$OFFLINE_CONFIG_BASE64\n", + "\n", + "!kubectl wait --for=condition=available deployment/offline-server --timeout=2m" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"feast-online\" does not exist. Installing it now.\n", + "NAME: feast-online\n", + "LAST DEPLOYED: Tue Sep 17 13:14:55 2024\n", + "NAMESPACE: feast\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n", + "deployment.apps/online-server condition met\n" + ] + } + ], + "source": [ + "# Online\n", + "!helm upgrade --install feast-online ../../infra/charts/feast-feature-server \\\n", + "--set fullnameOverride=online-server --set feast_mode=online \\\n", + "--set image.repository=${FEAST_IMAGE_REPO} --set image.tag=${FEAST_IMAGE_VERSION} \\\n", + "--set feature_store_yaml_base64=$ONLINE_CONFIG_BASE64\n", + "\n", + "!kubectl wait --for=condition=available deployment/online-server --timeout=2m" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Validate deployment\n", + "Fist validate application and service status:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "offline-server ClusterIP 10.96.24.216 80/TCP 44s\n", + "online-server ClusterIP 10.96.36.113 80/TCP 22s\n", + "postgres NodePort 10.96.231.4 5432:30565/TCP 4m14s\n", + "registry-server ClusterIP 10.96.128.48 80/TCP 71s\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "offline-server 1/1 1 1 44s\n", + "online-server 1/1 1 1 22s\n", + "postgres 1/1 1 1 4m14s\n", + "registry-server 1/1 1 1 71s\n", + "NAME READY STATUS RESTARTS AGE\n", + "offline-server-6c59467c75-9jvq7 1/1 Running 0 45s\n", + "online-server-76968bbc48-qlvvj 1/1 Running 0 23s\n", + "postgres-76c8d94d6-pngvm 1/1 Running 0 4m15s\n", + "registry-server-597c5cd445-nrm75 1/1 Running 0 72s\n" + ] + } + ], + "source": [ + "!kubectl get svc\n", + "!kubectl get deployments\n", + "!kubectl get pods" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then verify the content of the local configuration file (it's stored in `/tmp/` folder with random subfolder)." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "registry:\n", + " registry_type: sql\n", + " path: postgresql://feast:feast@postgres:5432/feast\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "provider: local\n", + "entity_key_serialization_version: 2\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/registry-server -- find /tmp -name feature_store.yaml -exec cat {} \\;" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "provider: local\n", + "offline_store:\n", + " type: postgres\n", + " host: postgres\n", + " port: 5432\n", + " database: feast\n", + " db_schema: public\n", + " user: feast\n", + " password: feast\n", + "entity_key_serialization_version: 2\n", + "registry:\n", + " path: registry-server:80\n", + " registry_type: remote\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/offline-server -- find /tmp -name feature_store.yaml -exec cat {} \\;" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "provider: local\n", + "online_store:\n", + " type: postgres\n", + " host: postgres\n", + " port: 5432\n", + " database: feast\n", + " db_schema: public\n", + " user: feast\n", + " password: feast\n", + "entity_key_serialization_version: 2\n", + "registry:\n", + " path: registry-server:80\n", + " registry_type: remote\n", + "offline_store:\n", + " type: remote\n", + " host: offline-server\n", + " port: 80\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/online-server -- find /tmp -name feature_store.yaml -exec cat {} \\;" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let's verify the `feast` version in each server" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "Feast SDK Version: \"0.40.1\"\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "Feast SDK Version: \"0.40.1\"\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "Feast SDK Version: \"0.40.1\"\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/registry-server -- feast version\n", + "!kubectl exec deployment/offline-server -- feast version\n", + "!kubectl exec deployment/online-server -- feast version" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "feast3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/kind-quickstart/02-Client.ipynb b/examples/kind-quickstart/02-Client.ipynb new file mode 100644 index 0000000000..322a95a61b --- /dev/null +++ b/examples/kind-quickstart/02-Client.ipynb @@ -0,0 +1,606 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: feast==0.40.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (0.40.1)\n", + "Requirement already satisfied: click<9.0.0,>=7.0.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (8.1.7)\n", + "Requirement already satisfied: colorama<1,>=0.3.9 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (0.4.6)\n", + "Requirement already satisfied: dill~=0.3.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (0.3.8)\n", + "Requirement already satisfied: mypy-protobuf>=3.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (3.3.0)\n", + "Requirement already satisfied: Jinja2<4,>=2 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (3.1.4)\n", + "Requirement already satisfied: jsonschema in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (4.22.0)\n", + "Requirement already satisfied: mmh3 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (4.1.0)\n", + "Requirement already satisfied: numpy<2,>=1.22 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (1.26.4)\n", + "Requirement already satisfied: pandas<3,>=1.4.3 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (2.2.2)\n", + "Requirement already satisfied: protobuf<5.0.0,>=4.24.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (4.25.3)\n", + "Requirement already satisfied: pyarrow>=4 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (15.0.2)\n", + "Requirement already satisfied: pydantic>=2.0.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (2.7.4)\n", + "Requirement already satisfied: pygments<3,>=2.12.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (2.18.0)\n", + "Requirement already satisfied: PyYAML<7,>=5.4.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (6.0.1)\n", + "Requirement already satisfied: requests in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (2.32.3)\n", + "Requirement already satisfied: SQLAlchemy>1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from SQLAlchemy[mypy]>1->feast==0.40.1) (2.0.31)\n", + "Requirement already satisfied: tabulate<1,>=0.8.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (0.9.0)\n", + "Requirement already satisfied: tenacity<9,>=7 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (8.4.2)\n", + "Requirement already satisfied: toml<1,>=0.10.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (0.10.2)\n", + "Requirement already satisfied: tqdm<5,>=4 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (4.66.4)\n", + "Requirement already satisfied: typeguard>=4.0.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (4.3.0)\n", + "Requirement already satisfied: fastapi>=0.68.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (0.111.0)\n", + "Requirement already satisfied: uvicorn<1,>=0.14.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.30.1)\n", + "Requirement already satisfied: dask>=2024.2.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask[dataframe]>=2024.2.1->feast==0.40.1) (2024.6.2)\n", + "Requirement already satisfied: gunicorn in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from feast==0.40.1) (22.0.0)\n", + "Requirement already satisfied: cloudpickle>=1.5.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (3.0.0)\n", + "Requirement already satisfied: fsspec>=2021.09.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (2023.12.2)\n", + "Requirement already satisfied: packaging>=20.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (24.1)\n", + "Requirement already satisfied: partd>=1.2.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (1.4.2)\n", + "Requirement already satisfied: toolz>=0.10.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (0.12.1)\n", + "Requirement already satisfied: importlib-metadata>=4.13.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (8.0.0)\n", + "Requirement already satisfied: dask-expr<1.2,>=1.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from dask[dataframe]>=2024.2.1->feast==0.40.1) (1.1.6)\n", + "Requirement already satisfied: starlette<0.38.0,>=0.37.2 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (0.37.2)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (4.12.2)\n", + "Requirement already satisfied: fastapi-cli>=0.0.2 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (0.0.4)\n", + "Requirement already satisfied: httpx>=0.23.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (0.27.0)\n", + "Requirement already satisfied: python-multipart>=0.0.7 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (0.0.9)\n", + "Requirement already satisfied: ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (5.10.0)\n", + "Requirement already satisfied: orjson>=3.2.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (3.10.5)\n", + "Requirement already satisfied: email_validator>=2.0.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi>=0.68.0->feast==0.40.1) (2.2.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from Jinja2<4,>=2->feast==0.40.1) (2.1.5)\n", + "Requirement already satisfied: types-protobuf>=3.19.12 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from mypy-protobuf>=3.1->feast==0.40.1) (3.19.22)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from pandas<3,>=1.4.3->feast==0.40.1) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from pandas<3,>=1.4.3->feast==0.40.1) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from pandas<3,>=1.4.3->feast==0.40.1) (2024.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from pydantic>=2.0.0->feast==0.40.1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.18.4 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from pydantic>=2.0.0->feast==0.40.1) (2.18.4)\n", + "Requirement already satisfied: mypy>=0.910 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from SQLAlchemy[mypy]>1->feast==0.40.1) (1.10.1)\n", + "Requirement already satisfied: h11>=0.8 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn<1,>=0.14.0->uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.14.0)\n", + "Requirement already satisfied: httptools>=0.5.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.6.1)\n", + "Requirement already satisfied: python-dotenv>=0.13 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (1.0.1)\n", + "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.19.0)\n", + "Requirement already satisfied: watchfiles>=0.13 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (0.22.0)\n", + "Requirement already satisfied: websockets>=10.4 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from uvicorn[standard]<1,>=0.14.0->feast==0.40.1) (12.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (23.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from jsonschema->feast==0.40.1) (0.18.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from requests->feast==0.40.1) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from requests->feast==0.40.1) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from requests->feast==0.40.1) (1.26.19)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from requests->feast==0.40.1) (2024.7.4)\n", + "Requirement already satisfied: dnspython>=2.0.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from email_validator>=2.0.0->fastapi>=0.68.0->feast==0.40.1) (2.6.1)\n", + "Requirement already satisfied: typer>=0.12.3 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from fastapi-cli>=0.0.2->fastapi>=0.68.0->feast==0.40.1) (0.12.3)\n", + "Requirement already satisfied: anyio in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from httpx>=0.23.0->fastapi>=0.68.0->feast==0.40.1) (4.4.0)\n", + "Requirement already satisfied: httpcore==1.* in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from httpx>=0.23.0->fastapi>=0.68.0->feast==0.40.1) (1.0.5)\n", + "Requirement already satisfied: sniffio in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from httpx>=0.23.0->fastapi>=0.68.0->feast==0.40.1) (1.3.1)\n", + "Requirement already satisfied: zipp>=0.5 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from importlib-metadata>=4.13.0->dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (3.19.1)\n", + "Requirement already satisfied: mypy-extensions>=1.0.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from mypy>=0.910->SQLAlchemy[mypy]>1->feast==0.40.1) (1.0.0)\n", + "Requirement already satisfied: locket in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from partd>=1.2.0->dask>=2024.2.1->dask[dataframe]>=2024.2.1->feast==0.40.1) (1.0.0)\n", + "Requirement already satisfied: six>=1.5 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas<3,>=1.4.3->feast==0.40.1) (1.16.0)\n", + "Requirement already satisfied: shellingham>=1.3.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from typer>=0.12.3->fastapi-cli>=0.0.2->fastapi>=0.68.0->feast==0.40.1) (1.5.4)\n", + "Requirement already satisfied: rich>=10.11.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from typer>=0.12.3->fastapi-cli>=0.0.2->fastapi>=0.68.0->feast==0.40.1) (13.7.1)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from rich>=10.11.0->typer>=0.12.3->fastapi-cli>=0.0.2->fastapi>=0.68.0->feast==0.40.1) (3.0.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.12.3->fastapi-cli>=0.0.2->fastapi>=0.68.0->feast==0.40.1) (0.1.2)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.1.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "# WE MUST ENSURE PYTHON CONSISTENCY BETWEEN NOTEBOOK AND FEAST SERVERS\n", + "# LAUNCH THIS NOTEBOOK FROM A CLEAN PYTHON ENVIRONMENT >3.9\n", + "%pip install feast==0.40.1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run a test client\n", + "\n", + "> 🚀 This test is developer to work only with the default feature store generated by `feast init`. \n", + "> \n", + "> To test a custom feature store you need to run a custom test application, but still using the same client configuration that we've prepared." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Apply the feature store definitions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The feature store cannot be initialized using remote services. \n", + "\n", + "We'll use the original `feature_store.yaml` from within a Kubernetes `Job` to run `feast apply`.\n", + "\n", + "For the same reason, we also run an initial materialization from the `Job`, otherwise it would fail because of uninmplemented APIs in the remote servers, like [online_write_batch](https://github.com/feast-dev/feast/blob/4a6b663f80bc91d6de35ed2ec428d34811d17a18/sdk/python/feast/infra/online_stores/remote.py#L50).\n", + "\n", + "First we create a `ConfigMap` holding the required code and configuration." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: FEATURE_REPO_DIR=sample/feature_repo\n", + "Error from server (NotFound): configmaps \"sample-repo\" not found\n", + "configmap/sample-repo created\n", + "\n", + "Inspect keys of sample-repo ConfigMap\n", + "example_repo.py\n", + "feature_store.yaml\n" + ] + } + ], + "source": [ + "%env FEATURE_REPO_DIR=sample/feature_repo\n", + "!kubectl delete configmap sample-repo\n", + "!kubectl create configmap sample-repo --from-file=${FEATURE_REPO_DIR}/example_repo.py,${FEATURE_REPO_DIR}/feature_store.yaml\n", + "!echo\n", + "!echo \"Inspect keys of sample-repo ConfigMap\"\n", + "!kubectl get configmaps sample-repo -oyaml | yq '.data[] | key'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we create the `Job` to apply the definitions, according to the [init-job.yaml](./init-job.yaml) manifest" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (NotFound): error when deleting \"init-job.yaml\": jobs.batch \"feast-apply-job\" not found\n", + "job.batch/feast-apply-job created\n" + ] + } + ], + "source": [ + "!kubectl delete -f init-job.yaml\n", + "!kubectl apply -f init-job.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Monitoring the log of the `Job`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "pod/feast-apply-job-tzscd condition met\n", + "Starting feast initialization job...\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "09/17/2024 11:18:10 AM feast.repo_config WARNING: The `path` of the `RegistryConfig` starts with a plain `postgresql` string. We are updating this to `postgresql+psycopg` to ensure that the `psycopg3` driver is used by `sqlalchemy`. If you want to use `psycopg2` pass `postgresql+psycopg2` explicitely to `path`. To silence this warning, pass `postgresql+psycopg` explicitely to `path`.\n", + "/usr/local/lib/python3.11/site-packages/feast/feature_store.py:590: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", + " warnings.warn(\n", + "Deploying infrastructure for driver_hourly_stats_fresh\n", + "Deploying infrastructure for driver_hourly_stats\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "09/17/2024 11:18:21 AM feast.repo_config WARNING: The `path` of the `RegistryConfig` starts with a plain `postgresql` string. We are updating this to `postgresql+psycopg` to ensure that the `psycopg3` driver is used by `sqlalchemy`. If you want to use `psycopg2` pass `postgresql+psycopg2` explicitely to `path`. To silence this warning, pass `postgresql+psycopg` explicitely to `path`.\n", + "09/17/2024 11:18:21 AM root WARNING: _list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2024-09-17 11:18:11+00:00\u001b[0m into the \u001b[1m\u001b[32mpostgres\u001b[0m online store.\n", + "\n", + "\u001b[1m\u001b[32mdriver_hourly_stats_fresh\u001b[0m from \u001b[1m\u001b[32m2024-09-16 11:18:21+00:00\u001b[0m to \u001b[1m\u001b[32m2024-09-17 11:18:11+00:00\u001b[0m:\n", + "100%|█████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 72.23it/s]\n", + "\u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m from \u001b[1m\u001b[32m2024-09-16 11:18:22+00:00\u001b[0m to \u001b[1m\u001b[32m2024-09-17 11:18:11+00:00\u001b[0m:\n", + "100%|████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 654.75it/s]\n", + "Feast initialization completed successfully.\n" + ] + } + ], + "source": [ + "!INIT_JOB_POD=$(kubectl get pods -l job-name=feast-apply-job -oname) && kubectl wait --for=condition=podscheduled $INIT_JOB_POD --timeout=2m\n", + "!INIT_JOB_POD=$(kubectl get pods -l job-name=feast-apply-job -oname) && kubectl logs -f $INIT_JOB_POD\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Forwarding the feast service ports" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To run the test client from the notebook, we need to forward the service ports to ports on the current host." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "offline-server ClusterIP 10.96.24.216 80/TCP 3m58s\n", + "online-server ClusterIP 10.96.36.113 80/TCP 3m36s\n", + "postgres NodePort 10.96.231.4 5432:30565/TCP 7m28s\n", + "registry-server ClusterIP 10.96.128.48 80/TCP 4m25s\n" + ] + } + ], + "source": [ + "!kubectl get svc" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Port-forwarding registry-server with process ID: 15094\n", + "Port-forwarding offline-server with process ID: 15095\n", + "Port-forwarding online-server with process ID: 15096\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Forwarding from 127.0.0.1:8002 -> 8815\n", + "Forwarding from 127.0.0.1:8003 -> 6566\n", + "Forwarding from 127.0.0.1:8001 -> 6570\n", + "Forwarding from [::1]:8002 -> 8815\n", + "Forwarding from [::1]:8003 -> 6566\n", + "Forwarding from [::1]:8001 -> 6570\n" + ] + } + ], + "source": [ + "from src.utils import port_forward\n", + "registry_process = port_forward(\"registry-server\", 8001)\n", + "offline_process = port_forward(\"offline-server\", 8002)\n", + "online_process = port_forward(\"online-server\", 8003)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 501 15094 13456 0 1:18PM ?? 0:00.06 kubectl port-forward service/registry-server 8001:80\n", + " 501 15095 13456 0 1:18PM ?? 0:00.05 kubectl port-forward service/offline-server 8002:80\n", + " 501 15096 13456 0 1:18PM ?? 0:00.06 kubectl port-forward service/online-server 8003:80\n", + " 501 15170 13456 0 1:18PM ttys051 0:00.14 /bin/zsh -c ps -ef | grep port-forward\n", + " 501 15173 15170 0 1:18PM ttys051 0:00.00 grep port-forward\n" + ] + } + ], + "source": [ + "!ps -ef | grep port-forward" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Client configuration\n", + "The client configuration is using only remote clients connected to the forwarded ports, from 8001 to 8003." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "registry:\n", + " path: localhost:8001\n", + " registry_type: remote\n", + "offline_store:\n", + " host: localhost\n", + " port: 8002\n", + " type: remote\n", + "online_store:\n", + " path: http://localhost:8003\n", + " type: remote\n", + "entity_key_serialization_version: 2\n", + "auth:\n", + " type: no_auth\n" + ] + } + ], + "source": [ + "!cat client/feature_store.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install test code\n", + "First we copy the test code from `sample/feature_repo` to `client` folder." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "client/__init__.py client/test_workflow.py\n" + ] + } + ], + "source": [ + "!cp sample/feature_repo/test_workflow.py client\n", + "!ls client/*.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We update the original test to comment the `apply`, `teardown` and `materialize-incremental` commands." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12,13c12,13\n", + "< # print(\"\\n--- Run feast apply to setup feature store on Postgres ---\")\n", + "< # subprocess.run([\"feast\", \"apply\"])\n", + "---\n", + "> print(\"\\n--- Run feast apply to setup feature store on Postgres ---\")\n", + "> subprocess.run([\"feast\", \"apply\"])\n", + "21,22c21,22\n", + "< # print(\"\\n--- Load features into online store ---\")\n", + "< # store.materialize_incremental(end_date=datetime.now())\n", + "---\n", + "> print(\"\\n--- Load features into online store ---\")\n", + "> store.materialize_incremental(end_date=datetime.now())\n", + "56,57c56,57\n", + "< # print(\"\\n--- Run feast teardown ---\")\n", + "< # subprocess.run([\"feast\", \"teardown\"])\n", + "---\n", + "> print(\"\\n--- Run feast teardown ---\")\n", + "> subprocess.run([\"feast\", \"teardown\"])\n" + ] + } + ], + "source": [ + "!sed -i.bk 's/subprocess.run/# subprocess.run/' client/test_workflow.py\n", + "!sed -i.bk 's/print(\"\\\\n--- Run feast/# print(\"\\\\n--- Run feast/' client/test_workflow.py\n", + "!sed -i.bk 's/store.materialize_incremental/# store.materialize_incremental/' client/test_workflow.py\n", + "!sed -i.bk 's/print(\"\\\\n--- Load features/# print(\"\\\\n--- Load features/' client/test_workflow.py\n", + "!diff client/test_workflow.py sample/feature_repo/test_workflow.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we run the full test suite from the client folder." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Handling connection for 8001\n", + "\n", + "--- Historical features for training ---\n", + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "Handling connection for 8002\n", + " driver_id event_timestamp ... conv_rate_plus_val1 conv_rate_plus_val2\n", + "0 1001 2021-04-12 10:59:42 ... 1.302426 10.302426\n", + "1 1002 2021-04-12 08:12:10 ... 2.436384 20.436384\n", + "2 1003 2021-04-12 16:40:26 ... 3.954102 30.954102\n", + "\n", + "[3 rows x 10 columns]\n", + "\n", + "--- Historical features for batch scoring ---\n", + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "Handling connection for 8002\n", + " driver_id ... conv_rate_plus_val2\n", + "0 1001 ... 10.798974\n", + "1 1002 ... 20.316096\n", + "2 1003 ... 30.202964\n", + "\n", + "[3 rows x 10 columns]\n", + "\n", + "--- Online features ---\n", + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "Handling connection for 8003\n", + "acc_rate : [0.22748562693595886, 0.9316393733024597]\n", + "conv_rate_plus_val1 : [1000.7989742159843, 1001.3160955905914]Handling connection for 8003\n", + "\n", + "conv_rate_plus_val2 : [2000.7989742159843, 2002.3160955905914]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Online features retrieved (instead) through a feature service---\n", + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "Handling connection for 8003\n", + "conv_rate : [0.7989742159843445, 0.31609559059143066]\n", + "conv_rate_plus_val1 : [1000.7989742159843, 1001.3160955905914]\n", + "conv_rate_plus_val2 : [2000.7989742159843, 2002.3160955905914]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Online features retrieved (using feature service v3, which uses a feature view with a push source---\n", + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "acc_rate : [0.22748562693595886, 0.9316393733024597]\n", + "avg_daily_trips : [451, 417]\n", + "conv_rate : [0.7989742159843445, 0.31609559059143066]\n", + "conv_rate_plus_val1 : [1000.7989742159843, 1001.3160955905914]\n", + "conv_rate_plus_val2 : [2000.7989742159843, 2002.3160955905914]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Simulate a stream event ingestion of the hourly stats df ---\n", + " driver_id event_timestamp ... acc_rate avg_daily_trips\n", + "0 1001 2024-09-17 13:19:54.105733 ... 1.0 1000\n", + "\n", + "[1 rows x 6 columns]\n", + "WARNING:root:list_feature_views will make breaking changes. Please use list_batch_feature_views instead. list_feature_views will behave like list_all_feature_views in the future.\n", + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "Traceback (most recent call last):\n", + " File \"/Users/dmartino/projects/AI/feast/feast/examples/kind-quickstart/client/test_workflow.py\", line 130, in \n", + " run_demo()\n", + " File \"/Users/dmartino/projects/AI/feast/feast/examples/kind-quickstart/client/test_workflow.py\", line 51, in run_demo\n", + " store.push(\"driver_stats_push_source\", event_df, to=PushMode.ONLINE_AND_OFFLINE)\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/feast/feature_store.py\", line 1423, in push\n", + " self.write_to_online_store(\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/feast/feature_store.py\", line 1449, in write_to_online_store\n", + " feature_view: FeatureView = self.get_stream_feature_view(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/feast/feature_store.py\", line 504, in get_stream_feature_view\n", + " return self._get_stream_feature_view(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/feast/feature_store.py\", line 514, in _get_stream_feature_view\n", + " stream_feature_view = self._registry.get_stream_feature_view(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/feast/infra/registry/remote.py\", line 209, in get_stream_feature_view\n", + " response = self.stub.GetStreamFeatureView(request)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/grpc/_channel.py\", line 1181, in __call__\n", + " return _end_unary_response_blocking(state, call, False, None)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/dmartino/miniconda3/envs/feast3.11/lib/python3.11/site-packages/grpc/_channel.py\", line 1006, in _end_unary_response_blocking\n", + " raise _InactiveRpcError(state) # pytype: disable=not-instantiable\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:\n", + "\tstatus = StatusCode.UNKNOWN\n", + "\tdetails = \"Exception calling application: Feature view driver_hourly_stats_fresh does not exist in project sample\"\n", + "\tdebug_error_string = \"UNKNOWN:Error received from peer {grpc_message:\"Exception calling application: Feature view driver_hourly_stats_fresh does not exist in project sample\", grpc_status:2, created_time:\"2024-09-17T13:19:54.127834+02:00\"}\"\n", + ">\n" + ] + } + ], + "source": [ + "!cd client && python test_workflow.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note If you see the following error, it is likely due to the [issue #4392](https://github.com/feast-dev/feast/issues/4392):\n", + "Remote registry client does not map application errors:\n", + "\n", + "```\n", + "Feature view driver_hourly_stats_fresh does not exist in project sample\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Terminate port forwarding" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 501 16434 13456 0 1:20PM ttys051 0:00.12 /bin/zsh -c ps -ef | grep port-forward\n", + " 501 16436 16434 0 1:20PM ttys051 0:00.00 grep port-forward\n" + ] + } + ], + "source": [ + "registry_process.terminate()\n", + "offline_process.terminate()\n", + "online_process.terminate()\n", + "!ps -ef | grep port-forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "feast3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/kind-quickstart/03-Uninstall.ipynb b/examples/kind-quickstart/03-Uninstall.ipynb new file mode 100644 index 0000000000..20874fc1b7 --- /dev/null +++ b/examples/kind-quickstart/03-Uninstall.ipynb @@ -0,0 +1,120 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Uninstall deployment\n", + "Use Helm to uninstall all the previous deployments" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"feast-online\" uninstalled\n", + "release \"feast-offline\" uninstalled\n", + "release \"feast-registry\" uninstalled\n", + "NAME\tNAMESPACE\tREVISION\tUPDATED\tSTATUS\tCHART\tAPP VERSION\n" + ] + } + ], + "source": [ + "!helm uninstall feast-online\n", + "!helm uninstall feast-offline\n", + "!helm uninstall feast-registry\n", + "!helm list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Delete the PostgreSQL deployment." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret \"postgres-secret\" deleted\n", + "persistentvolume \"postgres-volume\" deleted\n", + "persistentvolumeclaim \"postgres-volume-claim\" deleted\n", + "deployment.apps \"postgres\" deleted\n", + "service \"postgres\" deleted\n" + ] + } + ], + "source": [ + "!kubectl delete -f postgres/postgres.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No resources found in feast namespace.\n", + "No resources found in feast namespace.\n", + "NAME READY STATUS RESTARTS AGE\n", + "feast-apply-job-tzscd 0/1 Completed 0 2m40s\n" + ] + } + ], + "source": [ + "!kubectl get svc\n", + "!kubectl get deployments\n", + "!kubectl get pods" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "feast3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/kind-quickstart/README.md b/examples/kind-quickstart/README.md new file mode 100644 index 0000000000..25ecfc8ecf --- /dev/null +++ b/examples/kind-quickstart/README.md @@ -0,0 +1,7 @@ +# Install and run Feast with Kind + +The following notebooks will guide you through an end-to-end journey to install and validate a simple Feast feature store in a +Kind Kubernetes cluster: +* [01-Install.ipynb](./01-Install.ipynb): Install and configure the cluster, then the Feast components. +* [02-Client.ipynb](./02-Client.ipynb): Validate the feature store with a remote test application runnning on the notebook. +* [03-Uninstall.ipynb](./03-Uninstall.ipynb): Clear the installed deployments. diff --git a/examples/kind-quickstart/client/__init__.py b/examples/kind-quickstart/client/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/kind-quickstart/client/feature_store.yaml b/examples/kind-quickstart/client/feature_store.yaml new file mode 100644 index 0000000000..62acd3ead6 --- /dev/null +++ b/examples/kind-quickstart/client/feature_store.yaml @@ -0,0 +1,14 @@ +project: sample +registry: + path: localhost:8001 + registry_type: remote +offline_store: + host: localhost + port: 8002 + type: remote +online_store: + path: http://localhost:8003 + type: remote +entity_key_serialization_version: 2 +auth: + type: no_auth diff --git a/examples/kind-quickstart/init-job.yaml b/examples/kind-quickstart/init-job.yaml new file mode 100644 index 0000000000..68df35af73 --- /dev/null +++ b/examples/kind-quickstart/init-job.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: feast-apply-job +spec: + template: + spec: + containers: + - name: feast-apply + image: feastdev/feature-server:0.40.1 + command: ["/bin/sh", "-c"] + args: + - | + echo "Starting feast initialization job..."; + mkdir /tmp/sample; + cd /tmp/sample; + cp /sample/* .; + sed -i 's/localhost/postgres/' feature_store.yaml; + feast apply; + CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S"); + feast materialize-incremental $CURRENT_TIME; + echo "Feast initialization completed successfully."; + volumeMounts: + - name: sample-repo-files + mountPath: /sample + restartPolicy: Never + volumes: + - name: sample-repo-files + configMap: + name: sample-repo + backoffLimit: 1 diff --git a/examples/kind-quickstart/postgres/postgres.yaml b/examples/kind-quickstart/postgres/postgres.yaml new file mode 100644 index 0000000000..c89a01f0f4 --- /dev/null +++ b/examples/kind-quickstart/postgres/postgres.yaml @@ -0,0 +1,83 @@ +#https://www.digitalocean.com/community/tutorials/how-to-deploy-postgres-to-kubernetes-cluster +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + labels: + app: postgres +stringData: + POSTGRES_DB: feast + POSTGRES_USER: feast + POSTGRES_PASSWORD: feast +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: postgres-volume + labels: + type: local + app: postgres +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /data/postgresql +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-volume-claim + labels: + app: postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: 'postgres:15-alpine' + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5432 + envFrom: + - secretRef: + name: postgres-secret + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgresdata + volumes: + - name: postgresdata + persistentVolumeClaim: + claimName: postgres-volume-claim +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + type: NodePort + ports: + - port: 5432 + selector: + app: postgres \ No newline at end of file diff --git a/examples/kind-quickstart/src/__init__.py b/examples/kind-quickstart/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/kind-quickstart/src/utils.py b/examples/kind-quickstart/src/utils.py new file mode 100644 index 0000000000..ea549d7ed8 --- /dev/null +++ b/examples/kind-quickstart/src/utils.py @@ -0,0 +1,12 @@ +import subprocess + +def port_forward(service, external_port, local_port=80) : + """ + Run a background process to forward port 80 of the given `service` service to the given `external_port` port. + + Returns: the process instance + """ + command = ["kubectl", "port-forward", f"service/{service}", f"{external_port}:{local_port}"] + process = subprocess.Popen(command) + print(f"Port-forwarding {service} with process ID: {process.pid}") + return process