feat: Adding SSL support for online server (feast-dev#4677)

* * Adding the SSL support for online_server.
* Adding the SSL support for remote online client.
* Adding the integration test to run the remote online server in SSL and non SSL mode.
* Incorporated code review comments

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>

incorporating code review comments.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>

* Incorporating code review comment.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>

incorporating code review comments.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>

* Update docs/reference/feature-servers/python-feature-server.md

Co-authored-by: Francisco Arceo <farceo@redhat.com>
Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>

* * Update docs/reference/feature-servers/python-feature-server.md
* fixing the integration test failure.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>

---------

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Co-authored-by: Francisco Arceo <farceo@redhat.com>

docs/reference/feature-servers/python-feature-server.md

@@ -200,6 +200,28 @@ requests.post(
     data=json.dumps(push_data))
 ```
 
+## Starting the feature server in SSL mode
+
+Enabling SSL mode ensures that data between the Feast client and server is transmitted securely. For an ideal production environment, it is recommended to start the feature server in SSL mode.
+
+### Obtaining a self-signed SSL certificate and key
+In development mode we can generate a self-signed certificate for testing. In an actual production environment it is always recommended to get it from a trusted SSL certificate provider.
+
+```shell
+openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -days 365 -nodes
+```
+
+The above command will generate two files
+* `key.pem` : certificate private key
+* `cert.pem`: certificate public key
+
+### Starting the Online Server in SSL Mode
+To start the feature server in SSL mode, you need to provide the private and public keys using the `--ssl-key-path` and `--ssl-cert-path` arguments with the `feast serve` command.
+
+```shell
+feast serve --ssl-key-path key.pem --ssl-cert-path cert.pem
+```
+
 # Online Feature Server Permissions and Access Control
 
 ## API Endpoints and Permissions

docs/reference/online-stores/remote.md

@@ -16,12 +16,15 @@ provider: local
 online_store:
   path: http://localhost:6566
   type: remote
+  ssl_cert_path: /path/to/cert.pem
 entity_key_serialization_version: 2
 auth:
   type: no_auth
 ```
 {% endcode %}
 
+`ssl_cert_path` is an optional configuration to the public certificate path when the online server starts in SSL mode. This may be needed if the online server is started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`.
+
 ## How to configure Authentication and Authorization
 Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization.
 

sdk/python/feast/cli.py

@@ -911,6 +911,22 @@ def init_command(project_directory, minimal: bool, template: str):
     default=5,
     show_default=True,
 )
+@click.option(
+    "--ssl-key-path",
+    "-k",
+    type=click.STRING,
+    default="",
+    show_default=False,
+    help="path to SSL certificate private key. You need to pass ssl-cert-path as well to start server in SSL mode",
+)
+@click.option(
+    "--ssl-cert-path",
+    "-c",
+    type=click.STRING,
+    default="",
+    show_default=False,
+    help="path to SSL certificate public key. You need to pass ssl-key-path as well to start server in SSL mode",
+)
 @click.option(
     "--metrics",
     "-m",
@@ -928,9 +944,16 @@ def serve_command(
     workers: int,
     metrics: bool,
     keep_alive_timeout: int,
+    ssl_key_path: str,
+    ssl_cert_path: str,
     registry_ttl_sec: int = 5,
 ):
     """Start a feature server locally on a given port."""
+    if (ssl_key_path and not ssl_cert_path) or (not ssl_key_path and ssl_cert_path):
+        raise click.BadParameter(
+            "Please configure ssl-cert-path and ssl-key-path args to start the feature server in SSL mode."
+        )
+
     store = create_feature_store(ctx)
 
     store.serve(
@@ -941,6 +964,8 @@ def serve_command(
         workers=workers,
         metrics=metrics,
         keep_alive_timeout=keep_alive_timeout,
+        ssl_key_path=ssl_key_path,
+        ssl_cert_path=ssl_cert_path,
         registry_ttl_sec=registry_ttl_sec,
     )
 

sdk/python/feast/feature_server.py

@@ -339,6 +339,8 @@ def start_server(
     workers: int,
     keep_alive_timeout: int,
     registry_ttl_sec: int,
+    ssl_key_path: str,
+    ssl_cert_path: str,
     metrics: bool,
 ):
     if metrics:
@@ -364,16 +366,31 @@ def start_server(
     logger.debug("Auth manager initialized successfully")
 
     if sys.platform != "win32":
-        FeastServeApplication(
-            store=store,
-            bind=f"{host}:{port}",
-            accesslog=None if no_access_log else "-",
-            workers=workers,
-            keepalive=keep_alive_timeout,
-            registry_ttl_sec=registry_ttl_sec,
-        ).run()
+        options = {
+            "bind": f"{host}:{port}",
+            "accesslog": None if no_access_log else "-",
+            "workers": workers,
+            "keepalive": keep_alive_timeout,
+            "registry_ttl_sec": registry_ttl_sec,
+        }
+
+        # Add SSL options if the paths exist
+        if ssl_key_path and ssl_cert_path:
+            options["keyfile"] = ssl_key_path
+            options["certfile"] = ssl_cert_path
+        FeastServeApplication(store=store, **options).run()
     else:
         import uvicorn
 
         app = get_app(store, registry_ttl_sec)
-        uvicorn.run(app, host=host, port=port, access_log=(not no_access_log))
+        if ssl_key_path and ssl_cert_path:
+            uvicorn.run(
+                app,
+                host=host,
+                port=port,
+                access_log=(not no_access_log),
+                ssl_keyfile=ssl_key_path,
+                ssl_certfile=ssl_cert_path,
+            )
+        else:
+            uvicorn.run(app, host=host, port=port, access_log=(not no_access_log))

sdk/python/feast/feature_store.py

@@ -1896,6 +1896,8 @@ def serve(
         workers: int = 1,
         metrics: bool = False,
         keep_alive_timeout: int = 30,
+        ssl_key_path: str = "",
+        ssl_cert_path: str = "",
         registry_ttl_sec: int = 2,
     ) -> None:
         """Start the feature consumption server locally on a given port."""
@@ -1913,6 +1915,8 @@ def serve(
             workers=workers,
             metrics=metrics,
             keep_alive_timeout=keep_alive_timeout,
+            ssl_key_path=ssl_key_path,
+            ssl_cert_path=ssl_cert_path,
             registry_ttl_sec=registry_ttl_sec,
         )
 

sdk/python/feast/infra/online_stores/remote.py

@@ -41,6 +41,10 @@ class RemoteOnlineStoreConfig(FeastConfigBaseModel):
     """ str: Path to metadata store.
     If type is 'remote', then this is a URL for registry server """
 
+    ssl_cert_path: StrictStr = ""
+    """ str: Path to the public certificate when the online server starts in SSL mode. This may be needed if the online server started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`.
+    If type is 'remote', then this configuration is needed to connect to remote online server in SSL mode. """
+
 
 class RemoteOnlineStore(OnlineStore):
     """
@@ -170,6 +174,13 @@ def teardown(
 def get_remote_online_features(
     session: requests.Session, config: RepoConfig, req_body: str
 ) -> requests.Response:
-    return session.post(
-        f"{config.online_store.path}/get-online-features", data=req_body
-    )
+    if config.online_store.ssl_cert_path:
+        return session.post(
+            f"{config.online_store.path}/get-online-features",
+            data=req_body,
+            verify=config.online_store.ssl_cert_path,
+        )
+    else:
+        return session.post(
+            f"{config.online_store.path}/get-online-features", data=req_body
+        )

sdk/python/tests/integration/online_store/test_remote_online_store.py

@@ -15,11 +15,13 @@
     start_feature_server,
 )
 from tests.utils.cli_repo_creator import CliRunner
+from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert
 from tests.utils.http_server import free_port
 
 
+@pytest.mark.parametrize("ssl_mode", [True, False])
 @pytest.mark.integration
-def test_remote_online_store_read(auth_config):
+def test_remote_online_store_read(auth_config, ssl_mode):
     with tempfile.TemporaryDirectory() as remote_server_tmp_dir, tempfile.TemporaryDirectory() as remote_client_tmp_dir:
         permissions_list = [
             Permission(
@@ -41,11 +43,12 @@ def test_remote_online_store_read(auth_config):
                 actions=[AuthzedAction.READ_ONLINE],
             ),
         ]
-        server_store, server_url, registry_path = (
+        server_store, server_url, registry_path, ssl_cert_path = (
             _create_server_store_spin_feature_server(
                 temp_dir=remote_server_tmp_dir,
                 auth_config=auth_config,
                 permissions_list=permissions_list,
+                ssl_mode=ssl_mode,
             )
         )
         assert None not in (server_store, server_url, registry_path)
@@ -54,6 +57,7 @@ def test_remote_online_store_read(auth_config):
             server_registry_path=str(registry_path),
             feature_server_url=server_url,
             auth_config=auth_config,
+            ssl_cert_path=ssl_cert_path,
         )
         assert client_store is not None
         _assert_non_existing_entity_feature_views_entity(
@@ -159,21 +163,46 @@ def _assert_client_server_online_stores_are_matching(
 
 
 def _create_server_store_spin_feature_server(
-    temp_dir, auth_config: str, permissions_list
+    temp_dir, auth_config: str, permissions_list, ssl_mode: bool
 ):
     store = default_store(str(temp_dir), auth_config, permissions_list)
     feast_server_port = free_port()
+    if ssl_mode:
+        certificates_path = tempfile.mkdtemp()
+        ssl_key_path = os.path.join(certificates_path, "key.pem")
+        ssl_cert_path = os.path.join(certificates_path, "cert.pem")
+        generate_self_signed_cert(cert_path=ssl_cert_path, key_path=ssl_key_path)
+    else:
+        ssl_key_path = ""
+        ssl_cert_path = ""
+
     server_url = next(
         start_feature_server(
-            repo_path=str(store.repo_path), server_port=feast_server_port
+            repo_path=str(store.repo_path),
+            server_port=feast_server_port,
+            ssl_key_path=ssl_key_path,
+            ssl_cert_path=ssl_cert_path,
         )
     )
-    print(f"Server started successfully, {server_url}")
-    return store, server_url, os.path.join(store.repo_path, "data", "registry.db")
+    if ssl_cert_path and ssl_key_path:
+        print(f"Online Server started successfully in SSL mode, {server_url}")
+    else:
+        print(f"Server started successfully, {server_url}")
+
+    return (
+        store,
+        server_url,
+        os.path.join(store.repo_path, "data", "registry.db"),
+        ssl_cert_path,
+    )
 
 
 def _create_remote_client_feature_store(
-    temp_dir, server_registry_path: str, feature_server_url: str, auth_config: str
+    temp_dir,
+    server_registry_path: str,
+    feature_server_url: str,
+    auth_config: str,
+    ssl_cert_path: str = "",
 ) -> FeatureStore:
     project_name = "REMOTE_ONLINE_CLIENT_PROJECT"
     runner = CliRunner()
@@ -185,27 +214,35 @@ def _create_remote_client_feature_store(
         registry_path=server_registry_path,
         feature_server_url=feature_server_url,
         auth_config=auth_config,
+        ssl_cert_path=ssl_cert_path,
     )
 
     return FeatureStore(repo_path=repo_path)
 
 
 def _overwrite_remote_client_feature_store_yaml(
-    repo_path: str, registry_path: str, feature_server_url: str, auth_config: str
+    repo_path: str,
+    registry_path: str,
+    feature_server_url: str,
+    auth_config: str,
+    ssl_cert_path: str = "",
 ):
     repo_config = os.path.join(repo_path, "feature_store.yaml")
-    with open(repo_config, "w") as repo_config:
-        repo_config.write(
-            dedent(
-                f"""
-            project: {PROJECT_NAME}
-            registry: {registry_path}
-            provider: local
-            online_store:
-                path: {feature_server_url}
-                type: remote
-            entity_key_serialization_version: 2
-            """
-            )
-            + auth_config
-        )
+
+    config_content = "entity_key_serialization_version: 2\n" + auth_config
+    config_content += dedent(
+        f"""
+    project: {PROJECT_NAME}
+    registry: {registry_path}
+    provider: local
+    online_store:
+        path: {feature_server_url}
+        type: remote
+    """
+    )
+
+    if ssl_cert_path:
+        config_content += f"    ssl_cert_path: {ssl_cert_path}\n"
+
+    with open(repo_config, "w") as repo_config_file:
+        repo_config_file.write(config_content)

sdk/python/tests/utils/auth_permissions_util.py

@@ -54,7 +54,13 @@ def default_store(
     return fs
 
 
-def start_feature_server(repo_path: str, server_port: int, metrics: bool = False):
+def start_feature_server(
+    repo_path: str,
+    server_port: int,
+    metrics: bool = False,
+    ssl_key_path: str = "",
+    ssl_cert_path: str = "",
+):
     host = "0.0.0.0"
     cmd = [
         "feast",
@@ -65,6 +71,13 @@ def start_feature_server(repo_path: str, server_port: int, metrics: bool = False
         "--port",
         str(server_port),
     ]
+
+    if ssl_cert_path and ssl_cert_path:
+        cmd.append("--ssl-key-path")
+        cmd.append(ssl_key_path)
+        cmd.append("--ssl-cert-path")
+        cmd.append(ssl_cert_path)
+
     feast_server_process = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
     )
@@ -91,7 +104,13 @@ def start_feature_server(repo_path: str, server_port: int, metrics: bool = False
             "localhost", 8000
         ), "Prometheus server is running when it should be disabled."
 
-    yield f"http://localhost:{server_port}"
+    online_server_url = (
+        f"https://localhost:{server_port}"
+        if ssl_key_path and ssl_cert_path
+        else f"http://localhost:{server_port}"
+    )
+
+    yield (online_server_url)
 
     if feast_server_process is not None:
         feast_server_process.kill()