Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add local tests for s3 registry using minio #4029

Merged
merged 5 commits into from
Mar 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ test-python-unit:
python -m pytest -n 8 --color=yes sdk/python/tests

test-python-integration:
python -m pytest -n 8 --integration --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
python -m pytest -n 8 --integration -k "not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests

test-python-integration-local:
@(docker info > /dev/null 2>&1 && \
Expand Down
60 changes: 57 additions & 3 deletions sdk/python/tests/integration/registration/test_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@
import os
import time
from datetime import timedelta
from unittest import mock

import pytest
from pytest_lazyfixture import lazy_fixture
from testcontainers.core.container import DockerContainer

from feast import FileSource
from feast.data_format import ParquetFormat
Expand Down Expand Up @@ -60,12 +62,56 @@ def s3_registry() -> Registry:
return Registry("project", registry_config, None)


@pytest.fixture
def minio_registry() -> Registry:
minio_user = "minio99"
minio_password = "minio123"
bucket_name = "test-bucket"

container: DockerContainer = (
DockerContainer("quay.io/minio/minio")
.with_exposed_ports(9000, 9001)
.with_env("MINIO_ROOT_USER", minio_user)
.with_env("MINIO_ROOT_PASSWORD", minio_password)
.with_command('server /data --console-address ":9001"')
.with_exposed_ports()
)

container.start()

exposed_port = container.get_exposed_port("9000")
container_host = container.get_container_host_ip()

container.exec(f"mkdir /data/{bucket_name}")

registry_config = RegistryConfig(
path=f"s3://{bucket_name}/registry.db", cache_ttl_seconds=600
)

mock_environ = {
"FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}",
"AWS_ACCESS_KEY_ID": minio_user,
"AWS_SECRET_ACCESS_KEY": minio_password,
"AWS_SESSION_TOKEN": "",
}

with mock.patch.dict(os.environ, mock_environ):
yield Registry("project", registry_config, None)

container.stop()


@pytest.mark.integration
@pytest.mark.parametrize(
"test_registry",
[lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")],
[
lazy_fixture("gcs_registry"),
lazy_fixture("s3_registry"),
lazy_fixture("minio_registry"),
],
)
def test_apply_entity_integration(test_registry):

entity = Entity(
name="driver_car_id",
description="Car driver id",
Expand Down Expand Up @@ -106,7 +152,11 @@ def test_apply_entity_integration(test_registry):
@pytest.mark.integration
@pytest.mark.parametrize(
"test_registry",
[lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")],
[
lazy_fixture("gcs_registry"),
lazy_fixture("s3_registry"),
lazy_fixture("minio_registry"),
],
)
def test_apply_feature_view_integration(test_registry):
# Create Feature Views
Expand Down Expand Up @@ -183,7 +233,11 @@ def test_apply_feature_view_integration(test_registry):
@pytest.mark.integration
@pytest.mark.parametrize(
"test_registry",
[lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")],
[
lazy_fixture("gcs_registry"),
lazy_fixture("s3_registry"),
lazy_fixture("minio_registry"),
],
)
def test_apply_data_source_integration(test_registry: Registry):
validate_registry_data_source_apply(test_registry)
Loading