removed redis destination (#542)

we haven't supported this for a long time, lets remove it from the codebase
kaskada-ai · Aug 1, 2023 · 786d566 · 786d566
1 parent d5960e7
commit 786d566
Show file tree

Hide file tree

Showing 34 changed files with 219 additions and 425 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -94,7 +94,7 @@ Run `cargo build --release -p sparrow-main` to build a release (optimized) binar
 ### Testing & Building the API
 
 * ensure docker is running locally
-* run `make proto/generate` and `make ent/generate`.  See the `./wren/README.md` for more info on those.
+* run `make proto/generate`, `make ent/generate`, `make wren/generate-mocks`.  See the `./wren/README.md` for more info on those.
 * run `make wren/test`
 
 ### Testing & Building the Python Client
@@ -129,6 +129,7 @@ After making code changes, `ctrl-c` in the services window and restart it.
 
 #### locally, with the local backend
 
+* run `make test/int/docker-up-dependencies-only` in one terminal window to get the dependencies up
 * run `make sparrow/run` in one terminal window to get the Engine service up
 * run `make wren/run` in a second terminal window to get the Manager service up
 * run `make test/int/run-api` in a third another terminal window to run the integration tests
@@ -137,7 +138,7 @@ After making code changes, `ctrl-c` in the proper service window and restart it.
 
 #### locally, with the s3 backend
 
-* run `make test/int/docker-up-s3-only` in one terminal window to get the dependencies up
+* run `make test/int/docker-up-dependencies-only` in one terminal window to get the dependencies up
 * run `make sparrow/run-s3` in a second terminal window to get the Engine service up
 * run `make wren/run-s3` in a third terminal window to get the Manager service up
 * run `make test/int/run-api-s3` in a fourth terminal window to run the integration tests
@@ -157,4 +158,4 @@ After making code changes, `ctrl-c` in the proper service window and restart it.
     Provides the language server integration for Rust code.
 *  * [Even Better TOML](https://marketplace.visualstudio.com/items?itemName=tamasfe.even-better-toml). Optional.
 *  * [Cargo](https://marketplace.visualstudio.com/items?itemName=panicbit.cargo). Optional.
-*  * [Crates](https://marketplace.visualstudio.com/items?itemName=serayuzgur.crates) Optional.
+*  * [Crates](https://marketplace.visualstudio.com/items?itemName=serayuzgur.crates) Optional.
diff --git a/Makefile b/Makefile
@@ -64,8 +64,8 @@ test/int/docker-up:
 test/int/docker-up-s3:
 	docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.s3.yml up --build --remove-orphans --force-recreate
 
-test/int/docker-up-s3-only:
-	docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.s3.yml up --build --remove-orphans --force-recreate minio
+test/int/docker-up-dependencies-only:
+	docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.s3.yml up --build --remove-orphans --force-recreate minio pulsar
 
 test/int/docker-up-postgres:
 	docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.postgres.yml up --build --remove-orphans --force-recreate

diff --git a/clients/python/src/kaskada/formatters.py b/clients/python/src/kaskada/formatters.py
@@ -58,14 +58,6 @@ def get_query_response_content(resp_obj):
             path = resp_obj.file_results.paths[i]
             appendChildIfNotNone(nested_table, tr(td(pre(i)), td(a(path, _href=path))))
         resultsExist = True
-    elif hasattr(resp_obj, "redis_bulk"):
-        nested_table = table(_class="kda_table")
-        details.appendChild(html_table_row("redis_bulk", nested_table))
-        nested_table.appendChild(tr(th("index"), th("path")))
-        for i in range(len(resp_obj.redis_bulk.paths)):
-            path = resp_obj.redis_bulk.paths[i]
-            appendChildIfNotNone(nested_table, tr(td(pre(i)), td(a(path, _href=path))))
-        resultsExist = True
 
     (
         can_execute,
@@ -689,13 +681,6 @@ def try_init():
         html_formatter.for_type(
             "kaskada.materialization.MaterializationView", generic_object_html_formatter
         )
-        html_formatter.for_type(
-            "kaskada.materialization.RedisAIDestination", generic_object_html_formatter
-        )
-        html_formatter.for_type(
-            "kaskada.kaskada.v1alpha.materialization_service_pb2.RedisAI",
-            generic_object_html_formatter,
-        )
         html_formatter.for_type(
             "kaskada.kaskada.v1alpha.materialization_service_pb2.Destination",
             generic_object_html_formatter,

diff --git a/clients/python/src/kaskada/formatters_shared.py b/clients/python/src/kaskada/formatters_shared.py
@@ -123,20 +123,6 @@ def get_materialization_html_and_schema_df(obj):
     if hasattr(obj, "destination") and obj.HasField("destination"):
         destination = table(_class="kda_table")
         details.appendChild(html_table_row("destination", destination))
-        if hasattr(obj.destination, "redis_a_i") and obj.destination.HasField(
-            "redis_a_i"
-        ):
-            redis_a_i = table(_class="kda_table")
-            appendHtmlObjTableRowIfAttrExists(
-                redis_a_i, obj.destination.redis_a_i, "host"
-            )
-            appendHtmlObjTableRowIfAttrExists(
-                redis_a_i, obj.destination.redis_a_i, "port"
-            )
-            appendHtmlObjTableRowIfAttrExists(
-                redis_a_i, obj.destination.redis_a_i, "db"
-            )
-            destination.appendChild(html_table_row("redis_a_i", redis_a_i))
 
     if hasattr(obj, "slice"):
         details.appendChild(html_table_row("slice", get_slice_request_html(obj.slice)))

diff --git a/clients/python/src/kaskada/materialization.py b/clients/python/src/kaskada/materialization.py
@@ -23,44 +23,6 @@ def to_request(self) -> Dict[str, Any]:
         pass
 
 
-class RedisDestination(Destination):
-    def __init__(
-        self,
-        host_name: str,
-        port: int,
-        use_tls: bool,
-        database_number: int,
-        password: str,
-        tls_cert: str,
-        tls_key: str,
-        tls_ca_cert: str,
-        insecure_skip_verify: bool,
-    ) -> None:
-        super().__init__()
-        self._host_name = host_name
-        self._port = port
-        self._use_tls = use_tls
-        self._database_number = database_number
-        self._password = password
-        self._tls_cert = tls_cert
-        self._tls_key = tls_key
-        self._tls_ca_cert = tls_ca_cert
-        self._insecure_skip_verify = insecure_skip_verify
-
-    def to_request(self) -> Dict[str, Any]:
-        return {
-            "host_name": self._host_name,
-            "port": self._port,
-            "use_tls": self._use_tls,
-            "database_number": self._database_number,
-            "password": self._password,
-            "tls_cert": self._tls_cert,
-            "tls_key": self._tls_key,
-            "tls_ca_cert": self._tls_ca_cert,
-            "insecure_skip_verify": self._insecure_skip_verify,
-        }
-
-
 class FileType(Enum):
     FILE_TYPE_UNSPECIFIED = 0
     FILE_TYPE_PARQUET = 1
@@ -160,8 +122,6 @@ def create_materialization(
         }
         if isinstance(destination, ObjectStoreDestination):
             materialization["destination"] = {"object_store": destination.to_request()}
-        elif isinstance(destination, RedisDestination):
-            materialization["destination"] = {"redis": destination.to_request()}
         elif isinstance(destination, PulsarDestination):
             materialization["destination"] = {
                 "pulsar": {"config": destination.to_request()}

diff --git a/clients/python/tests/test_materialization.py b/clients/python/tests/test_materialization.py
@@ -11,7 +11,6 @@
     MaterializationView,
     ObjectStoreDestination,
     PulsarDestination,
-    RedisDestination,
     create_materialization,
     delete_materialization,
     get_materialization,
@@ -21,23 +20,6 @@
 from kaskada.slice_filters import EntityFilter
 
 
-def test_redis_destination_to_request():
-    params = {
-        "host_name": "my_host_name",
-        "port": 1234,
-        "use_tls": False,
-        "database_number": 4321,
-        "password": "my_password",
-        "tls_cert": "my_tls_cert",
-        "tls_key": "my_tls_key",
-        "tls_ca_cert": "my_tls_ca_cert",
-        "insecure_skip_verify": True,
-    }
-
-    result = RedisDestination(**params)
-    assert result.to_request() == params
-
-
 def test_object_store_destination_to_request():
     csv_file = FileType.FILE_TYPE_CSV
     output_prefix = "/my_prefix"
@@ -218,7 +200,7 @@ def to_request(self) -> Dict[str, Any]:
       entity_keys: "my_entity_b"
     }
   }
-  
+
 """
 
 
@@ -300,54 +282,6 @@ def test_create_materialization_object_store_parquet_destination(mockClient):
     )
 
 
-@patch("kaskada.client.Client")
-def test_create_materialization_redis_destination(mockClient):
-    params = {
-        "host_name": "my_host_name",
-        "port": 1234,
-        "use_tls": False,
-        "database_number": 4321,
-        "password": "my_password",
-        "tls_cert": "my_tls_cert",
-        "tls_key": "my_tls_key",
-        "tls_ca_cert": "my_tls_ca_cert",
-        "insecure_skip_verify": True,
-    }
-
-    redis_destination = RedisDestination(**params)
-
-    name = "my_awkward_tacos"
-    expression = "last(tacos)"
-    destination = redis_destination
-    views = [MaterializationView("my_second_view", "last(awkward)")]
-    slice_filter = EntityFilter(["my_entity_a", "my_entity_b"])
-
-    expected_request = material_pb.CreateMaterializationRequest(
-        **{
-            "materialization": {
-                "materialization_name": name,
-                "expression": expression,
-                "with_views": [
-                    {"name": "my_second_view", "expression": "last(awkward)"}
-                ],
-                "destination": {"redis": redis_destination.to_request()},
-                "slice": slice_filter.to_request(),
-            }
-        }
-    )
-    create_materialization(
-        name,
-        expression,
-        destination,
-        views,
-        slice_filter=slice_filter,
-        client=mockClient,
-    )
-    mockClient.materialization_stub.CreateMaterialization.assert_called_with(
-        expected_request, metadata=mockClient.get_metadata()
-    )
-
-
 @patch("kaskada.client.Client")
 def test_create_materialization_astra_streaming_destination(mockClient):
     params = {

diff --git a/crates/sparrow-runtime/src/execute/output.rs b/crates/sparrow-runtime/src/execute/output.rs
@@ -59,11 +59,6 @@ impl TryFrom<sparrow_api::kaskada::v1alpha::Destination> for Destination {
             sparrow_api::kaskada::v1alpha::destination::Destination::ObjectStore(destination) => {
                 Ok(Destination::ObjectStore(destination))
             }
-            sparrow_api::kaskada::v1alpha::destination::Destination::Redis(_) => {
-                error_stack::bail!(Error::FeatureNotEnabled {
-                    feature: "redis".to_owned()
-                })
-            }
             #[cfg(not(feature = "pulsar"))]
             sparrow_api::kaskada::v1alpha::destination::Destination::Pulsar(_) => {
                 error_stack::bail!(Error::FeatureNotEnabled {

diff --git a/crates/sparrow-runtime/src/execute/output/redis.rs b/crates/sparrow-runtime/src/execute/output/redis.rs
diff --git a/crates/sparrow-runtime/src/execute/progress_reporter.rs b/crates/sparrow-runtime/src/execute/progress_reporter.rs
@@ -181,9 +181,6 @@ impl ProgressTracker {
                     })),
                 })
             }
-            destination::Destination::Redis(_) => {
-                error_stack::bail!(Error::UnsupportedOutput { output: "redis" })
-            }
         }
     }
 }