Cleanup dead code

CHANGELOG.md

@@ -1,6 +1,6 @@
 # superduper.io Changelog
 
-All notable changes to this project will be documented in this file.
+All notable changes to this project will be documented in this file
 
 The format is inspired by (but not strictly follows) [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

plugins/anthropic/superduper_anthropic/model.py

@@ -5,8 +5,8 @@
 from anthropic import APIConnectionError, APIError, APIStatusError, APITimeoutError
 from superduper.backends.query_dataset import QueryDataset
 from superduper.components.model import APIBaseModel
-from superduper.ext.utils import format_prompt, get_key
 from superduper.misc.retry import Retry
+from superduper.misc.utils import format_prompt, get_key
 
 retry = Retry(
     exception_types=(APIConnectionError, APIError, APIStatusError, APITimeoutError)

plugins/cohere/superduper_cohere/model.py

@@ -6,8 +6,8 @@
 from cohere.error import CohereAPIError, CohereConnectionError
 from superduper.backends.query_dataset import QueryDataset
 from superduper.components.model import APIBaseModel
-from superduper.ext.utils import format_prompt, get_key
 from superduper.misc.retry import Retry
+from superduper.misc.utils import format_prompt, get_key
 
 retry = Retry(exception_types=(CohereAPIError, CohereConnectionError))
 

plugins/ibis/superduper_ibis/data_backend.py

@@ -96,7 +96,7 @@ def __init__(self, uri: str, flavour: t.Optional[str] = None):
         self.overwrite = False
         self._setup(conn)
 
-        self.datatype_presets = {'vector': 'superduper.ext.numpy.encoder.Array'}
+        self.datatype_presets = {'vector': 'superduper.components.datatype.Array'}
 
         if uri.startswith('snowflake://'):
             self.bytes_encoding = 'base64'

plugins/jina/superduper_jina/client.py

@@ -4,8 +4,8 @@
 import requests
 from aiohttp import ClientConnectionError, ClientResponseError
 from requests.exceptions import HTTPError
-from superduper.ext.utils import get_key
 from superduper.misc.retry import Retry
+from superduper.misc.utils import get_key
 
 JINA_API_URL: str = "https://api.jina.ai/v1/embeddings"
 KEY_NAME = 'JINA_API_KEY'

plugins/mongodb/plugin_test/test_query.py

@@ -3,9 +3,9 @@
 import numpy as np
 import pytest
 from superduper.base.document import Document
+from superduper.components.datatype import Array
 from superduper.components.schema import Schema
 from superduper.components.table import Table
-from superduper.ext.numpy.encoder import Array
 
 from superduper_mongodb.query import MongoQuery
 

plugins/mongodb/superduper_mongodb/artifacts.py

@@ -7,7 +7,6 @@
 import gridfs
 from superduper import CFG, logging
 from superduper.backends.base.artifacts import ArtifactStore
-from superduper.misc.colors import Colors
 from tqdm import tqdm
 
 from superduper_mongodb.utils import connection_callback
@@ -48,8 +47,8 @@ def drop(self, force: bool = False):
         """
         if not force:
             if not click.confirm(
-                f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
-                f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
+                '!!!WARNING USE WITH CAUTION AS YOU '
+                'WILL LOSE ALL DATA!!!]\n'
                 'Are you sure you want to drop all artifacts? ',
                 default=False,
             ):

plugins/mongodb/superduper_mongodb/data_backend.py

@@ -11,7 +11,6 @@
 from superduper.base.enums import DBType
 from superduper.components.datatype import BaseDataType
 from superduper.components.schema import Schema
-from superduper.misc.colors import Colors
 
 from superduper_mongodb.artifacts import MongoArtifactStore
 from superduper_mongodb.metadata import MongoMetaDataStore
@@ -113,8 +112,8 @@ def drop(self, force: bool = False):
         """
         if not force:
             if not click.confirm(
-                f"{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU "
-                f"WILL LOSE ALL DATA!!!]{Colors.RESET} "
+                '!!!WARNING USE WITH CAUTION AS YOU '
+                "WILL LOSE ALL DATA!!!]\n"
                 "Are you sure you want to drop the data-backend? ",
                 default=False,
             ):

plugins/mongodb/superduper_mongodb/metadata.py

@@ -5,7 +5,6 @@
 from superduper import logging
 from superduper.backends.base.metadata import MetaDataStore
 from superduper.components.component import Status
-from superduper.misc.colors import Colors
 
 
 class MongoMetaDataStore(MetaDataStore):
@@ -61,8 +60,8 @@ def drop(self, force: bool = False):
         """
         if not force:
             if not click.confirm(
-                f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
-                f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
+                '[!!!WARNING USE WITH CAUTION AS YOU '
+                'WILL LOSE ALL DATA!!!]\n '
                 'Are you sure you want to drop all meta-data? ',
                 default=False,
             ):

plugins/sqlalchemy/superduper_sqlalchemy/metadata.py

@@ -21,7 +21,6 @@
 from superduper import logging
 from superduper.backends.base.metadata import MetaDataStore, NonExistentMetadataError
 from superduper.components.component import Status
-from superduper.misc.colors import Colors
 
 from superduper_sqlalchemy.db_helper import get_db_config
 
@@ -331,8 +330,8 @@ def drop(self, force: bool = False):
         """
         if not force:
             if not click.confirm(
-                f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
-                f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
+                '!!!WARNING USE WITH CAUTION AS YOU '
+                'WILL LOSE ALL DATA!!!\n '
                 'Are you sure you want to drop all meta-data? ',
                 default=False,
             ):

plugins/torch/superduper_torch/encoder.py

@@ -3,54 +3,12 @@
 import numpy
 import torch
 from superduper.components.datatype import BaseDataType, DataTypeFactory
-from superduper.ext.numpy.encoder import DecodeArray, EncodeArray
-from superduper.ext.utils import str_shape
+from superduper.misc.utils import str_shape
 
 if t.TYPE_CHECKING:
     pass
 
 
-class EncodeTensor:
-    """Encode a tensor to bytes # noqa.
-
-    :param dtype: The dtype of the tensor, eg. torch.float32
-    """
-
-    def __init__(self, dtype):
-        self.dtype = dtype
-
-    def __call__(self, x, info: t.Optional[t.Dict] = None):
-        """Encode a tensor to bytes.
-
-        :param x: The tensor to encode.
-        :param info: Additional information.
-        """
-        if x.dtype != self.dtype:
-            raise TypeError(f"dtype was {x.dtype}, expected {self.dtype}")
-        return memoryview(x.numpy()).tobytes()
-
-
-class DecodeTensor:
-    """Decode a tensor from bytes # noqa.
-
-    :param dtype: The dtype of the tensor, eg. torch.float32
-    :param shape: The shape of the tensor, eg. (3, 4)
-    """
-
-    def __init__(self, dtype, shape):
-        self.dtype = torch.from_numpy(numpy.random.randn(1).astype(dtype)).dtype
-        self.shape = shape
-
-    def __call__(self, bytes, info: t.Optional[t.Dict] = None):
-        """Decode a tensor from bytes.
-
-        :param bytes: The bytes to decode.
-        :param info: Additional information.
-        """
-        array = numpy.frombuffer(bytes, dtype=self.dtype).reshape(self.shape)
-        return torch.from_numpy(array)
-
-
 class Tensor(BaseDataType):
     """Encode/ decode a numpy array as bytes.
 
@@ -74,19 +32,15 @@ def encode_data(self, item):
 
         :param item: item to encode.
         """
-        encoder = EncodeArray(self.dtype)
-        return encoder(item.numpy())
+        return memoryview(item.numpy()).tobytes()
 
     def decode_data(self, item):
         """Decode data.
 
         :param item: item to decode.
         """
-        shape = self.shape
-        if isinstance(shape, int):
-            shape = (self.shape,)
-        decoder = DecodeArray(self.dtype, shape=shape)
-        return torch.from_numpy(decoder(item))
+        array = numpy.frombuffer(item, dtype=self.dtype).reshape(self.shape)
+        return torch.from_numpy(array)
 
 
 class TorchDataTypeFactory(DataTypeFactory):

superduper/__init__.py

@@ -18,7 +18,6 @@
     __version__ = "dev"
 
 
-from .base.decorators import code
 from .base.document import Document
 from .base.leaf import imported, imported_value
 from .components.application import Application

superduper/backends/base/compute.py

@@ -30,6 +30,7 @@ def release_futures(self, context: str):
         """Release futures from backend."""
         pass
 
+    # TODO needed?
     @property
     @abstractmethod
     def remote(self) -> bool:
@@ -89,9 +90,6 @@ def shutdown(self) -> None:
         """Shuts down the compute cluster."""
         pass
 
-    def execute_task(self, job_id, dependencies, compute_kwargs={}):
-        """Execute task function for distributed backends."""
-
     def initialize(self):
         """Connect to address."""
 

superduper/backends/base/queue.py

@@ -21,7 +21,6 @@
 def _chunked_list(lst, batch_size=BATCH_SIZE):
     if len(lst) <= batch_size:
         return [lst]
-
     return [lst[i : i + batch_size] for i in range(0, len(lst), batch_size)]
 
 
@@ -118,8 +117,6 @@ class JobFutureException(Exception):
     # noqa
     """
 
-    ...
-
 
 def consume_streaming_events(events, table, db):
     """

superduper/backends/local/artifacts.py

@@ -7,7 +7,6 @@
 
 from superduper import logging
 from superduper.backends.base.artifacts import ArtifactStore
-from superduper.misc.colors import Colors
 
 
 class FileSystemArtifactStore(ArtifactStore):
@@ -60,8 +59,8 @@ def drop(self, force: bool = False):
         """
         if not force:
             if not click.confirm(
-                f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
-                f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
+                '!!!WARNING USE WITH CAUTION AS YOU '
+                'WILL LOSE ALL DATA!!!]\n'
                 'Are you sure you want to drop all artifacts? ',
                 default=False,
             ):

superduper/backends/local/cluster.py

@@ -16,7 +16,7 @@
 from superduper.backends.local.crontab import LocalCrontabBackend
 from superduper.backends.local.queue import LocalQueuePublisher
 from superduper.backends.local.vector_search import LocalVectorSearchBackend
-from superduper.misc.plugins import load_plugin
+from superduper.misc.importing import load_plugin
 
 
 class LocalCluster(Cluster):

superduper/base/apply.py

@@ -24,7 +24,7 @@ def apply(
     object: t.Union['Component', t.Sequence[t.Any], t.Any],
     force: bool | None = None,
     wait: bool = False,
-):
+) -> 'Component':
     """
     Add functionality in the form of components.
 
@@ -36,8 +36,6 @@ def apply(
     :param force: List of jobs which should execute before component
                             initialization begins.
     :param wait: Blocks execution till create events finish.
-
-    :return: Tuple containing the added object(s) and the original object(s).
     """
     if force is None:
         force = db.cfg.force_apply

superduper/base/build.py

@@ -10,7 +10,7 @@
 from superduper.base.config import Config
 from superduper.base.datalayer import Datalayer
 from superduper.misc.anonymize import anonymize_url
-from superduper.misc.plugins import load_plugin
+from superduper.misc.importing import load_plugin
 
 
 class _Loader:
@@ -103,16 +103,6 @@ def _build_metadata(uri):
     return db
 
 
-def _build_compute(cfg):
-    """
-    Helper function to build compute backend.
-
-    :param cfg: SuperDuper config.
-    """
-    backend = getattr(load_plugin(cfg.cluster.compute.backend), 'ComputeBackend')
-    return backend(uri=cfg.cluster.compute.uri, **cfg.cluster.compute.kwargs)
-
-
 def build_datalayer(cfg=None, **kwargs) -> Datalayer:
     """
     Build a Datalayer object as per ``db = superduper(db)`` from configuration.

superduper/base/config.py

@@ -172,6 +172,8 @@ class Config(BaseConfig):
     :param rest: Settings for rest server.
     """
 
+    # TODO make _fold optional
+
     envs: dc.InitVar[t.Optional[t.Dict[str, str]]] = None
 
     data_backend: str = "mongodb://localhost:27017/test_db"

superduper/base/datalayer.py

@@ -21,7 +21,6 @@
 from superduper.components.schema import Schema
 from superduper.components.table import Table
 from superduper.misc.annotations import deprecated
-from superduper.misc.colors import Colors
 from superduper.misc.importing import import_object
 from superduper.misc.retry import db_retry
 
@@ -125,8 +124,8 @@ def drop(self, force: bool = False, data: bool = False):
         :param force: Force drop.
         """
         if not force and not click.confirm(
-            f"{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU WILL"
-            f"LOSE ALL DATA!!!]{Colors.RESET} "
+            "!!!WARNING USE WITH CAUTION AS YOU WILL"
+            "LOSE ALL DATA!!!]\n"
             "Are you sure you want to drop the database? ",
             default=False,
         ):

superduper/base/document.py

@@ -38,6 +38,8 @@ class Getters:
     :param getters: A dictionary of getters.
     """
 
+    # TODO replace with temporary artifact store.
+
     def __init__(self, getters=None):
         self._getters = defaultdict(list)
         for k, v in (getters or {}).items():
@@ -46,10 +48,6 @@ def __init__(self, getters=None):
     def add_getter(self, name: str, getter: t.Callable):
         """Add a getter for a reference type."""
         self._getters[name].append(getter)
-        # if name == 'blob':
-        #     self._getters[name].append(_build_blob_getter(getter))
-        # else:
-        #     self._getters[name].append(getter)
 
     def run(self, name, data):
         """Run the getters one by one until one returns a value."""

superduper/base/exceptions.py

@@ -1,3 +1,5 @@
+# TODO add more exceptions
+# for example ArtifactNotFoundError
 from superduper import logging