Make db requirements optional

.github/workflows/CI.yaml

@@ -42,7 +42,7 @@ jobs:
         run: |
           source .tenv/bin/activate
           python -m build --wheel
-          pip install $(ls dist/*.whl)[stable,test] --cache-dir .pip-cache
+          pip install $(ls dist/*.whl)[db,test] --cache-dir .pip-cache
 
       - name: Run unit tests
         run: |

.pre-commit-config.yaml

@@ -22,11 +22,12 @@ repos:
         additional_dependencies:
           - numpy>=1.21
           - sqlalchemy[mypy]
+          - alembic
         args: [--install-types, --non-interactive]
         # Note that using the --install-types is problematic if running in
         # parallel as mutating the pre-commit env at runtime breaks cache.
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
 ...

pyproject.toml

@@ -44,40 +44,27 @@ dependencies = [
     "numpy >= 1.18",
     "pandas >= 1.2",
     "scipy >= 1.5",
-    "psycopg2 >= 2.9",
     "seaborn >= 0.11.0",
-    "sqlalchemy >= 1.4",
 ]
 
   [project.optional-dependencies]
-  stable = [
-    "numpy == 1.21.6",
-    "pandas == 1.3.5",
-    "scipy == 1.7.3",
-    "sqlalchemy_utils",
-    "synthesized_datasets == 0.6",
+  db = [
+    "alembic",
+    "sqlalchemy >= 2.0",
+    "psycopg2 >= 2.9",
   ]
+
   test = [
     "pytest",
     "pytest-cov",
   ]
   dev = [
     "pre-commit",
-    "black",
-    "mypy",
     "pylint",
   ]
-  doc = [
-    "sphinx == 4.5.0",
-    "pydata-sphinx-theme == 0.8.1",
-    "sphinx-panels == 0.6.0",
-    "sphinx-autodoc-typehints == 1.18.1",
-    "sphinx-inline-tabs == 2022.1.2b11",
-    "sphinxcontrib-spelling == 7.3.2",
-    "ipython == 8.2.0",
-    "myst-parser == 0.17.2",
-    "sphinxcontrib.asciinema == 0.3.3",
-  ]
+
+  [project.scripts]
+  insight-migrations = "insight.alembic.main:main"
 
   [project.urls]
   homepage = "https://github.com/synthesized-io/insight"

src/insight/__init__.py

@@ -1,3 +1,4 @@
+from . import metrics, plot
 from .check import ColumnCheck
 
-__all__ = ['ColumnCheck']
+__all__ = ['ColumnCheck', 'plot', 'metrics']

src/insight/alembic/alembic.ini

@@ -0,0 +1,4 @@
+[alembic]
+script_location = insight:alembic
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+sqlalchemy.url = postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DATABASE}

migrations/env.py → src/insight/alembic/env.py

@@ -1,5 +1,4 @@
 import os
-from logging.config import fileConfig
 
 from alembic import context
 from sqlalchemy import engine_from_config, pool
@@ -10,11 +9,6 @@
 # access to the values within the .ini file in use.
 config = context.config
 
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
-
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
@@ -35,7 +29,11 @@ def run_migrations_offline() -> None:
     script output.
 
     """
-    url = config.get_main_option("sqlalchemy.url").format(**os.environ)
+    url = config.get_main_option("sqlalchemy.url")
+    if url is None:
+        raise ValueError("No sqlalchemy.url specified in config file")
+
+    url = url.format(**os.environ)
     context.configure(
         url=url,
         target_metadata=target_metadata,
@@ -54,9 +52,26 @@ def run_migrations_online() -> None:
     and associate a connection with the context.
 
     """
-    config.set_main_option("sqlalchemy.url", config.get_main_option("sqlalchemy.url").format(**os.environ))
+
+    POSTGRES_PASSWORD = os.environ["POSTGRES_PASSWORD"]
+    POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost")
+    POSTGRES_PORT = os.environ.get("POSTGRES_PORT", "5432")
+    POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres")
+    POSTGRES_DATABASE = os.environ.get("POSTGRES_DATABASE", "postgres")
+
+    url = config.get_main_option("sqlalchemy.url")
+    if url is None:
+        raise ValueError("No sqlalchemy.url specified in config file")
+
+    config.set_main_option("sqlalchemy.url", url.format(
+        POSTGRES_USER=POSTGRES_USER,
+        POSTGRES_PASSWORD=POSTGRES_PASSWORD,
+        POSTGRES_HOST=POSTGRES_HOST,
+        POSTGRES_PORT=POSTGRES_PORT,
+        POSTGRES_DATABASE=POSTGRES_DATABASE
+    ))
     connectable = engine_from_config(
-        config.get_section(config.config_ini_section),
+        config.get_section(config.config_ini_section) or {},
         prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )

src/insight/alembic/main.py

@@ -0,0 +1,15 @@
+# Python script that will apply the migrations up to head
+import os
+
+import alembic.config
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+alembic_args = [
+    '-c', os.path.join(here, 'alembic.ini'),
+    'upgrade', 'head'
+]
+
+
+def main():
+    alembic.config.main(argv=alembic_args)

src/insight/database/schema.py

@@ -1,47 +1,50 @@
-from sqlalchemy import FLOAT, INTEGER, TIMESTAMP, VARCHAR, Column, ForeignKey
-from sqlalchemy.orm import declarative_base, relationship
+from sqlalchemy import FLOAT, INTEGER, TIMESTAMP, VARCHAR, ForeignKey
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 from sqlalchemy.sql import func
 
-Base = declarative_base()
+
+class Base(DeclarativeBase):
+    pass
+
 
 
 class Dataset(Base):
     __tablename__ = "dataset"
 
-    id = Column(INTEGER, primary_key=True)
-    name = Column(VARCHAR(50), nullable=False)
-    num_rows = Column(INTEGER)
-    num_columns = Column(INTEGER)
-    created_at = Column(TIMESTAMP, default=func.now())
+    id: Mapped[int] = mapped_column(INTEGER, primary_key=True)
+    name = mapped_column(VARCHAR(50), nullable=False)
+    num_rows = mapped_column(INTEGER)
+    num_columns = mapped_column(INTEGER)
+    created_at = mapped_column(TIMESTAMP, default=func.now())
 
 
 class Metric(Base):
     __tablename__ = "metric"
 
-    id = Column(INTEGER, primary_key=True)
-    name = Column(VARCHAR(50), nullable=False)
-    category = Column(VARCHAR(50))
-    created_at = Column(TIMESTAMP, default=func.now())
+    id = mapped_column(INTEGER, primary_key=True)
+    name = mapped_column(VARCHAR(50), nullable=False)
+    category = mapped_column(VARCHAR(50))
+    created_at = mapped_column(TIMESTAMP, default=func.now())
 
 
 class Version(Base):
     __tablename__ = "version"
 
-    id = Column(INTEGER, primary_key=True)
-    name = Column(VARCHAR(50), nullable=False, default="unversioned")
-    created_at = Column(TIMESTAMP, default=func.now())
+    id = mapped_column(INTEGER, primary_key=True)
+    name = mapped_column(VARCHAR(50), nullable=False, default="unversioned")
+    created_at = mapped_column(TIMESTAMP, default=func.now())
 
 
 class Result(Base):
     __tablename__ = "result"
 
-    id = Column(INTEGER, primary_key=True)
-    metric_id = Column(INTEGER, ForeignKey("metric.id"))
-    dataset_id = Column(INTEGER, ForeignKey("dataset.id"))
-    version_id = Column(INTEGER, ForeignKey("version.id"))
-    value = Column(FLOAT)
-    created_at = Column(TIMESTAMP, default=func.now())
+    id = mapped_column(INTEGER, primary_key=True)
+    metric_id = mapped_column(INTEGER, ForeignKey("metric.id"))
+    dataset_id = mapped_column(INTEGER, ForeignKey("dataset.id"))
+    version_id = mapped_column(INTEGER, ForeignKey("version.id"))
+    value = mapped_column(FLOAT)
+    created_at = mapped_column(TIMESTAMP, default=func.now())
 
-    metric: Metric = relationship("Metric")
-    dataset: Dataset = relationship("Dataset")
-    version: Version = relationship("Version")
+    metric: Mapped[Metric] = relationship("Metric")
+    dataset: Mapped[Dataset] = relationship("Dataset")
+    version: Mapped[Version] = relationship("Version")

src/insight/database/utils.py

@@ -109,8 +109,19 @@ def get_session() -> typing.Optional[Session]:
 
     """
     try:
+        POSTGRES_PASSWORD = os.environ["POSTGRES_PASSWORD"]
+        POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost")
+        POSTGRES_PORT = os.environ.get("POSTGRES_PORT", "5432")
+        POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres")
+        POSTGRES_DATABASE = os.environ.get("POSTGRES_DATABASE", "postgres")
         db_url = "postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@" \
-                 "{POSTGRES_HOST}:{POSTGRES_PORT}".format(**os.environ)
+                 "{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DATABASE}".format(
+            POSTGRES_HOST=POSTGRES_HOST,
+            POSTGRES_PORT=POSTGRES_PORT,
+            POSTGRES_USER=POSTGRES_USER,
+            POSTGRES_PASSWORD=POSTGRES_PASSWORD,
+            POSTGRES_DATABASE=POSTGRES_DATABASE
+        )
         engine = create_engine(db_url, future=True)
 
         session_constructor = sessionmaker(bind=engine, future=True)