Skip to content

Commit

Permalink
Make db requirements optional (#110)
Browse files Browse the repository at this point in the history
* Make db requirements optional
* Add alembic to package
* Update font and plotting module
* Fix mpl font file + DB defaults
  • Loading branch information
simonhkswan authored Mar 10, 2023
1 parent a318a2e commit 69d43ba
Show file tree
Hide file tree
Showing 18 changed files with 189 additions and 247 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/CI.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
run: |
source .tenv/bin/activate
python -m build --wheel
pip install $(ls dist/*.whl)[stable,test] --cache-dir .pip-cache
pip install $(ls dist/*.whl)[db,test] --cache-dir .pip-cache
- name: Run unit tests
run: |
Expand Down
3 changes: 2 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ repos:
additional_dependencies:
- numpy>=1.21
- sqlalchemy[mypy]
- alembic
args: [--install-types, --non-interactive]
# Note that using the --install-types is problematic if running in
# parallel as mutating the pre-commit env at runtime breaks cache.
- repo: https://github.com/pycqa/isort
rev: 5.10.1
rev: 5.12.0
hooks:
- id: isort
...
105 changes: 0 additions & 105 deletions alembic.ini

This file was deleted.

29 changes: 8 additions & 21 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,40 +44,27 @@ dependencies = [
"numpy >= 1.18",
"pandas >= 1.2",
"scipy >= 1.5",
"psycopg2 >= 2.9",
"seaborn >= 0.11.0",
"sqlalchemy >= 1.4",
]

[project.optional-dependencies]
stable = [
"numpy == 1.21.6",
"pandas == 1.3.5",
"scipy == 1.7.3",
"sqlalchemy_utils",
"synthesized_datasets == 0.6",
db = [
"alembic",
"sqlalchemy >= 2.0",
"psycopg2 >= 2.9",
]

test = [
"pytest",
"pytest-cov",
]
dev = [
"pre-commit",
"black",
"mypy",
"pylint",
]
doc = [
"sphinx == 4.5.0",
"pydata-sphinx-theme == 0.8.1",
"sphinx-panels == 0.6.0",
"sphinx-autodoc-typehints == 1.18.1",
"sphinx-inline-tabs == 2022.1.2b11",
"sphinxcontrib-spelling == 7.3.2",
"ipython == 8.2.0",
"myst-parser == 0.17.2",
"sphinxcontrib.asciinema == 0.3.3",
]

[project.scripts]
insight-migrations = "insight.alembic.main:main"

[project.urls]
homepage = "https://github.com/synthesized-io/insight"
Expand Down
3 changes: 2 additions & 1 deletion src/insight/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from . import metrics, plot
from .check import ColumnCheck

__all__ = ['ColumnCheck']
__all__ = ['ColumnCheck', 'plot', 'metrics']
File renamed without changes.
4 changes: 4 additions & 0 deletions src/insight/alembic/alembic.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[alembic]
script_location = insight:alembic
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
sqlalchemy.url = postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DATABASE}
33 changes: 24 additions & 9 deletions migrations/env.py → src/insight/alembic/env.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import os
from logging.config import fileConfig

from alembic import context
from sqlalchemy import engine_from_config, pool
Expand All @@ -10,11 +9,6 @@
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
fileConfig(config.config_file_name)

target_metadata = Base.metadata

# other values from the config, defined by the needs of env.py,
Expand All @@ -35,7 +29,11 @@ def run_migrations_offline() -> None:
script output.
"""
url = config.get_main_option("sqlalchemy.url").format(**os.environ)
url = config.get_main_option("sqlalchemy.url")
if url is None:
raise ValueError("No sqlalchemy.url specified in config file")

url = url.format(**os.environ)
context.configure(
url=url,
target_metadata=target_metadata,
Expand All @@ -54,9 +52,26 @@ def run_migrations_online() -> None:
and associate a connection with the context.
"""
config.set_main_option("sqlalchemy.url", config.get_main_option("sqlalchemy.url").format(**os.environ))

POSTGRES_PASSWORD = os.environ["POSTGRES_PASSWORD"]
POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost")
POSTGRES_PORT = os.environ.get("POSTGRES_PORT", "5432")
POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres")
POSTGRES_DATABASE = os.environ.get("POSTGRES_DATABASE", "postgres")

url = config.get_main_option("sqlalchemy.url")
if url is None:
raise ValueError("No sqlalchemy.url specified in config file")

config.set_main_option("sqlalchemy.url", url.format(
POSTGRES_USER=POSTGRES_USER,
POSTGRES_PASSWORD=POSTGRES_PASSWORD,
POSTGRES_HOST=POSTGRES_HOST,
POSTGRES_PORT=POSTGRES_PORT,
POSTGRES_DATABASE=POSTGRES_DATABASE
))
connectable = engine_from_config(
config.get_section(config.config_ini_section),
config.get_section(config.config_ini_section) or {},
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
Expand Down
15 changes: 15 additions & 0 deletions src/insight/alembic/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Python script that will apply the migrations up to head
import os

import alembic.config

here = os.path.dirname(os.path.abspath(__file__))

alembic_args = [
'-c', os.path.join(here, 'alembic.ini'),
'upgrade', 'head'
]


def main():
alembic.config.main(argv=alembic_args)
File renamed without changes.
51 changes: 27 additions & 24 deletions src/insight/database/schema.py
Original file line number Diff line number Diff line change
@@ -1,47 +1,50 @@
from sqlalchemy import FLOAT, INTEGER, TIMESTAMP, VARCHAR, Column, ForeignKey
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy import FLOAT, INTEGER, TIMESTAMP, VARCHAR, ForeignKey
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
from sqlalchemy.sql import func

Base = declarative_base()

class Base(DeclarativeBase):
pass



class Dataset(Base):
__tablename__ = "dataset"

id = Column(INTEGER, primary_key=True)
name = Column(VARCHAR(50), nullable=False)
num_rows = Column(INTEGER)
num_columns = Column(INTEGER)
created_at = Column(TIMESTAMP, default=func.now())
id: Mapped[int] = mapped_column(INTEGER, primary_key=True)
name = mapped_column(VARCHAR(50), nullable=False)
num_rows = mapped_column(INTEGER)
num_columns = mapped_column(INTEGER)
created_at = mapped_column(TIMESTAMP, default=func.now())


class Metric(Base):
__tablename__ = "metric"

id = Column(INTEGER, primary_key=True)
name = Column(VARCHAR(50), nullable=False)
category = Column(VARCHAR(50))
created_at = Column(TIMESTAMP, default=func.now())
id = mapped_column(INTEGER, primary_key=True)
name = mapped_column(VARCHAR(50), nullable=False)
category = mapped_column(VARCHAR(50))
created_at = mapped_column(TIMESTAMP, default=func.now())


class Version(Base):
__tablename__ = "version"

id = Column(INTEGER, primary_key=True)
name = Column(VARCHAR(50), nullable=False, default="unversioned")
created_at = Column(TIMESTAMP, default=func.now())
id = mapped_column(INTEGER, primary_key=True)
name = mapped_column(VARCHAR(50), nullable=False, default="unversioned")
created_at = mapped_column(TIMESTAMP, default=func.now())


class Result(Base):
__tablename__ = "result"

id = Column(INTEGER, primary_key=True)
metric_id = Column(INTEGER, ForeignKey("metric.id"))
dataset_id = Column(INTEGER, ForeignKey("dataset.id"))
version_id = Column(INTEGER, ForeignKey("version.id"))
value = Column(FLOAT)
created_at = Column(TIMESTAMP, default=func.now())
id = mapped_column(INTEGER, primary_key=True)
metric_id = mapped_column(INTEGER, ForeignKey("metric.id"))
dataset_id = mapped_column(INTEGER, ForeignKey("dataset.id"))
version_id = mapped_column(INTEGER, ForeignKey("version.id"))
value = mapped_column(FLOAT)
created_at = mapped_column(TIMESTAMP, default=func.now())

metric: Metric = relationship("Metric")
dataset: Dataset = relationship("Dataset")
version: Version = relationship("Version")
metric: Mapped[Metric] = relationship("Metric")
dataset: Mapped[Dataset] = relationship("Dataset")
version: Mapped[Version] = relationship("Version")
13 changes: 12 additions & 1 deletion src/insight/database/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,19 @@ def get_session() -> typing.Optional[Session]:
"""
try:
POSTGRES_PASSWORD = os.environ["POSTGRES_PASSWORD"]
POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost")
POSTGRES_PORT = os.environ.get("POSTGRES_PORT", "5432")
POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres")
POSTGRES_DATABASE = os.environ.get("POSTGRES_DATABASE", "postgres")
db_url = "postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@" \
"{POSTGRES_HOST}:{POSTGRES_PORT}".format(**os.environ)
"{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DATABASE}".format(
POSTGRES_HOST=POSTGRES_HOST,
POSTGRES_PORT=POSTGRES_PORT,
POSTGRES_USER=POSTGRES_USER,
POSTGRES_PASSWORD=POSTGRES_PASSWORD,
POSTGRES_DATABASE=POSTGRES_DATABASE
)
engine = create_engine(db_url, future=True)

session_constructor = sessionmaker(bind=engine, future=True)
Expand Down
Binary file added src/insight/fonts/SourceSansPro-Regular.ttf
Binary file not shown.
Binary file removed src/insight/fonts/inter-v3-latin-regular.ttf
Binary file not shown.
Loading

0 comments on commit 69d43ba

Please sign in to comment.