Update to sqlalchemy 2.0 (#304)

* update sqlalchemy to 2.0

* suppress pylint errror until upstream fix

* change to DeclarativeBase

* Changes for sqlalchemy 2.0

* add a database migration

* remove unused ignores

* updates tests

* update vscode settings for dev container

* update from query to execute api

device/.devcontainer/devcontainer.json

@@ -23,20 +23,20 @@
 		"vscode": {
 			"extensions": [
 				"ms-python.python",
+				"ms-python.pylint",
+				"ms-python.flake8",
+				"matangover.mypy",
+				"ms-python.black-formatter",
 				"eamodio.gitlens",
 				"ms-python.vscode-pylance",
 				"github.copilot"
 			],
 			"postCreateCommand": "git config --global core.autocrlf true && git config --global user.email 'nelsonstoik@gmail.com' && git config --global user.name 'Nelson'",
 			"settings": {
-				"python.linting.enabled": true,
-				"python.linting.pylintEnabled": true,
-				"python.linting.pylintArgs": ["--load-plugins"],
-				"python.linting.flake8Enabled": true,
-				"python.linting.flake8Args": ["--config=/workspaces/device/setup.cfg"],
-				"python.linting.mypyEnabled": true,
-				"python.linting.mypyArgs": ["--config=/workspaces/device/setup.cfg"],
-				"python.formatting.provider": "black"
+				"pylint.args": ["--load-plugins"],
+				"flake8.args": ["--config=/workspaces/device/setup.cfg"],
+				"mypy.configFile": "/workspaces/device/setup.cfg",
+				"mypy.runUsingActiveInterpreter": true
 			}
 		}
 	}

device/Pipfile

@@ -11,7 +11,7 @@ pytest-cov = "~=4.1"
 pytest-mock = "~=3.12"
 # Lint and code style
 black = "~=23.12"
-flake8 = "~=6.1.0"
+flake8 = "~=7.0"
 flake8-blind-except = "~=0.2.0"
 flake8-debugger = "~=4.1.2"
 flake8-docstrings = "~=1.7.0"
@@ -27,7 +27,7 @@ types-psutil = "~=5.9.5"
 types-requests = "~=2.31"
 
 [packages]
-SQLAlchemy = "~=1.4.42"
+SQLAlchemy = "~=2.0"
 # for Postgresql
 psycopg2 = "~=2.9.4"
 alembic = "~=1.13.1"

device/README.md

@@ -1,15 +1,22 @@
-# FD Device Devive repo
+# FD Device Device repo
 
 ## Vscode Dev Container
 
-Run ```pipenv install --dev``` inside the container after setting it up
+When the dev container is first built, you will need to configure the virtual environment inside the container after setting it up
 
 ```bash
 cd device
 pipenv install --dev
 ```
 
+
+Then you may be prompted to select a python interpreter and/or reload the window for some of the python extensions to work properly. This only needs to be done when the dev container is rebuilt.
+
+The installation of the dependencies and the setup of the virtual environment could be done automatically using a dockerfile step, but it is commented out to help speed up the dev container build time.
+
+
 ## Commands
+The available app commands are:
 - ```pipenv shell``` - start the virtual environment
 - ```pipenv install``` - install dependencies
 - ```pipenv install --dev``` - install dev dependencies

device/fd_device/cli/db/commands.py

@@ -3,10 +3,10 @@
 from alembic import command as al_command
 from alembic.config import Config as AlConfig
 
-from fd_device.database.base import (
+from fd_device.database.database import (
+    Base,
     create_all_tables,
     drop_all_tables,
-    get_base,
     get_session,
 )
 
@@ -39,9 +39,8 @@ def delete_all_data(confirm):
     else:
         click.echo("deleting all data from the database.")
 
-        base = get_base()
         session = get_session()
-        for table in reversed(base.metadata.sorted_tables):
+        for table in reversed(Base.metadata.sorted_tables):
             session.execute(table.delete())
         session.commit()
 

device/fd_device/cli/manage/setup_commands.py

@@ -2,9 +2,10 @@
 from datetime import datetime
 
 import click
+from sqlalchemy import select
 from sqlalchemy.orm.exc import NoResultFound
 
-from fd_device.database.base import get_session
+from fd_device.database.database import get_session
 from fd_device.database.device import Device, Grainbin
 from fd_device.database.system import Hardware, Software, SystemSetup
 from fd_device.device.temperature import get_connected_sensors
@@ -34,7 +35,7 @@ def first_setup(standalone):  # noqa: C901
     session = get_session()
 
     try:
-        system = session.query(SystemSetup).one()
+        system = session.scalars(select(SystemSetup)).one()
     except NoResultFound:
         system = SystemSetup()
         session.add(system)
@@ -147,14 +148,14 @@ def initialize_device():
     session = get_session()
 
     try:
-        hd = session.query(Hardware).one()
-        sd = session.query(Software).one()
+        hd = session.scalars(select(Hardware)).one()
+        sd = session.scalars(select(Software)).one()
     except NoResultFound:
         session.close()
         return
 
     try:
-        device = session.query(Device).one()
+        device = session.scalars(select(Device)).one()
     except NoResultFound:
         device = Device(
             device_id=hd.serial_number,

device/fd_device/database/database.py

@@ -1,49 +1,81 @@
 # -*- coding: utf-8 -*-
 """Database module, including the SQLAlchemy database object and DB-related utilities."""
-from sqlalchemy import Column, ForeignKey, Integer
+from sqlalchemy import ForeignKey, String, create_engine
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    Mapped,
+    mapped_column,
+    scoped_session,
+    sessionmaker,
+)
+from sqlalchemy.orm.session import Session
+from typing_extensions import Annotated
 
-from fd_device.database.base import get_base, get_session
+from ..settings import get_config
 
-Base = get_base(with_query=True)
+config = get_config()  # pylint: disable=invalid-name
+engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
+db_session = scoped_session(sessionmaker(bind=engine))
+
+# special types for SQLAlchemy Base class below
+str20 = Annotated[str, 20]  # pylint: disable=invalid-name
+str7 = Annotated[str, 7]  # pylint: disable=invalid-name
+
+
+def get_session() -> Session:
+    """Return the sqlalchemy db_session."""
+
+    return db_session()
+
+
+def create_all_tables():
+    """Create all tables."""
+    Base.metadata.create_all(bind=engine)
+
+
+def drop_all_tables():
+    """Drop all tables."""
+    Base.metadata.drop_all(bind=engine)
+
+
+class Base(DeclarativeBase):  # pylint: disable=too-few-public-methods
+    """Base class for SQLAlchemy model definitions."""
+
+    type_annontation_map = {
+        str20: String(20),
+        str7: String(7),
+    }
 
 
 class CRUDMixin:
     """Mixin that adds convenience methods for CRUD (create, read, update, delete) operations."""
 
     @classmethod
-    def create(cls, session=None, **kwargs):
+    def create(cls, **kwargs):
         """Create a new record and save it the database."""
-        if session is None:
-            session = get_session()
         instance = cls(**kwargs)
-        return instance.save(session)
+        return instance.save()
 
-    def update(self, session=None, commit=True, **kwargs):
+    def update(self, commit=True, **kwargs):
         """Update specific fields of a record."""
-        if session is None:
-            session = get_session()
         for attr, value in kwargs.items():
             setattr(self, attr, value)
-        return self.save(session) if commit else self
+        return self.save() if commit else self
 
-    def save(self, session=None, commit=True):
+    def save(self, commit=True):
         """Save the record."""
-        if session is None:
-            session = get_session()
-        session.add(self)
+        db_session.add(self)
         if commit:
-            session.commit()
+            db_session.commit()
         return self
 
-    def delete(self, session=None, commit=True):
+    def delete(self, commit=True):
         """Remove the record from the database."""
-        if session is None:
-            session = get_session()
-        session.delete(self)
-        return commit and session.commit()
+        db_session.delete(self)
+        return commit and db_session.commit()
 
 
-class Model(CRUDMixin, Base):  # type: ignore[valid-type, misc]
+class Model(CRUDMixin, Base):
     """Base model class that includes CRUD convenience methods."""
 
     __abstract__ = True
@@ -57,30 +89,29 @@ class SurrogatePK(Model):  # pylint: disable=too-few-public-methods
     __table_args__ = {"extend_existing": True}
     __abstract__ = True
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(primary_key=True)
 
     @classmethod
-    def get_by_id(cls, record_id, session=None):
+    def get_by_id(cls, record_id):
         """Get record by ID."""
         if any(
             (
                 isinstance(record_id, (str, bytes)) and record_id.isdigit(),
                 isinstance(record_id, (int, float)),
             ),
         ):
-            if session:
-                return session.query(cls).get(int(record_id))
-
-            return cls.query.get(int(record_id))
+            return db_session.get(cls, int(record_id))
         return None
 
 
-def reference_col(tablename, nullable=False, pk_name="id", **kwargs):
+def reference_col(tablename, nullable=False, pk_name="id", **kwargs) -> Mapped[int]:
     """Column that adds primary key foreign key reference.
 
     Usage: ::
 
-        category_id = reference_col('category')
-        category = relationship('Category', backref='categories')
+        category_id: Mapped[ForeignKey] = reference_col('category')
+        categorys: Mapped[List["Category"]] = relationship(backref='categories')
     """
-    return Column(ForeignKey(f"{tablename}.{pk_name}"), nullable=nullable, **kwargs)
+    return mapped_column(
+        ForeignKey(f"{tablename}.{pk_name}"), nullable=nullable, **kwargs
+    )