Issue 147 postgres integration

.github/workflows/main.yml

@@ -76,7 +76,16 @@ jobs:
         uses: codecov/codecov-action@v4.5.0
 
   pytest-integration:
+    name: Integration test - ${{ matrix.dbName }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - dbName: "MySQL"
+            cliOption: ""
+          - dbName: "PostgreSQL"
+            cliOption: "--postgres"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -92,15 +101,15 @@ jobs:
           pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/
       - name: Start demo
         run: |
-          git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
-          ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml $PWD
+          git clone -b diracx-issue-147-postgres-integration https://github.com/AcquaDiGiorgio/diracx-charts.git ../diracx-charts
+          ../diracx-charts/run_demo.sh ${{ matrix.cliOption }} --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml $PWD
       - name: Debugging information
         run: |
           DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
           export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
           export PATH=${DIRACX_DEMO_DIR}:$PATH
           kubectl get pods
-          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
+          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|postgres|rabbitmq|opensearch)'); do
             echo "${pod_name}"
             kubectl describe pod/"${pod_name}" || true
             for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do

diracx-core/src/diracx/core/settings.py

@@ -19,7 +19,10 @@
 T = TypeVar("T")
 
 SqlalchemyDsn = Annotated[
-    AnyUrl, UrlConstraints(allowed_schemes={"sqlite+aiosqlite", "mysql+aiomysql"})
+    AnyUrl,
+    UrlConstraints(
+        allowed_schemes={"sqlite+aiosqlite", "mysql+aiomysql", "postgresql+asyncpg"}
+    ),
 ]
 
 

diracx-db/pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
     "opensearch-py[async]",
     "pydantic >=2.4",
     "sqlalchemy[aiomysql,aiosqlite] >= 2",
+    "asyncpg",
 ]
 dynamic = ["version"]
 

diracx-db/src/diracx/db/sql/dummy/db.py

@@ -40,7 +40,7 @@ async def insert_owner(self, name: str) -> int:
         stmt = insert(Owners).values(name=name)
         result = await self.conn.execute(stmt)
         # await self.engine.commit()
-        return result.lastrowid
+        return result.inserted_primary_key[0]
 
     async def insert_car(self, license_plate: UUID, model: str, owner_id: int) -> int:
         stmt = insert(Cars).values(
@@ -49,4 +49,4 @@ async def insert_car(self, license_plate: UUID, model: str, owner_id: int) -> in
 
         result = await self.conn.execute(stmt)
         # await self.engine.commit()
-        return result.lastrowid
+        return result.inserted_primary_key[0]

diracx-db/src/diracx/db/sql/job/db.py

@@ -115,7 +115,7 @@ async def _insertNewJDL(self, jdl) -> int:
         )
         result = await self.conn.execute(stmt)
         # await self.engine.commit()
-        return result.lastrowid
+        return result.inserted_primary_key[0]
 
     async def _insertJob(self, jobData: dict[str, Any]):
         stmt = insert(Jobs).values(jobData)

diracx-db/src/diracx/db/sql/job/schema.py

@@ -22,7 +22,7 @@ class EnumBackedBool(types.TypeDecorator):
     cache_ok: bool = True
 
     def __init__(self) -> None:
-        super().__init__("True", "False")
+        super().__init__("True", "False", name="EnumBackedBool")
 
     def process_bind_param(self, value, dialect) -> str:
         if value is True:
@@ -58,12 +58,12 @@ class Jobs(JobDBBase):
     Owner = Column("Owner", String(64), default="Unknown")
     OwnerGroup = Column("OwnerGroup", String(128), default="Unknown")
     VO = Column("VO", String(32))
-    SubmissionTime = NullColumn("SubmissionTime", DateTime)
-    RescheduleTime = NullColumn("RescheduleTime", DateTime)
-    LastUpdateTime = NullColumn("LastUpdateTime", DateTime)
-    StartExecTime = NullColumn("StartExecTime", DateTime)
-    HeartBeatTime = NullColumn("HeartBeatTime", DateTime)
-    EndExecTime = NullColumn("EndExecTime", DateTime)
+    SubmissionTime = NullColumn("SubmissionTime", DateTime(timezone=True))
+    RescheduleTime = NullColumn("RescheduleTime", DateTime(timezone=True))
+    LastUpdateTime = NullColumn("LastUpdateTime", DateTime(timezone=True))
+    StartExecTime = NullColumn("StartExecTime", DateTime(timezone=True))
+    HeartBeatTime = NullColumn("HeartBeatTime", DateTime(timezone=True))
+    EndExecTime = NullColumn("EndExecTime", DateTime(timezone=True))
     Status = Column("Status", String(32), default="Received")
     MinorStatus = Column("MinorStatus", String(128), default="Unknown")
     ApplicationStatus = Column("ApplicationStatus", String(255), default="Unknown")
@@ -72,7 +72,9 @@ class Jobs(JobDBBase):
     VerifiedFlag = Column("VerifiedFlag", EnumBackedBool(), default=False)
     # TODO: Should this be True/False/"Failed"? Or True/False/Null?
     AccountedFlag = Column(
-        "AccountedFlag", Enum("True", "False", "Failed"), default="False"
+        "AccountedFlag",
+        Enum("True", "False", "Failed", name="AccountedFlag"),
+        default="False",
     )
 
     __table_args__ = (
@@ -138,15 +140,15 @@ class SiteMask(JobDBBase):
     __tablename__ = "SiteMask"
     Site = Column(String(64), primary_key=True)
     Status = Column(String(64))
-    LastUpdateTime = Column(DateTime)
+    LastUpdateTime = Column(DateTime(timezone=True))
     Author = Column(String(255))
     Comment = Column(Text)
 
 
 class SiteMaskLogging(JobDBBase):
     __tablename__ = "SiteMaskLogging"
     Site = Column(String(64), primary_key=True)
-    UpdateTime = Column(DateTime, primary_key=True)
+    UpdateTime = Column(DateTime(timezone=True), primary_key=True)
     Status = Column(String(64))
     Author = Column(String(255))
     Comment = Column(Text)
@@ -159,7 +161,7 @@ class HeartBeatLoggingInfo(JobDBBase):
     )
     Name = Column(String(100), primary_key=True)
     Value = Column(Text)
-    HeartBeatTime = Column(DateTime, primary_key=True)
+    HeartBeatTime = Column(DateTime(timezone=True), primary_key=True)
 
 
 class JobCommands(JobDBBase):
@@ -170,5 +172,5 @@ class JobCommands(JobDBBase):
     Command = Column(String(100))
     Arguments = Column(String(100))
     Status = Column(String(64), default="Received")
-    ReceptionTime = Column(DateTime, primary_key=True)
-    ExecutionTime = NullColumn(DateTime)
+    ReceptionTime = Column(DateTime(timezone=True), primary_key=True)
+    ExecutionTime = NullColumn(DateTime(timezone=True))

diracx-db/src/diracx/db/sql/sandbox_metadata/db.py

@@ -32,7 +32,7 @@ async def upsert_owner(self, user: UserInfo) -> int:
             VO=user.vo,
         )
         result = await self.conn.execute(stmt)
-        return result.lastrowid
+        return result.inserted_primary_key[0]
 
     @staticmethod
     def get_pfn(bucket_name: str, user: UserInfo, sandbox_info: SandboxInfo) -> str:

diracx-db/src/diracx/db/sql/utils/__init__.py

@@ -125,7 +125,9 @@ def substract_date(**kwargs: float) -> datetime:
 
 
 def EnumColumn(enum_type, **kwargs):
-    return Column(Enum(enum_type, native_enum=False, length=16), **kwargs)
+    return Column(
+        Enum(enum_type, native_enum=False, length=16, name=str(enum_type)), **kwargs
+    )
 
 
 class SQLDBError(Exception):