diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ca7be88..8f4f29db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,16 @@ jobs: uses: codecov/codecov-action@v4.5.0 pytest-integration: + name: Integration test - ${{ matrix.dbName }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - dbName: "MySQL" + cliOption: "" + - dbName: "PostgreSQL" + cliOption: "--postgres" steps: - name: Checkout code uses: actions/checkout@v4 @@ -92,15 +101,15 @@ jobs: pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/ - name: Start demo run: | - git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts - ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml $PWD + git clone -b diracx-issue-147-postgres-integration https://github.com/AcquaDiGiorgio/diracx-charts.git ../diracx-charts + ../diracx-charts/run_demo.sh ${{ matrix.cliOption }} --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml $PWD - name: Debugging information run: | DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf export PATH=${DIRACX_DEMO_DIR}:$PATH kubectl get pods - for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do + for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|postgres|rabbitmq|opensearch)'); do echo "${pod_name}" kubectl describe pod/"${pod_name}" || true for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 6e8ec11c..407fea18 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -19,7 +19,10 @@ T = TypeVar("T") SqlalchemyDsn = Annotated[ - AnyUrl, UrlConstraints(allowed_schemes={"sqlite+aiosqlite", "mysql+aiomysql"}) + AnyUrl, + UrlConstraints( + allowed_schemes={"sqlite+aiosqlite", "mysql+aiomysql", "postgresql+asyncpg"} + ), ] diff --git a/diracx-db/pyproject.toml b/diracx-db/pyproject.toml index 37d728be..ba738159 100644 --- a/diracx-db/pyproject.toml +++ b/diracx-db/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "opensearch-py[async]", "pydantic >=2.4", "sqlalchemy[aiomysql,aiosqlite] >= 2", + "asyncpg", ] dynamic = ["version"] diff --git a/diracx-db/src/diracx/db/sql/dummy/db.py b/diracx-db/src/diracx/db/sql/dummy/db.py index 9a033163..23502043 100644 --- a/diracx-db/src/diracx/db/sql/dummy/db.py +++ b/diracx-db/src/diracx/db/sql/dummy/db.py @@ -40,7 +40,7 @@ async def insert_owner(self, name: str) -> int: stmt = insert(Owners).values(name=name) result = await self.conn.execute(stmt) # await self.engine.commit() - return result.lastrowid + return result.inserted_primary_key[0] async def insert_car(self, license_plate: UUID, model: str, owner_id: int) -> int: stmt = insert(Cars).values( @@ -49,4 +49,4 @@ async def insert_car(self, license_plate: UUID, model: str, owner_id: int) -> in result = await self.conn.execute(stmt) # await self.engine.commit() - return result.lastrowid + return result.inserted_primary_key[0] diff --git a/diracx-db/src/diracx/db/sql/job/db.py b/diracx-db/src/diracx/db/sql/job/db.py index 364c30b2..f680937b 100644 --- a/diracx-db/src/diracx/db/sql/job/db.py +++ b/diracx-db/src/diracx/db/sql/job/db.py @@ -115,7 +115,7 @@ async def _insertNewJDL(self, jdl) -> int: ) result = await self.conn.execute(stmt) # await self.engine.commit() - return result.lastrowid + return result.inserted_primary_key[0] async def _insertJob(self, jobData: dict[str, Any]): stmt = insert(Jobs).values(jobData) diff --git a/diracx-db/src/diracx/db/sql/job/schema.py b/diracx-db/src/diracx/db/sql/job/schema.py index 7f77cb15..b7baf359 100644 --- a/diracx-db/src/diracx/db/sql/job/schema.py +++ b/diracx-db/src/diracx/db/sql/job/schema.py @@ -22,7 +22,7 @@ class EnumBackedBool(types.TypeDecorator): cache_ok: bool = True def __init__(self) -> None: - super().__init__("True", "False") + super().__init__("True", "False", name="EnumBackedBool") def process_bind_param(self, value, dialect) -> str: if value is True: @@ -58,12 +58,12 @@ class Jobs(JobDBBase): Owner = Column("Owner", String(64), default="Unknown") OwnerGroup = Column("OwnerGroup", String(128), default="Unknown") VO = Column("VO", String(32)) - SubmissionTime = NullColumn("SubmissionTime", DateTime) - RescheduleTime = NullColumn("RescheduleTime", DateTime) - LastUpdateTime = NullColumn("LastUpdateTime", DateTime) - StartExecTime = NullColumn("StartExecTime", DateTime) - HeartBeatTime = NullColumn("HeartBeatTime", DateTime) - EndExecTime = NullColumn("EndExecTime", DateTime) + SubmissionTime = NullColumn("SubmissionTime", DateTime(timezone=True)) + RescheduleTime = NullColumn("RescheduleTime", DateTime(timezone=True)) + LastUpdateTime = NullColumn("LastUpdateTime", DateTime(timezone=True)) + StartExecTime = NullColumn("StartExecTime", DateTime(timezone=True)) + HeartBeatTime = NullColumn("HeartBeatTime", DateTime(timezone=True)) + EndExecTime = NullColumn("EndExecTime", DateTime(timezone=True)) Status = Column("Status", String(32), default="Received") MinorStatus = Column("MinorStatus", String(128), default="Unknown") ApplicationStatus = Column("ApplicationStatus", String(255), default="Unknown") @@ -72,7 +72,9 @@ class Jobs(JobDBBase): VerifiedFlag = Column("VerifiedFlag", EnumBackedBool(), default=False) # TODO: Should this be True/False/"Failed"? Or True/False/Null? AccountedFlag = Column( - "AccountedFlag", Enum("True", "False", "Failed"), default="False" + "AccountedFlag", + Enum("True", "False", "Failed", name="AccountedFlag"), + default="False", ) __table_args__ = ( @@ -138,7 +140,7 @@ class SiteMask(JobDBBase): __tablename__ = "SiteMask" Site = Column(String(64), primary_key=True) Status = Column(String(64)) - LastUpdateTime = Column(DateTime) + LastUpdateTime = Column(DateTime(timezone=True)) Author = Column(String(255)) Comment = Column(Text) @@ -146,7 +148,7 @@ class SiteMask(JobDBBase): class SiteMaskLogging(JobDBBase): __tablename__ = "SiteMaskLogging" Site = Column(String(64), primary_key=True) - UpdateTime = Column(DateTime, primary_key=True) + UpdateTime = Column(DateTime(timezone=True), primary_key=True) Status = Column(String(64)) Author = Column(String(255)) Comment = Column(Text) @@ -159,7 +161,7 @@ class HeartBeatLoggingInfo(JobDBBase): ) Name = Column(String(100), primary_key=True) Value = Column(Text) - HeartBeatTime = Column(DateTime, primary_key=True) + HeartBeatTime = Column(DateTime(timezone=True), primary_key=True) class JobCommands(JobDBBase): @@ -170,5 +172,5 @@ class JobCommands(JobDBBase): Command = Column(String(100)) Arguments = Column(String(100)) Status = Column(String(64), default="Received") - ReceptionTime = Column(DateTime, primary_key=True) - ExecutionTime = NullColumn(DateTime) + ReceptionTime = Column(DateTime(timezone=True), primary_key=True) + ExecutionTime = NullColumn(DateTime(timezone=True)) diff --git a/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py b/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py index db72a7f9..6e3ef6a2 100644 --- a/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py +++ b/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py @@ -32,7 +32,7 @@ async def upsert_owner(self, user: UserInfo) -> int: VO=user.vo, ) result = await self.conn.execute(stmt) - return result.lastrowid + return result.inserted_primary_key[0] @staticmethod def get_pfn(bucket_name: str, user: UserInfo, sandbox_info: SandboxInfo) -> str: diff --git a/diracx-db/src/diracx/db/sql/utils/__init__.py b/diracx-db/src/diracx/db/sql/utils/__init__.py index c514499b..e5b49bb2 100644 --- a/diracx-db/src/diracx/db/sql/utils/__init__.py +++ b/diracx-db/src/diracx/db/sql/utils/__init__.py @@ -125,7 +125,9 @@ def substract_date(**kwargs: float) -> datetime: def EnumColumn(enum_type, **kwargs): - return Column(Enum(enum_type, native_enum=False, length=16), **kwargs) + return Column( + Enum(enum_type, native_enum=False, length=16, name=str(enum_type)), **kwargs + ) class SQLDBError(Exception):