From 1dc7d4df97103775fdc91fd397ece74dcd4a08b0 Mon Sep 17 00:00:00 2001 From: Gyubong Lee Date: Thu, 5 Dec 2024 06:32:33 +0000 Subject: [PATCH 1/5] feat: Replace `vfolder`'s `status_history`'s type `dict` with `list` --- ...replace_vfolders_status_history_s_type_.py | 65 +++++++++++++++++++ src/ai/backend/manager/models/vfolder.py | 28 ++++---- 2 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py diff --git a/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py b/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py new file mode 100644 index 0000000000..dfe3da50ab --- /dev/null +++ b/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py @@ -0,0 +1,65 @@ +"""Replace vfolders status_history's type map with list +Revision ID: 786be66ef4e5 +Revises: 8c8e90aebacd +Create Date: 2024-05-07 05:10:23.799723 +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "786be66ef4e5" +down_revision = "8c8e90aebacd" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + WITH data AS ( + SELECT id, + (jsonb_each(status_history)).key AS status, + (jsonb_each(status_history)).value AS timestamp + FROM vfolders + ) + UPDATE vfolders + SET status_history = ( + SELECT jsonb_agg( + jsonb_build_object('status', status, 'timestamp', timestamp) + ) + FROM data + WHERE data.id = vfolders.id + ); + """ + ) + + op.execute("UPDATE vfolders SET status_history = '[]'::jsonb WHERE status_history IS NULL;") + op.alter_column( + "vfolders", + "status_history", + nullable=False, + default=[], + ) + + +def downgrade(): + op.execute( + """ + WITH data AS ( + SELECT id, + jsonb_object_agg( + elem->>'status', elem->>'timestamp' + ) AS new_status_history + FROM vfolders, + jsonb_array_elements(status_history) AS elem + GROUP BY id + ) + UPDATE vfolders + SET status_history = data.new_status_history + FROM data + WHERE data.id = vfolders.id; + """ + ) + + op.alter_column("vfolders", "status_history", nullable=True, default=None) + op.execute("UPDATE vfolders SET status_history = NULL WHERE status_history = '[]'::jsonb;") diff --git a/src/ai/backend/manager/models/vfolder.py b/src/ai/backend/manager/models/vfolder.py index 878706e88e..b9457f3ab0 100644 --- a/src/ai/backend/manager/models/vfolder.py +++ b/src/ai/backend/manager/models/vfolder.py @@ -100,7 +100,12 @@ from .storage import PermissionContext as StorageHostPermissionContext from .storage import PermissionContextBuilder as StorageHostPermissionContextBuilder from .user import UserRole, UserRow -from .utils import ExtendedAsyncSAEngine, execute_with_retry, execute_with_txn_retry, sql_json_merge +from .utils import ( + ExtendedAsyncSAEngine, + execute_with_retry, + execute_with_txn_retry, + sql_append_dict_to_list, +) if TYPE_CHECKING: from ..api.context import BackgroundTaskManager @@ -378,14 +383,14 @@ class VFolderCloneInfo(NamedTuple): nullable=False, index=True, ), - # status_history records the most recent status changes for each status + # status_history records the status changes of the vfolder. # e.g) - # { - # "ready": "2022-10-22T10:22:30", - # "delete-pending": "2022-10-22T11:40:30", - # "delete-ongoing": "2022-10-25T10:22:30" - # } - sa.Column("status_history", pgsql.JSONB(), nullable=True, default=sa.null()), + # [ + # {"status": "ready", "timestamp": "2022-10-22T10:22:30"}, + # {"status": "delete-pending", "timestamp": "2022-10-22T11:40:30"}, + # {"status": "delete-ongoing", "timestamp": "2022-10-25T10:22:30"} + # ] + sa.Column("status_history", pgsql.JSONB(), nullable=False, default=[]), sa.Column("status_changed", sa.DateTime(timezone=True), nullable=True, index=True), ) @@ -1063,12 +1068,9 @@ async def _update() -> None: values = { "status": update_status, "status_changed": now, - "status_history": sql_json_merge( + "status_history": sql_append_dict_to_list( VFolderRow.status_history, - (), - { - update_status.name: now.isoformat(), - }, + {"status": update_status.name, "timestamp": now.isoformat()}, ), } if update_status == VFolderOperationStatus.DELETE_ONGOING: From a9b24260b2f071a824ff1c4008a8d1f878ea1f99 Mon Sep 17 00:00:00 2001 From: Gyubong Lee Date: Thu, 5 Dec 2024 06:34:56 +0000 Subject: [PATCH 2/5] chore: Add news fragment --- changes/3205.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/3205.feature.md diff --git a/changes/3205.feature.md b/changes/3205.feature.md new file mode 100644 index 0000000000..09cf71d02e --- /dev/null +++ b/changes/3205.feature.md @@ -0,0 +1 @@ +Change the type of `vfolders.status_history` from a mapping of status and timestamps to a list of log entries containing status and timestamps, to preserve the log entries. \ No newline at end of file From 84a39fea67f9941b3dc6e57f18bdb173bca65625 Mon Sep 17 00:00:00 2001 From: Gyubong Lee Date: Fri, 6 Dec 2024 02:11:52 +0000 Subject: [PATCH 3/5] fix: Order by timestamp in migration script --- .../786be66ef4e5_replace_vfolders_status_history_s_type_.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py b/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py index dfe3da50ab..b0b7b62db1 100644 --- a/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py +++ b/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py @@ -26,6 +26,7 @@ def upgrade(): SET status_history = ( SELECT jsonb_agg( jsonb_build_object('status', status, 'timestamp', timestamp) + ORDER BY timestamp ) FROM data WHERE data.id = vfolders.id From 4e3e2e418727f3b4083faf157b4077e7205ce4f6 Mon Sep 17 00:00:00 2001 From: Gyubong Lee Date: Sun, 8 Dec 2024 06:45:24 +0000 Subject: [PATCH 4/5] fix: Add missing `status_history` GQL field --- src/ai/backend/manager/models/vfolder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ai/backend/manager/models/vfolder.py b/src/ai/backend/manager/models/vfolder.py index b9457f3ab0..0ec60874d3 100644 --- a/src/ai/backend/manager/models/vfolder.py +++ b/src/ai/backend/manager/models/vfolder.py @@ -1398,6 +1398,8 @@ class Meta: cloneable = graphene.Boolean() status = graphene.String() + status_history = graphene.JSONString(description="Added in 24.12.0.") + @classmethod def from_row(cls, ctx: GraphQueryContext, row: Row | VFolderRow) -> Optional[VirtualFolder]: if row is None: @@ -1432,6 +1434,7 @@ def _get_field(name: str) -> Any: cloneable=row["cloneable"], status=row["status"], cur_size=row["cur_size"], + status_history=row["status_history"], ) @classmethod @@ -1457,6 +1460,7 @@ def from_orm_row(cls, row: VFolderRow) -> VirtualFolder: cloneable=row.cloneable, status=row.status, cur_size=row.cur_size, + status_history=row.status_history, ) async def resolve_num_files(self, info: graphene.ResolveInfo) -> int: @@ -1519,6 +1523,7 @@ async def resolve_num_files(self, info: graphene.ResolveInfo) -> int: "cloneable": ("vfolders_cloneable", None), "status": ("vfolders_status", None), "cur_size": ("vfolders_cur_size", None), + "status_history": ("vfolders_status_history", None), } @classmethod From 0c53d122fedcfeee1ce35e456dfb53c0a8a09ea4 Mon Sep 17 00:00:00 2001 From: Gyubong Lee Date: Sun, 8 Dec 2024 06:49:41 +0000 Subject: [PATCH 5/5] chore: update GraphQL schema dump Co-authored-by: octodog --- src/ai/backend/manager/api/schema.graphql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ai/backend/manager/api/schema.graphql b/src/ai/backend/manager/api/schema.graphql index 6e8766a8b5..4a36e0fd7a 100644 --- a/src/ai/backend/manager/api/schema.graphql +++ b/src/ai/backend/manager/api/schema.graphql @@ -896,6 +896,9 @@ type VirtualFolder implements Item { cur_size: BigInt cloneable: Boolean status: String + + """Added in 24.12.0.""" + status_history: JSONString } type ComputeSession implements Item {