diff --git a/changes/3205.feature.md b/changes/3205.feature.md new file mode 100644 index 0000000000..09cf71d02e --- /dev/null +++ b/changes/3205.feature.md @@ -0,0 +1 @@ +Change the type of `vfolders.status_history` from a mapping of status and timestamps to a list of log entries containing status and timestamps, to preserve the log entries. \ No newline at end of file diff --git a/src/ai/backend/manager/api/schema.graphql b/src/ai/backend/manager/api/schema.graphql index 6e8766a8b5..4a36e0fd7a 100644 --- a/src/ai/backend/manager/api/schema.graphql +++ b/src/ai/backend/manager/api/schema.graphql @@ -896,6 +896,9 @@ type VirtualFolder implements Item { cur_size: BigInt cloneable: Boolean status: String + + """Added in 24.12.0.""" + status_history: JSONString } type ComputeSession implements Item { diff --git a/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py b/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py new file mode 100644 index 0000000000..b0b7b62db1 --- /dev/null +++ b/src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py @@ -0,0 +1,66 @@ +"""Replace vfolders status_history's type map with list +Revision ID: 786be66ef4e5 +Revises: 8c8e90aebacd +Create Date: 2024-05-07 05:10:23.799723 +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "786be66ef4e5" +down_revision = "8c8e90aebacd" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + WITH data AS ( + SELECT id, + (jsonb_each(status_history)).key AS status, + (jsonb_each(status_history)).value AS timestamp + FROM vfolders + ) + UPDATE vfolders + SET status_history = ( + SELECT jsonb_agg( + jsonb_build_object('status', status, 'timestamp', timestamp) + ORDER BY timestamp + ) + FROM data + WHERE data.id = vfolders.id + ); + """ + ) + + op.execute("UPDATE vfolders SET status_history = '[]'::jsonb WHERE status_history IS NULL;") + op.alter_column( + "vfolders", + "status_history", + nullable=False, + default=[], + ) + + +def downgrade(): + op.execute( + """ + WITH data AS ( + SELECT id, + jsonb_object_agg( + elem->>'status', elem->>'timestamp' + ) AS new_status_history + FROM vfolders, + jsonb_array_elements(status_history) AS elem + GROUP BY id + ) + UPDATE vfolders + SET status_history = data.new_status_history + FROM data + WHERE data.id = vfolders.id; + """ + ) + + op.alter_column("vfolders", "status_history", nullable=True, default=None) + op.execute("UPDATE vfolders SET status_history = NULL WHERE status_history = '[]'::jsonb;") diff --git a/src/ai/backend/manager/models/vfolder.py b/src/ai/backend/manager/models/vfolder.py index 878706e88e..0ec60874d3 100644 --- a/src/ai/backend/manager/models/vfolder.py +++ b/src/ai/backend/manager/models/vfolder.py @@ -100,7 +100,12 @@ from .storage import PermissionContext as StorageHostPermissionContext from .storage import PermissionContextBuilder as StorageHostPermissionContextBuilder from .user import UserRole, UserRow -from .utils import ExtendedAsyncSAEngine, execute_with_retry, execute_with_txn_retry, sql_json_merge +from .utils import ( + ExtendedAsyncSAEngine, + execute_with_retry, + execute_with_txn_retry, + sql_append_dict_to_list, +) if TYPE_CHECKING: from ..api.context import BackgroundTaskManager @@ -378,14 +383,14 @@ class VFolderCloneInfo(NamedTuple): nullable=False, index=True, ), - # status_history records the most recent status changes for each status + # status_history records the status changes of the vfolder. # e.g) - # { - # "ready": "2022-10-22T10:22:30", - # "delete-pending": "2022-10-22T11:40:30", - # "delete-ongoing": "2022-10-25T10:22:30" - # } - sa.Column("status_history", pgsql.JSONB(), nullable=True, default=sa.null()), + # [ + # {"status": "ready", "timestamp": "2022-10-22T10:22:30"}, + # {"status": "delete-pending", "timestamp": "2022-10-22T11:40:30"}, + # {"status": "delete-ongoing", "timestamp": "2022-10-25T10:22:30"} + # ] + sa.Column("status_history", pgsql.JSONB(), nullable=False, default=[]), sa.Column("status_changed", sa.DateTime(timezone=True), nullable=True, index=True), ) @@ -1063,12 +1068,9 @@ async def _update() -> None: values = { "status": update_status, "status_changed": now, - "status_history": sql_json_merge( + "status_history": sql_append_dict_to_list( VFolderRow.status_history, - (), - { - update_status.name: now.isoformat(), - }, + {"status": update_status.name, "timestamp": now.isoformat()}, ), } if update_status == VFolderOperationStatus.DELETE_ONGOING: @@ -1396,6 +1398,8 @@ class Meta: cloneable = graphene.Boolean() status = graphene.String() + status_history = graphene.JSONString(description="Added in 24.12.0.") + @classmethod def from_row(cls, ctx: GraphQueryContext, row: Row | VFolderRow) -> Optional[VirtualFolder]: if row is None: @@ -1430,6 +1434,7 @@ def _get_field(name: str) -> Any: cloneable=row["cloneable"], status=row["status"], cur_size=row["cur_size"], + status_history=row["status_history"], ) @classmethod @@ -1455,6 +1460,7 @@ def from_orm_row(cls, row: VFolderRow) -> VirtualFolder: cloneable=row.cloneable, status=row.status, cur_size=row.cur_size, + status_history=row.status_history, ) async def resolve_num_files(self, info: graphene.ResolveInfo) -> int: @@ -1517,6 +1523,7 @@ async def resolve_num_files(self, info: graphene.ResolveInfo) -> int: "cloneable": ("vfolders_cloneable", None), "status": ("vfolders_status", None), "cur_size": ("vfolders_cur_size", None), + "status_history": ("vfolders_status_history", None), } @classmethod