From d36eab9bcc54c829ccdbf0b261538e4318e15683 Mon Sep 17 00:00:00 2001 From: maslow Date: Fri, 31 Mar 2023 03:17:17 +0800 Subject: [PATCH] fix(server): fix tasks state handler logic (#987) --- .../application/application-task.service.ts | 55 ++------ .../src/gateway/bucket-domain-task.service.ts | 102 ++++---------- .../gateway/runtime-domain-task.service.ts | 87 ++++-------- server/src/gateway/website-task.service.ts | 5 +- server/src/instance/instance-task.service.ts | 127 ++++-------------- server/src/instance/instance.service.ts | 4 +- server/src/storage/bucket-task.service.ts | 18 ++- server/src/trigger/trigger-task.service.ts | 6 +- 8 files changed, 110 insertions(+), 294 deletions(-) diff --git a/server/src/application/application-task.service.ts b/server/src/application/application-task.service.ts index 11feb72d5c..ab824238c8 100644 --- a/server/src/application/application-task.service.ts +++ b/server/src/application/application-task.service.ts @@ -58,8 +58,6 @@ export class ApplicationTaskService { // State `Deleted` this.handleDeletedState() - - this.clearTimeoutLocks() } /** @@ -78,20 +76,12 @@ export class ApplicationTaskService { .findOneAndUpdate( { phase: ApplicationPhase.Creating, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) - if (!res.value) { - return - } + if (!res.value) return const app = res.value const appid = app.appid @@ -145,11 +135,8 @@ export class ApplicationTaskService { } // update application phase to `Created` - const updated = await db.collection('Application').updateOne( - { - _id: app._id, - phase: ApplicationPhase.Creating, - }, + await db.collection('Application').updateOne( + { _id: app._id, phase: ApplicationPhase.Creating }, { $set: { phase: ApplicationPhase.Created, @@ -157,8 +144,8 @@ export class ApplicationTaskService { }, }, ) - if (updated.modifiedCount > 0) - this.logger.debug('app phase updated to `Created`:', app.appid) + + this.logger.log('app phase updated to `Created`: ' + app.appid) } /** @@ -275,11 +262,8 @@ export class ApplicationTaskService { } // update phase to `Deleted` - const updated = await db.collection('Application').updateOne( - { - _id: app._id, - phase: ApplicationPhase.Deleting, - }, + await db.collection('Application').updateOne( + { _id: app._id, phase: ApplicationPhase.Deleting }, { $set: { phase: ApplicationPhase.Deleted, @@ -287,8 +271,8 @@ export class ApplicationTaskService { }, }, ) - if (updated.modifiedCount > 0) - this.logger.debug('app phase updated to `Deleted`:', app.appid) + + this.logger.log('app phase updated to `Deleted`: ' + app.appid) } /** @@ -319,8 +303,7 @@ export class ApplicationTaskService { ) await db.collection('Application').deleteMany({ - // TODO: support reset app or not? keep this line now. - // state: ApplicationState.Deleted, + state: ApplicationState.Deleted, phase: ApplicationPhase.Deleted, }) } @@ -334,18 +317,4 @@ export class ApplicationTaskService { .collection('Application') .updateOne({ appid: appid }, { $set: { lockedAt: TASK_LOCK_INIT_TIME } }) } - - /** - * Clear timeout locks - */ - async clearTimeoutLocks() { - const db = SystemDatabase.db - - await db - .collection('Application') - .updateMany( - { lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) } }, - { $set: { lockedAt: TASK_LOCK_INIT_TIME } }, - ) - } } diff --git a/server/src/gateway/bucket-domain-task.service.ts b/server/src/gateway/bucket-domain-task.service.ts index ed6b63c354..9d70a613a9 100644 --- a/server/src/gateway/bucket-domain-task.service.ts +++ b/server/src/gateway/bucket-domain-task.service.ts @@ -4,7 +4,6 @@ import { RegionService } from 'src/region/region.service' import { ApisixService } from './apisix.service' import * as assert from 'node:assert' import { Cron, CronExpression } from '@nestjs/schedule' -import { times } from 'lodash' import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants' import { SystemDatabase } from 'src/database/system-database' @@ -26,10 +25,10 @@ export class BucketDomainTaskService { } // Phase `Creating` -> `Created` - times(this.concurrency, () => this.handleCreatingPhase()) + this.handleCreatingPhase() // Phase `Deleting` -> `Deleted` - times(this.concurrency, () => this.handleDeletingPhase()) + this.handleDeletingPhase() // Phase `Created` -> `Deleting` this.handleInactiveState() @@ -39,9 +38,6 @@ export class BucketDomainTaskService { // Phase `Deleting` -> `Deleted` this.handleDeletedState() - - // Clear timeout locks - this.clearTimeoutLocks() } /** @@ -57,15 +53,9 @@ export class BucketDomainTaskService { .findOneAndUpdate( { phase: DomainPhase.Creating, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return @@ -85,15 +75,9 @@ export class BucketDomainTaskService { // update phase to `Created` const updated = await db.collection('BucketDomain').updateOne( + { _id: doc._id, phase: DomainPhase.Creating }, { - _id: doc._id, - phase: DomainPhase.Creating, - }, - { - $set: { - phase: DomainPhase.Created, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Created, lockedAt: TASK_LOCK_INIT_TIME }, }, ) @@ -114,15 +98,9 @@ export class BucketDomainTaskService { .findOneAndUpdate( { phase: DomainPhase.Deleting, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return @@ -131,25 +109,19 @@ export class BucketDomainTaskService { const region = await this.regionService.findByAppId(doc.appid) assert(region, 'region not found') - // delete route first - const route = await this.apisixService.deleteBucketRoute( - region, - doc.bucketName, - ) - - this.logger.debug('bucket route deleted:', route) + // delete route if exists + const id = `bucket-${doc.bucketName}` + const route = await this.apisixService.getRoute(region, id) + if (route) { + await this.apisixService.deleteBucketRoute(region, doc.bucketName) + this.logger.log('bucket route deleted: ' + doc.bucketName) + } // update phase to `Deleted` const updated = await db.collection('BucketDomain').updateOne( + { _id: doc._id, phase: DomainPhase.Deleting }, { - _id: doc._id, - phase: DomainPhase.Deleting, - }, - { - $set: { - phase: DomainPhase.Deleted, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Deleted, lockedAt: TASK_LOCK_INIT_TIME }, }, ) @@ -168,12 +140,10 @@ export class BucketDomainTaskService { { state: DomainState.Active, phase: DomainPhase.Deleted, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { - $set: { - phase: DomainPhase.Creating, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Creating, lockedAt: TASK_LOCK_INIT_TIME }, }, ) } @@ -188,13 +158,11 @@ export class BucketDomainTaskService { await db.collection('BucketDomain').updateMany( { state: DomainState.Inactive, - phase: DomainPhase.Created, + phase: { $in: [DomainPhase.Created, DomainPhase.Creating] }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { - $set: { - phase: DomainPhase.Deleting, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, }, ) } @@ -211,12 +179,10 @@ export class BucketDomainTaskService { { state: DomainState.Deleted, phase: DomainPhase.Created, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { - $set: { - phase: DomainPhase.Deleting, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, }, ) @@ -225,24 +191,4 @@ export class BucketDomainTaskService { phase: DomainPhase.Deleted, }) } - - /** - * Clear timeout locks - */ - async clearTimeoutLocks() { - const db = SystemDatabase.db - - await db.collection('BucketDomain').updateMany( - { - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: TASK_LOCK_INIT_TIME, - }, - }, - ) - } } diff --git a/server/src/gateway/runtime-domain-task.service.ts b/server/src/gateway/runtime-domain-task.service.ts index 2f2fa4a09f..afcbd2a27f 100644 --- a/server/src/gateway/runtime-domain-task.service.ts +++ b/server/src/gateway/runtime-domain-task.service.ts @@ -37,9 +37,6 @@ export class RuntimeDomainTaskService { // Phase `Deleting` -> `Deleted` this.handleDeletedState() - - // Clear timeout locks - this.clearTimeoutLocks() } /** @@ -117,15 +114,9 @@ export class RuntimeDomainTaskService { .findOneAndUpdate( { phase: DomainPhase.Deleting, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return @@ -134,28 +125,24 @@ export class RuntimeDomainTaskService { const region = await this.regionService.findByAppId(doc.appid) assert(region, 'region not found') - // delete route first - const route = await this.apisixService.deleteAppRoute(region, doc.appid) - this.logger.debug('app route deleted:', route) + // delete route first if exists + const id = `app-${doc.appid}` + const route = await this.apisixService.getRoute(region, id) + if (route) { + await this.apisixService.deleteAppRoute(region, doc.appid) + this.logger.log('app route deleted: ' + doc.appid) + this.logger.debug(route) + } // update phase to `Deleted` - const updated = await db - .collection('RuntimeDomain') - .updateOne( - { - _id: doc._id, - phase: DomainPhase.Deleting, - }, - { - $set: { - phase: DomainPhase.Deleted, - lockedAt: TASK_LOCK_INIT_TIME, - }, - }, - ) + await db.collection('RuntimeDomain').updateOne( + { _id: doc._id, phase: DomainPhase.Deleting }, + { + $set: { phase: DomainPhase.Deleted, lockedAt: TASK_LOCK_INIT_TIME }, + }, + ) - if (updated.modifiedCount > 0) - this.logger.debug('app domain phase updated to Deleted', doc) + this.logger.log('app domain phase updated to Deleted: ' + doc.appid) } /** @@ -169,12 +156,10 @@ export class RuntimeDomainTaskService { { state: DomainState.Active, phase: DomainPhase.Deleted, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { - $set: { - phase: DomainPhase.Creating, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Creating, lockedAt: TASK_LOCK_INIT_TIME }, }, ) } @@ -189,13 +174,11 @@ export class RuntimeDomainTaskService { await db.collection('RuntimeDomain').updateMany( { state: DomainState.Inactive, - phase: DomainPhase.Created, + phase: { $in: [DomainPhase.Created, DomainPhase.Creating] }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { - $set: { - phase: DomainPhase.Deleting, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, }, ) } @@ -212,12 +195,10 @@ export class RuntimeDomainTaskService { { state: DomainState.Deleted, phase: DomainPhase.Created, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { - $set: { - phase: DomainPhase.Deleting, - lockedAt: TASK_LOCK_INIT_TIME, - }, + $set: { phase: DomainPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, }, ) @@ -226,24 +207,4 @@ export class RuntimeDomainTaskService { phase: DomainPhase.Deleted, }) } - - /** - * Clear timeout locks - */ - async clearTimeoutLocks() { - const db = SystemDatabase.db - - await db.collection('RuntimeDomain').updateMany( - { - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: TASK_LOCK_INIT_TIME, - }, - }, - ) - } } diff --git a/server/src/gateway/website-task.service.ts b/server/src/gateway/website-task.service.ts index 46a91e9ef0..f2ff5dc894 100644 --- a/server/src/gateway/website-task.service.ts +++ b/server/src/gateway/website-task.service.ts @@ -237,7 +237,8 @@ export class WebsiteTaskService { await db.collection('WebsiteHosting').updateMany( { state: DomainState.Inactive, - phase: DomainPhase.Created, + phase: { $in: [DomainPhase.Created, DomainPhase.Creating] }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { $set: { @@ -260,6 +261,7 @@ export class WebsiteTaskService { { state: DomainState.Active, phase: DomainPhase.Deleted, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { $set: { @@ -283,6 +285,7 @@ export class WebsiteTaskService { { state: DomainState.Deleted, phase: DomainPhase.Created, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { $set: { diff --git a/server/src/instance/instance-task.service.ts b/server/src/instance/instance-task.service.ts index b86d4e765b..230dd77ad2 100644 --- a/server/src/instance/instance-task.service.ts +++ b/server/src/instance/instance-task.service.ts @@ -62,34 +62,16 @@ export class InstanceTaskService { /** * State `Running`: - * - create instance * - move phase `Created` or `Stopped` to `Starting` */ async handleRunningState() { const db = SystemDatabase.db - const res = await db - .collection('Application') - .findOneAndUpdate( - { - state: ApplicationState.Running, - phase: { $in: [ApplicationPhase.Created, ApplicationPhase.Stopped] }, - lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, - }, - { $set: { lockedAt: new Date() } }, - ) - - if (!res.value) return - const app = res.value - - // create instance - await this.instanceService.create(app) - - // update phase to `Starting` - await db.collection('Application').updateOne( + await db.collection('Application').updateMany( { - appid: app.appid, + state: ApplicationState.Running, phase: { $in: [ApplicationPhase.Created, ApplicationPhase.Stopped] }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { $set: { @@ -99,8 +81,6 @@ export class InstanceTaskService { }, }, ) - - this.logger.log(`Application ${app.appid} updated to phase starting`) } /** @@ -124,15 +104,14 @@ export class InstanceTaskService { if (!res.value) return const app = res.value - const waitingTime = Date.now() - app.updatedAt.getTime() + // create instance + await this.instanceService.create(app) // if waiting time is more than 5 minutes, stop the application + const waitingTime = Date.now() - app.updatedAt.getTime() if (waitingTime > 1000 * 60 * 5) { await db.collection('Application').updateOne( - { - appid: app.appid, - phase: ApplicationPhase.Starting, - }, + { appid: app.appid, phase: ApplicationPhase.Starting }, { $set: { state: ApplicationState.Stopped, @@ -143,9 +122,7 @@ export class InstanceTaskService { }, ) - this.logger.debug( - `Application ${app.appid} updated to state Stopped due to timeout`, - ) + this.logger.log(`${app.appid} updated to state Stopped due to timeout`) return } @@ -189,45 +166,25 @@ export class InstanceTaskService { /** * State `Stopped`: - * - remove instance * - move phase `Started` to `Stopping` */ async handleStoppedState() { const db = SystemDatabase.db - const res = await db - .collection('Application') - .findOneAndUpdate( - { - state: ApplicationState.Stopped, - phase: ApplicationPhase.Started, - lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, - }, - { $set: { lockedAt: new Date() } }, - ) - - if (!res.value) return - const app = res.value - - // remove instance - await this.instanceService.remove(app) - - // update phase to `Stopping` - await db.collection('Application').updateOne( + await db.collection('Application').updateMany( { - appid: app.appid, + state: ApplicationState.Stopped, phase: ApplicationPhase.Started, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, { $set: { - phase: ApplicationPhase.Stopping, lockedAt: TASK_LOCK_INIT_TIME, + phase: ApplicationPhase.Stopping, updatedAt: new Date(), }, }, ) - - this.logger.log(`Application ${app.appid} updated to phase stopping`) } /** @@ -257,12 +214,14 @@ export class InstanceTaskService { // check if the instance is removed const instance = await this.instanceService.get(app) if (instance.deployment) { + await this.instanceService.remove(app) await this.relock(appid, waitingTime) return } // check if the service is removed if (instance.service) { + await this.instanceService.remove(app) await this.relock(appid, waitingTime) return } @@ -284,32 +243,17 @@ export class InstanceTaskService { /** * State `Restarting`: - * - remove instance * - move phase `Started` to `Stopping` */ async handleRestartingStateDown() { const db = SystemDatabase.db - const res = await db - .collection('Application') - .findOneAndUpdate( - { - state: ApplicationState.Restarting, - phase: ApplicationPhase.Started, - lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, - }, - { $set: { lockedAt: new Date() } }, - ) - - if (!res.value) return - const app = res.value - - // remove instance - await this.instanceService.remove(app) - - // update phase to `Stopping` - await db.collection('Application').updateOne( - { appid: app.appid, phase: ApplicationPhase.Started }, + await db.collection('Application').updateMany( + { + state: ApplicationState.Restarting, + phase: ApplicationPhase.Started, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, + }, { $set: { phase: ApplicationPhase.Stopping, @@ -318,38 +262,21 @@ export class InstanceTaskService { }, }, ) - - this.logger.debug(`${app.appid} updated to stopping for restarting`) } /** * State `Restarting`: - * - create instance * - move phase `Stopped` to `Starting` */ async handleRestartingStateUp() { const db = SystemDatabase.db - const res = await db - .collection('Application') - .findOneAndUpdate( - { - state: ApplicationState.Restarting, - phase: ApplicationPhase.Stopped, - lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, - }, - { $set: { lockedAt: new Date() } }, - ) - - if (!res.value) return - const app = res.value - - // create instance - await this.instanceService.create(app) - - // update phase to `Starting` - await db.collection('Application').updateOne( - { appid: app.appid, phase: ApplicationPhase.Stopped }, + await db.collection('Application').updateMany( + { + state: ApplicationState.Restarting, + phase: ApplicationPhase.Stopped, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, + }, { $set: { phase: ApplicationPhase.Starting, @@ -358,8 +285,6 @@ export class InstanceTaskService { }, }, ) - - this.logger.debug(`${app.appid} updated starting for restarting`) } /** diff --git a/server/src/instance/instance.service.ts b/server/src/instance/instance.service.ts index 7442e5ae27..e4b62be7e5 100644 --- a/server/src/instance/instance.service.ts +++ b/server/src/instance/instance.service.ts @@ -268,9 +268,7 @@ export class InstanceService { region, app.appid, ) - if (!namespace) { - return - } + if (!namespace) return const appsV1Api = this.clusterService.makeAppsV1Api(region) const coreV1Api = this.clusterService.makeCoreV1Api(region) diff --git a/server/src/storage/bucket-task.service.ts b/server/src/storage/bucket-task.service.ts index 94a80b43d5..0100916534 100644 --- a/server/src/storage/bucket-task.service.ts +++ b/server/src/storage/bucket-task.service.ts @@ -184,7 +184,11 @@ export class BucketTaskService { const db = SystemDatabase.db await db.collection('StorageBucket').updateMany( - { state: DomainState.Active, phase: DomainPhase.Deleted }, + { + state: DomainState.Active, + phase: DomainPhase.Deleted, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, + }, { $set: { phase: DomainPhase.Creating, lockedAt: TASK_LOCK_INIT_TIME }, }, @@ -199,7 +203,11 @@ export class BucketTaskService { const db = SystemDatabase.db await db.collection('StorageBucket').updateMany( - { state: DomainState.Inactive, phase: DomainPhase.Created }, + { + state: DomainState.Inactive, + phase: DomainPhase.Created, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, + }, { $set: { phase: DomainPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, }, @@ -215,7 +223,11 @@ export class BucketTaskService { const db = SystemDatabase.db await db.collection('StorageBucket').updateMany( - { state: DomainState.Deleted, phase: DomainPhase.Created }, + { + state: DomainState.Deleted, + phase: { $in: [DomainPhase.Created, DomainPhase.Creating] }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, + }, { $set: { phase: DomainPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, }, diff --git a/server/src/trigger/trigger-task.service.ts b/server/src/trigger/trigger-task.service.ts index 23c70068f8..5222b896f4 100644 --- a/server/src/trigger/trigger-task.service.ts +++ b/server/src/trigger/trigger-task.service.ts @@ -4,7 +4,6 @@ import { TASK_LOCK_INIT_TIME } from 'src/constants' import { SystemDatabase } from 'src/database/system-database' import { CronJobService } from './cron-job.service' import { CronTrigger, TriggerPhase, TriggerState } from '@prisma/client' -import * as assert from 'node:assert' @Injectable() export class TriggerTaskService { @@ -151,7 +150,10 @@ export class TriggerTaskService { const db = SystemDatabase.db await db.collection('CronTrigger').updateMany( - { state: TriggerState.Inactive, phase: TriggerPhase.Created }, + { + state: TriggerState.Inactive, + phase: { $in: [TriggerPhase.Created, TriggerPhase.Creating] }, + }, { $set: { phase: TriggerPhase.Deleting, lockedAt: TASK_LOCK_INIT_TIME }, },