From 9ff7d80f01aef67e702053573994a11082950126 Mon Sep 17 00:00:00 2001 From: Lemonynade Date: Fri, 13 Dec 2024 06:54:30 -0500 Subject: [PATCH] feat: removes queue logic from post record and splits it out into post-queue (#322) * feat: post queue separated from post record ledger * feat: place most post record logic into post queue * test: adds tests for post queue * test: some fixes, improvements, and tests for post queue --- .../entities/directory-watcher.entity.ts | 5 +- .../src/app/database/entities/index.ts | 2 + .../entities/post-queue-record.entity.ts | 39 +++ .../database/entities/post-record.entity.ts | 17 +- .../database/entities/submission.entity.ts | 21 +- .../src/app/database/mikro-orm.providers.ts | 2 + .../app/post/dtos/post-queue-action.dto.ts | 10 + .../src/app/post/post.controller.ts | 17 +- .../client-server/src/app/post/post.module.ts | 11 +- .../src/app/post/post.service.spec.ts | 4 +- .../src/app/post/post.service.ts | 169 +---------- .../post-file-resizer.service.spec.ts | 4 +- .../post-file-resizer.service.ts | 4 +- .../post-manager.service.spec.ts | 68 ++--- .../post-manager}/post-manager.service.ts | 74 ++--- .../post-queue/post-queue.controller.ts | 30 ++ .../post-queue/post-queue.service.spec.ts | 215 ++++++++++++++ .../services/post-queue/post-queue.service.ts | 270 ++++++++++++++++++ .../src/app/settings/settings.constants.ts | 1 + apps/client-server/src/main.ts | 3 + .../src/api/form-generator.api.ts | 8 +- apps/postybirb-ui/src/api/post-queue.api.ts | 22 ++ apps/postybirb-ui/src/api/post.api.ts | 8 - .../post-selected-submissions-action.tsx | 4 +- .../submission-view-card-actions.tsx | 6 +- .../pages/submission/edit-submission-page.tsx | 6 +- libs/types/src/dtos/index.ts | 3 + .../src/dtos/post/post-queue-action.dto.ts | 5 + .../src/dtos/post/post-queue-record.dto.ts | 10 + libs/types/src/dtos/post/post-record.dto.ts | 3 +- .../src/dtos/submission/submission.dto.ts | 6 +- libs/types/src/models/index.ts | 2 + .../post/post-queue-record.interface.ts | 9 + .../src/models/post/post-record.interface.ts | 7 + .../post/website-post-record.interface.ts | 1 + .../settings/settings-options.interface.ts | 6 + .../models/submission/submission.interface.ts | 7 + package.json | 1 + yarn.lock | 7 + 39 files changed, 781 insertions(+), 306 deletions(-) create mode 100644 apps/client-server/src/app/database/entities/post-queue-record.entity.ts create mode 100644 apps/client-server/src/app/post/dtos/post-queue-action.dto.ts rename apps/client-server/src/app/post/{ => services/post-file-resizer}/post-file-resizer.service.spec.ts (97%) rename apps/client-server/src/app/post/{ => services/post-file-resizer}/post-file-resizer.service.ts (97%) rename apps/client-server/src/app/post/{ => services/post-manager}/post-manager.service.spec.ts (56%) rename apps/client-server/src/app/post/{ => services/post-manager}/post-manager.service.ts (90%) create mode 100644 apps/client-server/src/app/post/services/post-queue/post-queue.controller.ts create mode 100644 apps/client-server/src/app/post/services/post-queue/post-queue.service.spec.ts create mode 100644 apps/client-server/src/app/post/services/post-queue/post-queue.service.ts create mode 100644 apps/postybirb-ui/src/api/post-queue.api.ts create mode 100644 libs/types/src/dtos/post/post-queue-action.dto.ts create mode 100644 libs/types/src/dtos/post/post-queue-record.dto.ts create mode 100644 libs/types/src/models/post/post-queue-record.interface.ts diff --git a/apps/client-server/src/app/database/entities/directory-watcher.entity.ts b/apps/client-server/src/app/database/entities/directory-watcher.entity.ts index 8b09e3dd6..523bb9b16 100644 --- a/apps/client-server/src/app/database/entities/directory-watcher.entity.ts +++ b/apps/client-server/src/app/database/entities/directory-watcher.entity.ts @@ -10,6 +10,7 @@ import { DirectoryWatcherDto, DirectoryWatcherImportAction, IDirectoryWatcher, + ISubmission, } from '@postybirb/types'; import { PostyBirbRepository } from '../repositories/postybirb-repository'; import { PostyBirbEntity } from './postybirb-entity'; @@ -36,13 +37,13 @@ export class DirectoryWatcher serializer: (s) => s?.id, cascade: [], }) - template?: Rel; + template?: Rel; constructor(directoryWatcher: IDirectoryWatcher) { super(); this.path = directoryWatcher.path; this.importAction = directoryWatcher.importAction; - this.template = directoryWatcher.template as Submission; + this.template = directoryWatcher.template; } toJSON(): DirectoryWatcherDto { diff --git a/apps/client-server/src/app/database/entities/index.ts b/apps/client-server/src/app/database/entities/index.ts index d00718f2d..3930a8f71 100644 --- a/apps/client-server/src/app/database/entities/index.ts +++ b/apps/client-server/src/app/database/entities/index.ts @@ -1,5 +1,6 @@ export * from './account.entity'; export * from './directory-watcher.entity'; +export * from './post-queue-record.entity'; export * from './post-record.entity'; export * from './settings.entity'; export * from './submission-file.entity'; @@ -10,3 +11,4 @@ export * from './user-specified-website-options.entity'; export * from './website-data.entity'; export * from './website-options.entity'; export * from './website-post-record.entity'; + diff --git a/apps/client-server/src/app/database/entities/post-queue-record.entity.ts b/apps/client-server/src/app/database/entities/post-queue-record.entity.ts new file mode 100644 index 000000000..5b3a3b178 --- /dev/null +++ b/apps/client-server/src/app/database/entities/post-queue-record.entity.ts @@ -0,0 +1,39 @@ +import { Entity, OneToOne, Rel, serialize } from '@mikro-orm/core'; +import { + IPostQueueRecord, + IPostRecord, + ISubmission, + PostQueueRecordDto, +} from '@postybirb/types'; +import { PostRecord } from './post-record.entity'; +import { PostyBirbEntity } from './postybirb-entity'; +import { Submission } from './submission.entity'; + +@Entity() +export class PostQueueRecord + extends PostyBirbEntity + implements IPostQueueRecord +{ + @OneToOne(() => PostRecord, { + nullable: true, + orphanRemoval: false, + serializer: (pr) => pr.id, + eager: true, + inversedBy: 'postQueueRecord', + }) + postRecord?: Rel; + + @OneToOne({ + entity: () => Submission, + nullable: false, + serializer: (s) => s.id, + mappedBy: 'postQueueRecord', + orphanRemoval: false, + eager: true, + }) + submission: Rel; + + toJSON(): PostQueueRecordDto { + return serialize(this) as PostQueueRecordDto; + } +} diff --git a/apps/client-server/src/app/database/entities/post-record.entity.ts b/apps/client-server/src/app/database/entities/post-record.entity.ts index 72a0f978a..c7f38a4c7 100644 --- a/apps/client-server/src/app/database/entities/post-record.entity.ts +++ b/apps/client-server/src/app/database/entities/post-record.entity.ts @@ -4,11 +4,13 @@ import { EntityRepositoryType, ManyToOne, OneToMany, + OneToOne, Property, Rel, serialize, } from '@mikro-orm/core'; import { + IPostQueueRecord, IPostRecord, ISubmission, IWebsitePostRecord, @@ -17,6 +19,7 @@ import { PostRecordState, } from '@postybirb/types'; import { PostyBirbRepository } from '../repositories/postybirb-repository'; +import { PostQueueRecord } from './post-queue-record.entity'; import { PostyBirbEntity } from './postybirb-entity'; import { Submission } from './submission.entity'; import { WebsitePostRecord } from './website-post-record.entity'; @@ -52,6 +55,14 @@ export class PostRecord extends PostyBirbEntity implements IPostRecord { }) children: Collection; + @OneToOne(() => PostQueueRecord, { + orphanRemoval: true, + eager: true, + nullable: true, + mappedBy: 'postRecord', + }) + postQueueRecord?: Rel; + @Property({ type: 'string', nullable: false }) state: PostRecordState = PostRecordState.PENDING; @@ -61,7 +72,7 @@ export class PostRecord extends PostyBirbEntity implements IPostRecord { constructor( postRecord: Pick< IPostRecord, - 'parent' | 'completedAt' | 'state' | 'resumeMode' + 'parent' | 'completedAt' | 'state' | 'resumeMode' | 'postQueueRecord' >, ) { super(); @@ -73,6 +84,8 @@ export class PostRecord extends PostyBirbEntity implements IPostRecord { toJSON(): PostRecordDto { // eslint-disable-next-line @typescript-eslint/no-explicit-any - return serialize(this as any, { populate: ['children'] }) as PostRecordDto; + return serialize(this as any, { + populate: ['children', 'postQueueRecord'], + }) as PostRecordDto; } } diff --git a/apps/client-server/src/app/database/entities/submission.entity.ts b/apps/client-server/src/app/database/entities/submission.entity.ts index 79ea43da8..eaee591d4 100644 --- a/apps/client-server/src/app/database/entities/submission.entity.ts +++ b/apps/client-server/src/app/database/entities/submission.entity.ts @@ -3,8 +3,9 @@ import { Entity, EntityRepositoryType, OneToMany, + OneToOne, Property, - serialize, + serialize } from '@mikro-orm/core'; import { IPostRecord, @@ -14,11 +15,12 @@ import { ISubmissionMetadata, ISubmissionScheduleInfo, IWebsiteFormFields, - SubmissionType, + SubmissionType } from '@postybirb/types'; import { PostyBirbRepository } from '../repositories/postybirb-repository'; import { DirectoryWatcher } from './directory-watcher.entity'; +import { PostQueueRecord } from './post-queue-record.entity'; import { PostRecord } from './post-record.entity'; import { PostyBirbEntity } from './postybirb-entity'; import { SubmissionFile } from './submission-file.entity'; @@ -73,6 +75,13 @@ export class Submission }) posts: Collection; + @OneToOne(() => PostQueueRecord, { + orphanRemoval: true, + eager: true, + nullable: true, + }) + postQueueRecord?: PostQueueRecord; + @Property({ type: 'number', nullable: true }) order: number; @@ -87,7 +96,13 @@ export class Submission toJSON(): ISubmissionDto { // eslint-disable-next-line @typescript-eslint/no-explicit-any return serialize(this as any, { - populate: ['files', 'options', 'options.account', 'posts'], + populate: [ + 'files', + 'options', + 'options.account', + 'posts', + 'postQueueRecord', + ], }) as ISubmissionDto; } } diff --git a/apps/client-server/src/app/database/mikro-orm.providers.ts b/apps/client-server/src/app/database/mikro-orm.providers.ts index 9a40dcf6a..53e482b92 100644 --- a/apps/client-server/src/app/database/mikro-orm.providers.ts +++ b/apps/client-server/src/app/database/mikro-orm.providers.ts @@ -6,6 +6,7 @@ import { Account, AltFile, DirectoryWatcher, + PostQueueRecord, PostRecord, PrimaryFile, Settings, @@ -36,6 +37,7 @@ const entities = [ WebsiteData, PostRecord, WebsitePostRecord, + PostQueueRecord, ]; const mikroOrmOptions: MikroOrmModuleSyncOptions = { diff --git a/apps/client-server/src/app/post/dtos/post-queue-action.dto.ts b/apps/client-server/src/app/post/dtos/post-queue-action.dto.ts new file mode 100644 index 000000000..c70a00db6 --- /dev/null +++ b/apps/client-server/src/app/post/dtos/post-queue-action.dto.ts @@ -0,0 +1,10 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IPostQueueActionDto } from '@postybirb/types'; +import { ArrayNotEmpty, IsArray } from 'class-validator'; + +export class PostQueueActionDto implements IPostQueueActionDto { + @ApiProperty() + @IsArray() + @ArrayNotEmpty() + submissionIds: string[]; +} diff --git a/apps/client-server/src/app/post/post.controller.ts b/apps/client-server/src/app/post/post.controller.ts index 749f5437a..e8710bb6f 100644 --- a/apps/client-server/src/app/post/post.controller.ts +++ b/apps/client-server/src/app/post/post.controller.ts @@ -1,8 +1,7 @@ -import { Body, Controller, Post } from '@nestjs/common'; -import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { Controller } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; import { PostyBirbController } from '../common/controller/postybirb-controller'; import { PostRecord } from '../database/entities'; -import { QueuePostRecordRequestDto } from './dtos/queue-post-record.dto'; import { PostService } from './post.service'; /** @@ -15,16 +14,4 @@ export class PostController extends PostyBirbController { constructor(readonly service: PostService) { super(service); } - - @Post('enqueue') - @ApiOkResponse({ description: 'Post(s) queued.' }) - async enqueue(@Body() request: QueuePostRecordRequestDto) { - return this.service.enqueue(request); - } - - @Post('dequeue') - @ApiOkResponse({ description: 'Post(s) dequeued.' }) - async dequeue(@Body() request: QueuePostRecordRequestDto) { - this.service.dequeue(request); - } } diff --git a/apps/client-server/src/app/post/post.module.ts b/apps/client-server/src/app/post/post.module.ts index c3350694e..ecafe46d1 100644 --- a/apps/client-server/src/app/post/post.module.ts +++ b/apps/client-server/src/app/post/post.module.ts @@ -2,14 +2,17 @@ import { Module } from '@nestjs/common'; import { DatabaseModule } from '../database/database.module'; import { FileConverterModule } from '../file-converter/file-converter.module'; import { PostParsersModule } from '../post-parsers/post-parsers.module'; +import { SettingsModule } from '../settings/settings.module'; import { ValidationModule } from '../validation/validation.module'; import { WebsiteOptionsModule } from '../website-options/website-options.module'; import { WebsiteImplProvider } from '../websites/implementations'; import { WebsitesModule } from '../websites/websites.module'; -import { PostFileResizerService } from './post-file-resizer.service'; -import { PostManagerService } from './post-manager.service'; import { PostController } from './post.controller'; import { PostService } from './post.service'; +import { PostFileResizerService } from './services/post-file-resizer/post-file-resizer.service'; +import { PostManagerService } from './services/post-manager/post-manager.service'; +import { PostQueueController } from './services/post-queue/post-queue.controller'; +import { PostQueueService } from './services/post-queue/post-queue.service'; @Module({ imports: [ @@ -19,13 +22,15 @@ import { PostService } from './post.service'; PostParsersModule, ValidationModule, FileConverterModule, + SettingsModule, ], - controllers: [PostController], + controllers: [PostController, PostQueueController], providers: [ PostService, PostManagerService, PostFileResizerService, WebsiteImplProvider, + PostQueueService, ], }) export class PostModule {} diff --git a/apps/client-server/src/app/post/post.service.spec.ts b/apps/client-server/src/app/post/post.service.spec.ts index 800628ef6..8ca96c373 100644 --- a/apps/client-server/src/app/post/post.service.spec.ts +++ b/apps/client-server/src/app/post/post.service.spec.ts @@ -14,10 +14,10 @@ import { UserSpecifiedWebsiteOptionsModule } from '../user-specified-website-opt import { ValidationModule } from '../validation/validation.module'; import { WebsiteOptionsModule } from '../website-options/website-options.module'; import { WebsitesModule } from '../websites/websites.module'; -import { PostFileResizerService } from './post-file-resizer.service'; -import { PostManagerService } from './post-manager.service'; import { PostModule } from './post.module'; import { PostService } from './post.service'; +import { PostFileResizerService } from './services/post-file-resizer/post-file-resizer.service'; +import { PostManagerService } from './services/post-manager/post-manager.service'; describe('PostService', () => { let service: PostService; diff --git a/apps/client-server/src/app/post/post.service.ts b/apps/client-server/src/app/post/post.service.ts index b4c4da439..7646ec08a 100644 --- a/apps/client-server/src/app/post/post.service.ts +++ b/apps/client-server/src/app/post/post.service.ts @@ -1,184 +1,21 @@ import { InjectRepository } from '@mikro-orm/nestjs'; -import { Inject, Injectable, Optional, forwardRef } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; -import { - PostRecordResumeMode, - PostRecordState, - SubmissionId, -} from '@postybirb/types'; -import { Cron as CronGenerator } from 'croner'; -import { uniq } from 'lodash'; +import { Injectable, Optional } from '@nestjs/common'; import { PostyBirbService } from '../common/service/postybirb-service'; -import { PostRecord, Submission } from '../database/entities'; +import { PostRecord } from '../database/entities'; import { PostyBirbRepository } from '../database/repositories/postybirb-repository'; -import { IsTestEnvironment } from '../utils/test.util'; import { WSGateway } from '../web-socket/web-socket-gateway'; -import { WebsiteOptionsService } from '../website-options/website-options.service'; -import { QueuePostRecordRequestDto } from './dtos/queue-post-record.dto'; -import { PostManagerService } from './post-manager.service'; /** - * Handles enqueue and dequeue of post records. + * Simple entity service for post records. * @class PostService */ @Injectable() export class PostService extends PostyBirbService { constructor( - @Inject(forwardRef(() => PostManagerService)) - private readonly postManagerService: PostManagerService, @InjectRepository(PostRecord) repository: PostyBirbRepository, - @InjectRepository(Submission) - private readonly submissionRepository: PostyBirbRepository, - private readonly submissionOptionsService: WebsiteOptionsService, @Optional() webSocket?: WSGateway, ) { super(repository, webSocket); } - - /** - * CRON run queue scheduled. - */ - @Cron(CronExpression.EVERY_30_SECONDS) - private async run() { - if (!IsTestEnvironment()) { - const entities = await this.submissionRepository.find({ - isScheduled: true, - }); - const now = Date.now(); - const sorted = entities - .filter((e) => new Date(e.schedule.scheduledFor).getTime() <= now) // Only those that are ready to be posted. - .sort( - (a, b) => - new Date(a.schedule.scheduledFor).getTime() - - new Date(b.schedule.scheduledFor).getTime(), - ); // Sort by oldest first. - this.enqueue({ ids: sorted.map((s) => s.id) }); - - sorted - .filter((s) => s.schedule.cron) - .forEach((s) => { - const next = CronGenerator(s.schedule.cron).nextRun()?.toISOString(); - if (next) { - // eslint-disable-next-line no-param-reassign - s.schedule.scheduledFor = next; - this.submissionRepository.persistAndFlush(s); - } - }); - } - } - - /** - * Enqueues a post record for posting in order. - * @param {QueuePostRecordRequestDto} request - * @return {*} {Promise} - */ - async enqueue(request: QueuePostRecordRequestDto): Promise { - if (request.ids.length === 0) { - return []; - } - this.logger.debug(`Enqueueing ${request.ids} post records.`); - const existing = await this.repository.find({ - parent: { $in: request.ids }, - }); - - // Filter out any already queued that are not in a completed state. - // It may be better to move completed to a separate table to avoid this check. - const unqueued = uniq( - request.ids.filter( - (id) => !existing.some((e) => e.parent.id === id && !e.completedAt), - ), - ); - - const created: SubmissionId[] = []; - // eslint-disable-next-line no-restricted-syntax - for (const id of unqueued) { - const submission = await this.submissionRepository.findOne(id); - if (await this.verifyCanQueue(submission)) { - const postRecord = new PostRecord({ - parent: submission, - resumeMode: PostRecordResumeMode.CONTINUE, - state: PostRecordState.PENDING, - }); - await this.repository.persistAndFlush(postRecord); - created.push(postRecord.id); - } - } - - if (created.length > 0) { - // Attempt to start the post manager if it is not already running. - this.postManagerService.startPost(await this.getNext()); - } - - return created; - } - - /** - * Dequeues a post record from the queue. - * @param {QueuePostRecordRequestDto} request - * @return {*} {Promise} - */ - async dequeue(request: QueuePostRecordRequestDto): Promise { - this.logger.debug(`Dequeueing ${request.ids} post records.`); - const existing = await this.repository.find({ - parent: { $in: request.ids }, - }); - - // Only remove those that are not marked as done as to protect the archived posts. - // Ignore the running posts as they are in progress and will be handled naturally through throws. - const incomplete = existing - .filter((e: PostRecord) => e.completedAt === undefined) - .filter((e: PostRecord) => e.state === PostRecordState.PENDING); - - request.ids.forEach((id) => this.postManagerService.cancelIfRunning(id)); - await Promise.all(incomplete.map((i) => this.remove(i.id))); - await this.repository.flush(); - } - - /** - * Does basic validation to ensure the submission can be queued safely. - * @param {Submission} submission - * @return {*} {Promise} - */ - private async verifyCanQueue(submission: Submission): Promise { - if (!submission) { - return false; - } - - if (submission.options.length === 0) { - return false; - } - - const validations = await this.submissionOptionsService.validateSubmission( - submission.id, - ); - if (validations.some((v) => v.errors.length > 0)) { - return false; - } - - return true; - } - - /** - * Returns the next post record to be posted. - * @return {*} {Promise} - */ - async getNext(): Promise { - const entity = await this.repository.findOne( - { - completedAt: null, - }, - { - orderBy: { createdAt: 'ASC' }, - populate: [ - 'parent', - 'parent.options', - 'parent.options.account', - 'children', - 'children.account', - ], - }, - ); - return entity; - } } diff --git a/apps/client-server/src/app/post/post-file-resizer.service.spec.ts b/apps/client-server/src/app/post/services/post-file-resizer/post-file-resizer.service.spec.ts similarity index 97% rename from apps/client-server/src/app/post/post-file-resizer.service.spec.ts rename to apps/client-server/src/app/post/services/post-file-resizer/post-file-resizer.service.spec.ts index d9080fcec..e1683cf52 100644 --- a/apps/client-server/src/app/post/post-file-resizer.service.spec.ts +++ b/apps/client-server/src/app/post/services/post-file-resizer/post-file-resizer.service.spec.ts @@ -3,8 +3,8 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ISubmission, ISubmissionFile } from '@postybirb/types'; import { readFileSync } from 'fs'; import { join } from 'path'; -import { DatabaseModule } from '../database/database.module'; -import { ImageUtil } from '../file/utils/image.util'; +import { DatabaseModule } from '../../../database/database.module'; +import { ImageUtil } from '../../../file/utils/image.util'; import { PostFileResizerService } from './post-file-resizer.service'; describe('PostFileResizerService', () => { diff --git a/apps/client-server/src/app/post/post-file-resizer.service.ts b/apps/client-server/src/app/post/services/post-file-resizer/post-file-resizer.service.ts similarity index 97% rename from apps/client-server/src/app/post/post-file-resizer.service.ts rename to apps/client-server/src/app/post/services/post-file-resizer/post-file-resizer.service.ts index 239e2ca69..37345cf77 100644 --- a/apps/client-server/src/app/post/post-file-resizer.service.ts +++ b/apps/client-server/src/app/post/services/post-file-resizer/post-file-resizer.service.ts @@ -13,8 +13,8 @@ import fastq from 'fastq'; import { cpus } from 'os'; import { parse } from 'path'; import { Sharp } from 'sharp'; -import { ImageUtil } from '../file/utils/image.util'; -import { PostingFile, ThumbnailOptions } from './models/posting-file'; +import { ImageUtil } from '../../../file/utils/image.util'; +import { PostingFile, ThumbnailOptions } from '../../models/posting-file'; type ResizeRequest = { file: ISubmissionFile; diff --git a/apps/client-server/src/app/post/post-manager.service.spec.ts b/apps/client-server/src/app/post/services/post-manager/post-manager.service.spec.ts similarity index 56% rename from apps/client-server/src/app/post/post-manager.service.spec.ts rename to apps/client-server/src/app/post/services/post-manager/post-manager.service.spec.ts index 2a164aa4c..cc4f2d3ba 100644 --- a/apps/client-server/src/app/post/post-manager.service.spec.ts +++ b/apps/client-server/src/app/post/services/post-manager/post-manager.service.spec.ts @@ -5,28 +5,28 @@ import { SubmissionRating, SubmissionType, } from '@postybirb/types'; -import { AccountModule } from '../account/account.module'; -import { AccountService } from '../account/account.service'; -import { CreateAccountDto } from '../account/dtos/create-account.dto'; -import { DatabaseModule } from '../database/database.module'; -import { FileConverterModule } from '../file-converter/file-converter.module'; -import { FileConverterService } from '../file-converter/file-converter.service'; -import { PostParsersModule } from '../post-parsers/post-parsers.module'; -import { SettingsService } from '../settings/settings.service'; -import { CreateSubmissionDto } from '../submission/dtos/create-submission.dto'; -import { SubmissionService } from '../submission/services/submission.service'; -import { SubmissionModule } from '../submission/submission.module'; -import { UserSpecifiedWebsiteOptionsModule } from '../user-specified-website-options/user-specified-website-options.module'; -import { ValidationService } from '../validation/validation.service'; -import { CreateWebsiteOptionsDto } from '../website-options/dtos/create-website-options.dto'; -import { WebsiteOptionsModule } from '../website-options/website-options.module'; -import { WebsiteOptionsService } from '../website-options/website-options.service'; -import { WebsiteRegistryService } from '../websites/website-registry.service'; -import { WebsitesModule } from '../websites/websites.module'; -import { PostFileResizerService } from './post-file-resizer.service'; +import { AccountModule } from '../../../account/account.module'; +import { AccountService } from '../../../account/account.service'; +import { CreateAccountDto } from '../../../account/dtos/create-account.dto'; +import { DatabaseModule } from '../../../database/database.module'; +import { FileConverterModule } from '../../../file-converter/file-converter.module'; +import { FileConverterService } from '../../../file-converter/file-converter.service'; +import { PostParsersModule } from '../../../post-parsers/post-parsers.module'; +import { SettingsService } from '../../../settings/settings.service'; +import { CreateSubmissionDto } from '../../../submission/dtos/create-submission.dto'; +import { SubmissionService } from '../../../submission/services/submission.service'; +import { SubmissionModule } from '../../../submission/submission.module'; +import { UserSpecifiedWebsiteOptionsModule } from '../../../user-specified-website-options/user-specified-website-options.module'; +import { ValidationService } from '../../../validation/validation.service'; +import { CreateWebsiteOptionsDto } from '../../../website-options/dtos/create-website-options.dto'; +import { WebsiteOptionsModule } from '../../../website-options/website-options.module'; +import { WebsiteOptionsService } from '../../../website-options/website-options.service'; +import { WebsiteRegistryService } from '../../../websites/website-registry.service'; +import { WebsitesModule } from '../../../websites/websites.module'; +import { PostModule } from '../../post.module'; +import { PostService } from '../../post.service'; +import { PostFileResizerService } from '../post-file-resizer/post-file-resizer.service'; import { PostManagerService } from './post-manager.service'; -import { PostModule } from './post.module'; -import { PostService } from './post.service'; describe('PostManagerService', () => { let service: PostManagerService; @@ -130,20 +130,20 @@ describe('PostManagerService', () => { expect(service).toBeDefined(); }); - it('should handle Message submission', async () => { - const submission = await submissionService.create(createSubmissionDto()); - const account = await accountService.create(createAccountDto()); - expect(registryService.findInstance(account)).toBeDefined(); + // it('should handle Message submission', async () => { + // const submission = await submissionService.create(createSubmissionDto()); + // const account = await accountService.create(createAccountDto()); + // expect(registryService.findInstance(account)).toBeDefined(); - await websiteOptionsService.create( - createWebsiteOptionsDto(submission.id, account.id), - ); + // await websiteOptionsService.create( + // createWebsiteOptionsDto(submission.id, account.id), + // ); - await postService.enqueue({ ids: [submission.id] }); - const postRecord = await postService.getNext(); - expect(postRecord).toBeDefined(); + // await postService.enqueue({ ids: [submission.id] }); + // const postRecord = await postService.getNext(); + // expect(postRecord).toBeDefined(); - await service.startPost(postRecord); - expect(postRecord.children).toBeDefined(); - }); + // await service.startPost(postRecord); + // expect(postRecord.children).toBeDefined(); + // }); }); diff --git a/apps/client-server/src/app/post/post-manager.service.ts b/apps/client-server/src/app/post/services/post-manager/post-manager.service.ts similarity index 90% rename from apps/client-server/src/app/post/post-manager.service.ts rename to apps/client-server/src/app/post/services/post-manager/post-manager.service.ts index 584c7342a..380049ff7 100644 --- a/apps/client-server/src/app/post/post-manager.service.ts +++ b/apps/client-server/src/app/post/services/post-manager/post-manager.service.ts @@ -1,7 +1,7 @@ /* eslint-disable no-param-reassign */ import { EntityDTO, Loaded, wrap } from '@mikro-orm/core'; import { InjectRepository } from '@mikro-orm/nestjs'; -import { forwardRef, Inject, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { Logger } from '@postybirb/logger'; import { EntityId, @@ -26,23 +26,21 @@ import { } from '@postybirb/types'; import { getFileType } from '@postybirb/utils/file-type'; import { chunk } from 'lodash'; -import { PostRecord, WebsitePostRecord } from '../database/entities'; -import { PostyBirbRepository } from '../database/repositories/postybirb-repository'; -import { FileConverterService } from '../file-converter/file-converter.service'; -import { PostParsersService } from '../post-parsers/post-parsers.service'; -import { IsTestEnvironment } from '../utils/test.util'; -import { ValidationService } from '../validation/validation.service'; +import { PostRecord, WebsitePostRecord } from '../../../database/entities'; +import { PostyBirbRepository } from '../../../database/repositories/postybirb-repository'; +import { FileConverterService } from '../../../file-converter/file-converter.service'; +import { PostParsersService } from '../../../post-parsers/post-parsers.service'; +import { ValidationService } from '../../../validation/validation.service'; import { ImplementedFileWebsite, isFileWebsite, -} from '../websites/models/website-modifiers/file-website'; -import { MessageWebsite } from '../websites/models/website-modifiers/message-website'; -import { Website } from '../websites/website'; -import { WebsiteRegistryService } from '../websites/website-registry.service'; -import { CancellableToken } from './models/cancellable-token'; -import { PostingFile } from './models/posting-file'; -import { PostFileResizerService } from './post-file-resizer.service'; -import { PostService } from './post.service'; +} from '../../../websites/models/website-modifiers/file-website'; +import { MessageWebsite } from '../../../websites/models/website-modifiers/message-website'; +import { Website } from '../../../websites/website'; +import { WebsiteRegistryService } from '../../../websites/website-registry.service'; +import { CancellableToken } from '../../models/cancellable-token'; +import { PostingFile } from '../../models/posting-file'; +import { PostFileResizerService } from '../post-file-resizer/post-file-resizer.service'; type LoadedPostRecord = Loaded; @@ -69,29 +67,12 @@ export class PostManagerService { private readonly postRepository: PostyBirbRepository, @InjectRepository(WebsitePostRecord) private readonly websitePostRecordRepository: PostyBirbRepository, - @Inject(forwardRef(() => PostService)) - private readonly postService: PostService, private readonly websiteRegistry: WebsiteRegistryService, private readonly resizerService: PostFileResizerService, private readonly postParserService: PostParsersService, private readonly validationService: ValidationService, private readonly fileConverterService: FileConverterService, - ) { - setTimeout(() => this.check(), 60_000); - } - - /** - * Checks for any posts that need to be posted. - */ - private async check() { - if (!IsTestEnvironment()) { - const nextToPost = await this.postService.getNext(); - if (nextToPost && this.currentPost?.id !== nextToPost.id) { - this.logger.info(`Found next to post: ${nextToPost.id}`); - this.startPost(nextToPost); - } - } - } + ) {} private async protectedUpdate( entity: LoadedPostRecord, @@ -111,19 +92,18 @@ export class PostManagerService { * @param {SubmissionId} id */ public async cancelIfRunning(id: SubmissionId): Promise { - if (this.currentPost) { - if (!this.currentPost.parent) { - const loaded = await wrap(this.currentPost).init(true, ['parent']); - if (loaded.parent.id === id) { - this.logger.info(`Cancelling current post`); - this.cancelToken.cancel(); - return true; - } - } + if (this.currentPost && this.currentPost.parent?.id === id) { + this.logger.info(`Cancelling current post`); + this.cancelToken.cancel(); + return true; } return false; } + public isPosting(): boolean { + return !!this.currentPost; + } + /** * Starts a post attempt. * @param {PostRecord} entity @@ -139,11 +119,6 @@ export class PostManagerService { state: PostRecordState.RUNNING, }); - // Ensure parent (submission) is loaded - if (!entity.parent) { - entity = await wrap(entity).init(true, ['parent']); - } - await this.createWebsitePostRecords(entity); // Posts order occurs in batched groups @@ -158,14 +133,11 @@ export class PostManagerService { websites.map((w) => this.post(entity, w.record, w.instance)), ); } - await this.finishPost(entity); this.logger.info(`Finished posting to websites`); } catch (error) { this.logger.withError(error).error(`Error posting`); - await this.finishPost(entity); - throw error; } finally { - this.check(); + await this.finishPost(entity); } } diff --git a/apps/client-server/src/app/post/services/post-queue/post-queue.controller.ts b/apps/client-server/src/app/post/services/post-queue/post-queue.controller.ts new file mode 100644 index 000000000..647e88c84 --- /dev/null +++ b/apps/client-server/src/app/post/services/post-queue/post-queue.controller.ts @@ -0,0 +1,30 @@ +import { Body, Controller, Post } from '@nestjs/common'; +import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { PostyBirbController } from '../../../common/controller/postybirb-controller'; +import { PostQueueRecord } from '../../../database/entities'; +import { PostQueueActionDto } from '../../dtos/post-queue-action.dto'; +import { PostQueueService } from './post-queue.service'; + +/** + * Queue operations for Post data. + * @class PostController + */ +@ApiTags('post-queue') +@Controller('post-queue') +export class PostQueueController extends PostyBirbController { + constructor(readonly service: PostQueueService) { + super(service); + } + + @Post('enqueue') + @ApiOkResponse({ description: 'Post(s) queued.' }) + async enqueue(@Body() request: PostQueueActionDto) { + return this.service.enqueue(request.submissionIds); + } + + @Post('dequeue') + @ApiOkResponse({ description: 'Post(s) dequeued.' }) + async dequeue(@Body() request: PostQueueActionDto) { + this.service.dequeue(request.submissionIds); + } +} diff --git a/apps/client-server/src/app/post/services/post-queue/post-queue.service.spec.ts b/apps/client-server/src/app/post/services/post-queue/post-queue.service.spec.ts new file mode 100644 index 000000000..6544544c1 --- /dev/null +++ b/apps/client-server/src/app/post/services/post-queue/post-queue.service.spec.ts @@ -0,0 +1,215 @@ +import { MikroORM } from '@mikro-orm/core'; +import { Test, TestingModule } from '@nestjs/testing'; +import { + DefaultDescription, + PostRecordState, + SubmissionRating, + SubmissionType, +} from '@postybirb/types'; +import { AccountModule } from '../../../account/account.module'; +import { AccountService } from '../../../account/account.service'; +import { CreateAccountDto } from '../../../account/dtos/create-account.dto'; +import { DatabaseModule } from '../../../database/database.module'; +import { FileConverterModule } from '../../../file-converter/file-converter.module'; +import { FileConverterService } from '../../../file-converter/file-converter.service'; +import { PostParsersModule } from '../../../post-parsers/post-parsers.module'; +import { SettingsModule } from '../../../settings/settings.module'; +import { SettingsService } from '../../../settings/settings.service'; +import { CreateSubmissionDto } from '../../../submission/dtos/create-submission.dto'; +import { SubmissionService } from '../../../submission/services/submission.service'; +import { SubmissionModule } from '../../../submission/submission.module'; +import { UserSpecifiedWebsiteOptionsModule } from '../../../user-specified-website-options/user-specified-website-options.module'; +import { ValidationService } from '../../../validation/validation.service'; +import { CreateWebsiteOptionsDto } from '../../../website-options/dtos/create-website-options.dto'; +import { WebsiteOptionsModule } from '../../../website-options/website-options.module'; +import { WebsiteOptionsService } from '../../../website-options/website-options.service'; +import { WebsiteRegistryService } from '../../../websites/website-registry.service'; +import { WebsitesModule } from '../../../websites/websites.module'; +import { PostModule } from '../../post.module'; +import { PostService } from '../../post.service'; +import { PostFileResizerService } from '../post-file-resizer/post-file-resizer.service'; +import { PostManagerService } from '../post-manager/post-manager.service'; +import { PostQueueService } from './post-queue.service'; + +describe('PostQueueService', () => { + let service: PostQueueService; + let module: TestingModule; + let orm: MikroORM; + let submissionService: SubmissionService; + let accountService: AccountService; + let websiteOptionsService: WebsiteOptionsService; + let registryService: WebsiteRegistryService; + let postService: PostService; + let postManager: PostManagerService; + + beforeEach(async () => { + try { + module = await Test.createTestingModule({ + imports: [ + DatabaseModule, + SubmissionModule, + AccountModule, + WebsiteOptionsModule, + WebsitesModule, + UserSpecifiedWebsiteOptionsModule, + PostParsersModule, + PostModule, + FileConverterModule, + SettingsModule, + ], + providers: [ + PostQueueService, + PostManagerService, + PostService, + PostFileResizerService, + ValidationService, + FileConverterService, + SettingsService, + ], + }).compile(); + + service = module.get(PostQueueService); + submissionService = module.get(SubmissionService); + accountService = module.get(AccountService); + const settingsService = module.get(SettingsService); + websiteOptionsService = module.get( + WebsiteOptionsService, + ); + registryService = module.get( + WebsiteRegistryService, + ); + postService = module.get(PostService); + postManager = module.get(PostManagerService); + orm = module.get(MikroORM); + try { + await orm.getSchemaGenerator().refreshDatabase(); + } catch { + // none + } + await accountService.onModuleInit(); + await settingsService.onModuleInit(); + } catch (err) { + console.log(err); + } + }); + + function createSubmissionDto(): CreateSubmissionDto { + const dto = new CreateSubmissionDto(); + dto.name = 'Test'; + dto.type = SubmissionType.MESSAGE; + return dto; + } + + function createAccountDto(): CreateAccountDto { + const dto = new CreateAccountDto(); + dto.name = 'Test'; + dto.website = 'test'; + return dto; + } + + function createWebsiteOptionsDto( + submissionId: string, + accountId: string, + ): CreateWebsiteOptionsDto { + const dto = new CreateWebsiteOptionsDto(); + dto.submission = submissionId; + dto.account = accountId; + dto.data = { + title: 'Test Title', + tags: { + overrideDefault: true, + tags: ['test'], + }, + description: { + overrideDefault: true, + description: DefaultDescription(), + }, + rating: SubmissionRating.GENERAL, + }; + return dto; + } + + afterAll(async () => { + await orm.close(true); + await module.close(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should handle pausing and resuming the queue', async () => { + await service.pause(); + expect(await service.isPaused()).toBe(true); + await service.resume(); + expect(await service.isPaused()).toBe(false); + }); + + it('should handle enqueue and dequeue of submissions', async () => { + await service.pause(); // Just to test the function + const submission = await submissionService.create(createSubmissionDto()); + const account = await accountService.create(createAccountDto()); + expect(registryService.findInstance(account)).toBeDefined(); + + await websiteOptionsService.create( + createWebsiteOptionsDto(submission.id, account.id), + ); + + await service.enqueue([submission.id, submission.id]); + expect((await service.findAll()).length).toBe(1); + const top = await service.peek(); + expect(top).toBeDefined(); + expect(top.submission.id).toBe(submission.id); + + await service.dequeue([submission.id]); + expect((await service.findAll()).length).toBe(0); + expect(await service.peek()).toBeUndefined(); + }); + + async function waitForPostManager(): Promise { + while (postManager.isPosting()) { + await new Promise((resolve) => { + setTimeout(resolve, 100); + }); + } + } + + it('should insert posts into the post manager', async () => { + const submission = await submissionService.create(createSubmissionDto()); + const account = await accountService.create(createAccountDto()); + expect(registryService.findInstance(account)).toBeDefined(); + + await websiteOptionsService.create( + createWebsiteOptionsDto(submission.id, account.id), + ); + + await service.enqueue([submission.id]); + expect((await service.findAll()).length).toBe(1); + + // We expect the creation of a record and a start of the post manager + await service.execute(); + let postRecord = (await postService.findAll())[0]; + let queueRecord = await service.peek(); + expect(postRecord).toBeDefined(); + expect(postRecord.parent.id).toBe(submission.id); + expect(postManager.isPosting()).toBe(true); + expect(queueRecord).toBeDefined(); + expect(queueRecord.postRecord).toBeDefined(); + + expect(await postManager.cancelIfRunning(submission.id)).toBeTruthy(); + await waitForPostManager(); + expect(postManager.isPosting()).toBe(false); + + queueRecord = await service.peek(); + expect(queueRecord).toBeDefined(); + expect(queueRecord.postRecord).toBeDefined(); + + // We expect the post to be in a terminal state and cleanup of the record. + // The post record should remain after the queue record is deleted. + await service.execute(); + expect((await service.findAll()).length).toBe(0); + postRecord = await postService.findById(postRecord.id); + expect(postRecord.state).toBe(PostRecordState.FAILED); + expect(postRecord.completedAt).toBeDefined(); + }); +}); diff --git a/apps/client-server/src/app/post/services/post-queue/post-queue.service.ts b/apps/client-server/src/app/post/services/post-queue/post-queue.service.ts new file mode 100644 index 000000000..92822129d --- /dev/null +++ b/apps/client-server/src/app/post/services/post-queue/post-queue.service.ts @@ -0,0 +1,270 @@ +import { InjectRepository } from '@mikro-orm/nestjs'; +import { + Injectable, + InternalServerErrorException, + Optional, +} from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { + IPostRecord, + PostRecordResumeMode, + PostRecordState, + SubmissionId, +} from '@postybirb/types'; +import { Mutex } from 'async-mutex'; +import { Cron as CronGenerator } from 'croner'; +import { PostyBirbService } from '../../../common/service/postybirb-service'; +import { + PostQueueRecord, + PostRecord, + Submission, +} from '../../../database/entities'; +import { PostyBirbRepository } from '../../../database/repositories/postybirb-repository'; +import { SettingsService } from '../../../settings/settings.service'; +import { IsTestEnvironment } from '../../../utils/test.util'; +import { WSGateway } from '../../../web-socket/web-socket-gateway'; +import { PostManagerService } from '../post-manager/post-manager.service'; + +/** + * Handles the queue of posts to be posted. + * This service is responsible for managing the queue of posts to be posted. + * It will create post records and start the post manager when a post is ready to be posted. + * @class PostQueueService + */ +@Injectable() +export class PostQueueService extends PostyBirbService { + private readonly queueModificationMutex = new Mutex(); + + private readonly queueMutex = new Mutex(); + + private readonly initTime = Date.now(); + + constructor( + @InjectRepository(PostQueueRecord) + readonly repository: PostyBirbRepository, + @InjectRepository(Submission) + private readonly submissionRepository: PostyBirbRepository, + private readonly postManager: PostManagerService, + private readonly postRecordRepository: PostyBirbRepository, + private readonly settingsService: SettingsService, + @Optional() webSocket?: WSGateway, + ) { + super(repository, webSocket); + } + + public async isPaused(): Promise { + return (await this.settingsService.getDefaultSettings()).settings + .queuePaused; + } + + public async pause() { + this.logger.info('Queue paused'); + const settings = await this.settingsService.getDefaultSettings(); + await this.settingsService.update(settings.id, { + settings: { ...settings.settings, queuePaused: true }, + }); + } + + public async resume() { + this.logger.info('Queue resumed'); + const settings = await this.settingsService.getDefaultSettings(); + await this.settingsService.update(settings.id, { + settings: { ...settings.settings, queuePaused: false }, + }); + } + + public override remove(id: string) { + return this.dequeue([id]); + } + + public async enqueue(submissionIds: SubmissionId[]) { + const release = await this.queueModificationMutex.acquire(); + this.logger.withMetadata({ submissionIds }).info('Enqueueing posts'); + + try { + for (const submissionId of submissionIds) { + if (!(await this.repository.findOne({ submission: submissionId }))) { + const record = this.repository.create({ + submission: submissionId, + }); + + await this.repository.persistAndFlush(record); + } + } + } catch (error) { + this.logger.withMetadata({ error }).error('Failed to enqueue posts'); + throw new InternalServerErrorException(error.message); + } finally { + release(); + } + } + + public async dequeue(submissionIds: SubmissionId[]) { + const release = await this.queueModificationMutex.acquire(); + this.logger.withMetadata({ submissionIds }).info('Dequeueing posts'); + + try { + const records = await this.repository.find({ + submission: { $in: submissionIds }, + }); + + submissionIds.forEach((id) => this.postManager.cancelIfRunning(id)); + + await this.repository.removeAndFlush(records); + } catch (error) { + this.logger.withMetadata({ error }).error('Failed to dequeue posts'); + throw new InternalServerErrorException(error.message); + } finally { + release(); + } + } + + /** + * CRON based enqueueing of scheduled submissions. + */ + @Cron(CronExpression.EVERY_30_SECONDS) + public async checkForScheduledSubmissions() { + if (IsTestEnvironment()) { + return; + } + + const entities = await this.submissionRepository.find({ + isScheduled: true, + }); + const now = Date.now(); + const sorted = entities + .filter((e) => new Date(e.schedule.scheduledFor).getTime() <= now) // Only those that are ready to be posted. + .sort( + (a, b) => + new Date(a.schedule.scheduledFor).getTime() - + new Date(b.schedule.scheduledFor).getTime(), + ); // Sort by oldest first. + await this.enqueue(sorted.map((s) => s.id)); + sorted + .filter((s) => s.schedule.cron) + .forEach((s) => { + const next = CronGenerator(s.schedule.cron).nextRun()?.toISOString(); + if (next) { + // eslint-disable-next-line no-param-reassign + s.schedule.scheduledFor = next; + this.submissionRepository.persistAndFlush(s); + } + }); + } + + /** + * This runs a check every second on the state of queue items. + * This aims to have simple logic. Each run will either create a post record and start the post manager, + * or remove a submission from the queue if it is in a terminal state. + * Nothing happens if the queue is empty. + */ + @Cron(CronExpression.EVERY_SECOND) + public async run() { + if (!(this.initTime + 60_000 <= Date.now())) { + // Only run after 1 minute to allow the application to start up. + this.logger.info('Waiting for queue grace period to end'); + return; + } + + if (IsTestEnvironment()) { + return; + } + + await this.execute(); + } + + /** + * Manages the queue by peeking at the top of the queue and deciding what to do based on the + * state of the queue. + * + * Made public for testing purposes. + */ + public async execute() { + if (this.queueMutex.isLocked()) { + return; + } + + const release = await this.queueMutex.acquire(); + + try { + const top = await this.peek(); + // Queue Empty + if (!top) { + return; + } + + const isPaused = await this.isPaused(); + const { postRecord: record, submission } = top; + if (!record) { + // No record present, create one and start the post manager (if not paused) + if (this.postManager.isPosting()) { + // !NOTE - Not sure this could actually happen, but it's here just in case since it would be bad. + this.logger.warn( + 'The post manager is already posting, but no record is present in the top of the queue', + ); + return; + } + + if (isPaused) { + this.logger.info('Queue is paused'); + return; + } + const postRecord = new PostRecord({ + parent: submission, + resumeMode: PostRecordResumeMode.CONTINUE, + state: PostRecordState.PENDING, + postQueueRecord: top, + }); + top.postRecord = postRecord as IPostRecord; + this.logger + .withMetadata({ postRecord }) + .info('Creating PostRecord and starting PostManager'); + await this.postRecordRepository.persistAndFlush(postRecord); + this.postManager.startPost(postRecord); + } else if ( + record.state === PostRecordState.DONE || + record.state === PostRecordState.FAILED + ) { + // Post is in a terminal state, remove from queue + await this.dequeue([submission.id]); + } else if (!this.postManager.isPosting()) { + // Post is not in a terminal state, but the post manager is not posting, so restart it. + if (isPaused) { + this.logger.info('Queue is paused'); + return; + } + this.logger + .withMetadata({ record }) + .info( + 'PostManager is not posting, but record is not in terminal state. Resuming record.', + ); + this.postManager.startPost(record as unknown as PostRecord); + } + } catch (error) { + this.logger.withMetadata({ error }).error('Failed to run queue'); + } finally { + release(); + } + } + + /** + * Peeks at the next item in the queue. + * Based on the createdAt date. + */ + public async peek(): Promise { + const all = await this.repository.findAll({ + limit: 1, + orderBy: { createdAt: 'ASC' }, + populate: [ + 'postRecord', + 'postRecord.children', + 'postRecord.parent', + 'postRecord.parent.options', + 'postRecord.parent.options.account', + 'submission', + ], + }); + + return all[0]; + } +} diff --git a/apps/client-server/src/app/settings/settings.constants.ts b/apps/client-server/src/app/settings/settings.constants.ts index d30c6b6f1..96d80201a 100644 --- a/apps/client-server/src/app/settings/settings.constants.ts +++ b/apps/client-server/src/app/settings/settings.constants.ts @@ -7,5 +7,6 @@ export class SettingsConstants { hiddenWebsites: [], language: 'en', allowAd: true, + queuePaused: false, }; } diff --git a/apps/client-server/src/main.ts b/apps/client-server/src/main.ts index a8ae46090..599b6fa65 100644 --- a/apps/client-server/src/main.ts +++ b/apps/client-server/src/main.ts @@ -72,6 +72,8 @@ async function bootstrap() { .addTag('file') .addTag('file-submission') .addTag('form-generator') + .addTag('post') + .addTag('post-queue') .addTag('submission') .addTag('tag-converters') .addTag('tag-groups') @@ -95,3 +97,4 @@ async function bootstrap() { } export { bootstrap as bootstrapClientServer }; + diff --git a/apps/postybirb-ui/src/api/form-generator.api.ts b/apps/postybirb-ui/src/api/form-generator.api.ts index 7d05a7da6..abf588408 100644 --- a/apps/postybirb-ui/src/api/form-generator.api.ts +++ b/apps/postybirb-ui/src/api/form-generator.api.ts @@ -6,15 +6,11 @@ class FormGeneratorApi { private readonly client: HttpClient = new HttpClient('form-generator'); getDefaultForm(type: SubmissionType) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return this.client.get>(`default/${type}`); + return this.client.get(`default/${type}`); } getForm(dto: IFormGenerationRequestDto) { - return this.client.post< - // eslint-disable-next-line @typescript-eslint/no-explicit-any - FormBuilderMetadata - >('', dto); + return this.client.post('', dto); } } diff --git a/apps/postybirb-ui/src/api/post-queue.api.ts b/apps/postybirb-ui/src/api/post-queue.api.ts new file mode 100644 index 000000000..81957dbb6 --- /dev/null +++ b/apps/postybirb-ui/src/api/post-queue.api.ts @@ -0,0 +1,22 @@ +import { IPostQueueActionDto, PostQueueRecordDto } from '@postybirb/types'; +import { BaseApi } from './base.api'; + +class PostQueueApi extends BaseApi< + PostQueueRecordDto, + IPostQueueActionDto, + IPostQueueActionDto +> { + constructor() { + super('post-queue'); + } + + enqueue(ids: string[]) { + return this.client.post('enqueue', { ids }); + } + + dequeue(ids: string[]) { + return this.client.post('dequeue', { ids }); + } +} + +export default new PostQueueApi(); diff --git a/apps/postybirb-ui/src/api/post.api.ts b/apps/postybirb-ui/src/api/post.api.ts index 8b146f876..66ecef613 100644 --- a/apps/postybirb-ui/src/api/post.api.ts +++ b/apps/postybirb-ui/src/api/post.api.ts @@ -9,14 +9,6 @@ class PostApi extends BaseApi< constructor() { super('post'); } - - enqueue(ids: string[]) { - return this.client.post('enqueue', { ids }); - } - - dequeue(ids: string[]) { - return this.client.post('dequeue', { ids }); - } } export default new PostApi(); diff --git a/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-actions/post-selected-submissions-action.tsx b/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-actions/post-selected-submissions-action.tsx index 08b098632..c91816751 100644 --- a/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-actions/post-selected-submissions-action.tsx +++ b/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-actions/post-selected-submissions-action.tsx @@ -8,7 +8,7 @@ import { Text, } from '@mantine/core'; import { IconSend } from '@tabler/icons-react'; -import postApi from '../../../../../api/post.api'; +import postQueueApi from '../../../../../api/post-queue.api'; import { SubmissionViewActionProps } from './submission-view-actions.props'; export function PostSelectedSubmissionsActions({ @@ -40,7 +40,7 @@ export function PostSelectedSubmissionsActions({ variant="light" leftSection={} onClick={() => { - postApi.enqueue(selected.map((s) => s.id)); + postQueueApi.enqueue(selected.map((s) => s.id)); }} > Post Selected diff --git a/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-card-actions.tsx b/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-card-actions.tsx index c40e90424..488d087a8 100644 --- a/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-card-actions.tsx +++ b/apps/postybirb-ui/src/components/submissions/submission-view/submission-view-card/submission-view-card-actions.tsx @@ -12,7 +12,7 @@ import { IconSend, } from '@tabler/icons-react'; import { useNavigate } from 'react-router'; -import postApi from '../../../../api/post.api'; +import postQueueApi from '../../../../api/post-queue.api'; import submissionApi from '../../../../api/submission.api'; import { SubmissionDto } from '../../../../models/dtos/submission.dto'; import { EditSubmissionPath } from '../../../../pages/route-paths'; @@ -139,7 +139,7 @@ export function SubmissionViewCardActions( c={canSetForPosting ? 'teal' : 'grey'} leftSection={} onClick={() => { - postApi + postQueueApi .enqueue([submission.id]) .then(() => { notifications.show({ @@ -167,7 +167,7 @@ export function SubmissionViewCardActions( c="red" leftSection={} onClick={() => { - postApi.dequeue([submission.id]); + postQueueApi.dequeue([submission.id]); }} > Cancel diff --git a/apps/postybirb-ui/src/pages/submission/edit-submission-page.tsx b/apps/postybirb-ui/src/pages/submission/edit-submission-page.tsx index 752e0adbb..bb4834b65 100644 --- a/apps/postybirb-ui/src/pages/submission/edit-submission-page.tsx +++ b/apps/postybirb-ui/src/pages/submission/edit-submission-page.tsx @@ -13,7 +13,7 @@ import { } from '@tabler/icons-react'; import { useMemo, useState } from 'react'; import { useParams } from 'react-router'; -import postApi from '../../api/post.api'; +import postQueueApi from '../../api/post-queue.api'; import submissionApi from '../../api/submission.api'; import websiteOptionsApi from '../../api/website-options.api'; import { PageHeader } from '../../components/page-header/page-header'; @@ -151,7 +151,7 @@ function PostAction({ submission }: { submission: SubmissionDto }) { c="red" leftSection={} onClick={() => { - postApi.dequeue([submission.id]); + postQueueApi.dequeue([submission.id]); }} > Cancel @@ -173,7 +173,7 @@ function PostAction({ submission }: { submission: SubmissionDto }) { c={canSetForPosting ? 'teal' : 'grey'} leftSection={} onClick={() => { - postApi + postQueueApi .enqueue([submission.id]) .then(() => { notifications.show({ diff --git a/libs/types/src/dtos/index.ts b/libs/types/src/dtos/index.ts index 8e8443c57..04ce816e2 100644 --- a/libs/types/src/dtos/index.ts +++ b/libs/types/src/dtos/index.ts @@ -5,6 +5,8 @@ export * from './database/entity.dto'; export * from './directory-watcher/create-directory-watcher.dto'; export * from './directory-watcher/directory-watcher.dto'; export * from './directory-watcher/update-directory-watcher.dto'; +export * from './post/post-queue-action.dto'; +export * from './post/post-queue-record.dto'; export * from './post/post-record.dto'; export * from './post/queue-post-record-request.dto'; export * from './post/website-post-record.dto'; @@ -36,3 +38,4 @@ export * from './website/oauth-website-request.dto'; export * from './website/set-website-data-request.dto'; export * from './website/website-data.dto'; export * from './website/website-info.dto'; + diff --git a/libs/types/src/dtos/post/post-queue-action.dto.ts b/libs/types/src/dtos/post/post-queue-action.dto.ts new file mode 100644 index 000000000..c040bdc46 --- /dev/null +++ b/libs/types/src/dtos/post/post-queue-action.dto.ts @@ -0,0 +1,5 @@ +import { SubmissionId } from '../../models'; + +export type IPostQueueActionDto = { + submissionIds: SubmissionId[]; +}; diff --git a/libs/types/src/dtos/post/post-queue-record.dto.ts b/libs/types/src/dtos/post/post-queue-record.dto.ts new file mode 100644 index 000000000..19e5e1ee5 --- /dev/null +++ b/libs/types/src/dtos/post/post-queue-record.dto.ts @@ -0,0 +1,10 @@ +import { EntityId, IPostQueueRecord, SubmissionId } from '../../models'; +import { IEntityDto } from '../database/entity.dto'; + +export type PostQueueRecordDto = IEntityDto< + Omit +> & { + postRecord?: EntityId; + + submission: SubmissionId; +}; diff --git a/libs/types/src/dtos/post/post-record.dto.ts b/libs/types/src/dtos/post/post-record.dto.ts index 2f35c0e49..73fbda12d 100644 --- a/libs/types/src/dtos/post/post-record.dto.ts +++ b/libs/types/src/dtos/post/post-record.dto.ts @@ -3,9 +3,10 @@ import { IEntityDto } from '../database/entity.dto'; import { WebsitePostRecordDto } from './website-post-record.dto'; export type PostRecordDto = IEntityDto< - Omit + Omit > & { parentId: string; completedAt: string; children: WebsitePostRecordDto[]; + postQueueRecord?: string; }; diff --git a/libs/types/src/dtos/submission/submission.dto.ts b/libs/types/src/dtos/submission/submission.dto.ts index f7d53108c..c0c86fa8c 100644 --- a/libs/types/src/dtos/submission/submission.dto.ts +++ b/libs/types/src/dtos/submission/submission.dto.ts @@ -5,15 +5,19 @@ import { ValidationResult, } from '../../models'; import { IEntityDto } from '../database/entity.dto'; +import { PostQueueRecordDto } from '../post/post-queue-record.dto'; import { PostRecordDto } from '../post/post-record.dto'; import { WebsiteOptionsDto } from '../website-options/website-options.dto'; import { ISubmissionFileDto } from './submission-file.dto'; export type ISubmissionDto< T extends ISubmissionMetadata = ISubmissionMetadata, -> = IEntityDto, 'files' | 'options' | 'posts'>> & { +> = IEntityDto< + Omit, 'files' | 'options' | 'posts' | 'postQueueRecord'> +> & { files: ISubmissionFileDto[]; options: WebsiteOptionsDto[]; posts: PostRecordDto[]; validations: ValidationResult[]; + postQueueRecord?: PostQueueRecordDto; }; diff --git a/libs/types/src/models/index.ts b/libs/types/src/models/index.ts index 8638dbba1..3b1cec0d3 100644 --- a/libs/types/src/models/index.ts +++ b/libs/types/src/models/index.ts @@ -4,6 +4,7 @@ export * from './database/entity.interface'; export * from './directory-watcher/directory-watcher.interface'; export * from './file/file-buffer.interface'; export * from './file/file-dimensions.interface'; +export * from './post/post-queue-record.interface'; export * from './post/post-record.interface'; export * from './post/post-response.type'; export * from './post/website-post-record.interface'; @@ -44,3 +45,4 @@ export * from './website/website-data.interface'; export * from './website/website-info.interface'; export * from './website/website-login-type'; export * from './website/website.type'; + diff --git a/libs/types/src/models/post/post-queue-record.interface.ts b/libs/types/src/models/post/post-queue-record.interface.ts new file mode 100644 index 000000000..28fedbad2 --- /dev/null +++ b/libs/types/src/models/post/post-queue-record.interface.ts @@ -0,0 +1,9 @@ +import { IEntity } from '../database/entity.interface'; +import { ISubmission } from '../submission/submission.interface'; +import { IPostRecord } from './post-record.interface'; + +export interface IPostQueueRecord extends IEntity { + postRecord?: IPostRecord; + + submission: ISubmission; +} diff --git a/libs/types/src/models/post/post-record.interface.ts b/libs/types/src/models/post/post-record.interface.ts index 4e0df7f0f..e084735e8 100644 --- a/libs/types/src/models/post/post-record.interface.ts +++ b/libs/types/src/models/post/post-record.interface.ts @@ -2,6 +2,7 @@ import { Collection, Rel } from '@mikro-orm/core'; import { PostRecordResumeMode, PostRecordState } from '../../enums'; import { IEntity } from '../database/entity.interface'; import { ISubmission } from '../submission/submission.interface'; +import { IPostQueueRecord } from './post-queue-record.interface'; import { IWebsitePostRecord } from './website-post-record.interface'; /** @@ -40,4 +41,10 @@ export interface IPostRecord extends IEntity { * @type {IWebsitePostRecord[]} */ children: Collection; + + /** + * The post queue record associated with the post record. + * @type {IPostQueueRecord} + */ + postQueueRecord?: IPostQueueRecord; } diff --git a/libs/types/src/models/post/website-post-record.interface.ts b/libs/types/src/models/post/website-post-record.interface.ts index 46a1d30c0..42932df6e 100644 --- a/libs/types/src/models/post/website-post-record.interface.ts +++ b/libs/types/src/models/post/website-post-record.interface.ts @@ -63,6 +63,7 @@ export interface IPostRecordMetadata { postedFiles: EntityId[]; /** + * TODO - Ensure this value is saved to the database as post runs. * The next batch number. * More of an internal tracker for resuming posts. * @type {number} diff --git a/libs/types/src/models/settings/settings-options.interface.ts b/libs/types/src/models/settings/settings-options.interface.ts index 92a2b41af..e5694e937 100644 --- a/libs/types/src/models/settings/settings-options.interface.ts +++ b/libs/types/src/models/settings/settings-options.interface.ts @@ -21,4 +21,10 @@ export interface ISettingsOptions { * @type {boolean} */ allowAd: boolean; + + /** + * Whether the queue is paused by the user. + * @type {boolean} + */ + queuePaused: boolean; } diff --git a/libs/types/src/models/submission/submission.interface.ts b/libs/types/src/models/submission/submission.interface.ts index ed8feee9e..50ce879e1 100644 --- a/libs/types/src/models/submission/submission.interface.ts +++ b/libs/types/src/models/submission/submission.interface.ts @@ -1,6 +1,7 @@ import { Collection } from '@mikro-orm/core'; import { SubmissionType } from '../../enums'; import { EntityId, IEntity } from '../database/entity.interface'; +import { IPostQueueRecord } from '../post/post-queue-record.interface'; import { IPostRecord } from '../post/post-record.interface'; import { IWebsiteOptions } from '../website-options/website-options.interface'; import { ISubmissionFile } from './submission-file.interface'; @@ -31,6 +32,12 @@ export interface ISubmission< */ options: Collection>; + /** + * The post queue record associated with the submission. + * @type {IPostQueueRecord} + */ + postQueueRecord?: IPostQueueRecord; + /** * Indicates whether the submission is scheduled. * @type {boolean} diff --git a/package.json b/package.json index 1a735fee9..dedc2dc3f 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@nestjs/websockets": "10.0.2", "@tabler/icons-react": "^3.19.0", "@types/sortablejs": "^1.15.8", + "async-mutex": "^0.5.0", "cheerio": "^1.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", diff --git a/yarn.lock b/yarn.lock index 8bd9cd4ce..82bdfd936 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5446,6 +5446,13 @@ async-exit-hook@^2.0.1: resolved "https://registry.yarnpkg.com/async-exit-hook/-/async-exit-hook-2.0.1.tgz#8bd8b024b0ec9b1c01cccb9af9db29bd717dfaf3" integrity sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw== +async-mutex@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.5.0.tgz#353c69a0b9e75250971a64ac203b0ebfddd75482" + integrity sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA== + dependencies: + tslib "^2.4.0" + async@^2.6.4: version "2.6.4" resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221"