From 418b946703e70cc28f43e5be7e5a8fc48d0eb41b Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 25 Nov 2019 13:38:49 +0000 Subject: [PATCH 01/45] refactor(task-manager): declaritively construct markAvailableTasksAsClaimed query --- .../mark_available_tasks_as_claimed.test.ts | 164 ++++++++++++++++++ .../mark_available_tasks_as_claimed.ts | 77 ++++++++ .../task_manager/queries/query_clauses.ts | 89 ++++++++++ .../legacy/plugins/task_manager/task_store.ts | 119 ++++--------- 4 files changed, 368 insertions(+), 81 deletions(-) create mode 100644 x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts create mode 100644 x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts create mode 100644 x-pack/legacy/plugins/task_manager/queries/query_clauses.ts diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts new file mode 100644 index 0000000000000..060f04c1c18d6 --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import _ from 'lodash'; +import { + asUpdateByQuery, + shouldBeOneOf, + mustBeAllOf, + ExistsBoolClause, + TermBoolClause, + RangeBoolClause, +} from './query_clauses'; + +import { + updateFields, + IdleTaskWithExpiredRunAt, + RunningOrClaimingTaskWithExpiredRetryAt, + RecuringTaskWithInterval, + taskWithLessThanMaxAttempts, + SortByRunAtAndRetryAt, +} from './mark_available_tasks_as_claimed'; + +import { TaskDictionary, TaskDefinition } from '../task'; + +describe('mark_available_tasks_as_claimed', () => { + test('generates query matching tasks to be claimed when polling for tasks', () => { + const definitions: TaskDictionary = { + sampleTask: { + type: 'sampleTask', + title: 'title', + maxAttempts: 5, + createTaskRunner: () => ({ run: () => Promise.resolve() }), + }, + otherTask: { + type: 'otherTask', + title: 'title', + createTaskRunner: () => ({ run: () => Promise.resolve() }), + }, + }; + const defaultMaxAttempts = 1; + const taskManagerId = '3478fg6-82374f6-83467gf5-384g6f'; + const claimOwnershipUntil = '2019-02-12T21:01:22.479Z'; + + expect( + asUpdateByQuery({ + query: mustBeAllOf( + // Either a task with idle status and runAt <= now or + // status running or claiming with a retryAt <= now. + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), + // Either task has an interval or the attempts < the maximum configured + shouldBeOneOf( + RecuringTaskWithInterval, + ...Object.entries(definitions).map(([type, { maxAttempts }]) => + taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) + ) + ) + ), + update: updateFields({ + ownerId: taskManagerId, + status: 'claiming', + retryAt: claimOwnershipUntil, + }), + sort: SortByRunAtAndRetryAt, + }) + ).toEqual({ + query: { + bool: { + must: [ + // Either a task with idle status and runAt <= now or + // status running or claiming with a retryAt <= now. + { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { range: { 'task.runAt': { lte: 'now' } } }, + ], + }, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + }, + }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], + }, + }, + ], + }, + }, + // Either task has an interval or the attempts < the maximum configured + { + bool: { + should: [ + { exists: { field: 'task.interval' } }, + { + bool: { + must: [ + { term: { 'task.taskType': 'sampleTask' } }, + { + range: { + 'task.attempts': { + lt: 5, + }, + }, + }, + ], + }, + }, + { + bool: { + must: [ + { term: { 'task.taskType': 'otherTask' } }, + { + range: { + 'task.attempts': { + lt: 1, + }, + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + sort: { + _script: { + type: 'number', + order: 'asc', + script: { + lang: 'expression', + source: `doc['task.retryAt'].value || doc['task.runAt'].value`, + }, + }, + }, + seq_no_primary_term: true, + script: { + source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + lang: 'painless', + params: { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + status: 'claiming', + }, + }, + }); + }); +}); diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts new file mode 100644 index 0000000000000..63809e9f6d38f --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + BoolClause, + SortClause, + ScriptClause, + ExistsBoolClause, + TermBoolClause, + RangeBoolClause, +} from './query_clauses'; + +export const RecuringTaskWithInterval: ExistsBoolClause = { exists: { field: 'task.interval' } }; +export function taskWithLessThanMaxAttempts( + type: string, + maxAttempts: number +): BoolClause { + return { + bool: { + must: [ + { term: { 'task.taskType': type } }, + { + range: { + 'task.attempts': { + lt: maxAttempts, + }, + }, + }, + ], + }, + }; +} + +export const IdleTaskWithExpiredRunAt: BoolClause = { + bool: { + must: [{ term: { 'task.status': 'idle' } }, { range: { 'task.runAt': { lte: 'now' } } }], + }, +}; + +export const RunningOrClaimingTaskWithExpiredRetryAt: BoolClause< + TermBoolClause | RangeBoolClause +> = { + bool: { + must: [ + { + bool: { + should: [{ term: { 'task.status': 'running' } }, { term: { 'task.status': 'claiming' } }], + }, + }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], + }, +}; + +export const SortByRunAtAndRetryAt: SortClause = { + _script: { + type: 'number', + order: 'asc', + script: { + lang: 'expression', + source: `doc['task.retryAt'].value || doc['task.runAt'].value`, + }, + }, +}; + +export const updateFields = (fieldUpdates: { + [field: string]: string | number | Date; +}): ScriptClause => ({ + source: Object.keys(fieldUpdates) + .map(field => `ctx._source.task.${field}=params.${field};`) + .join(' '), + lang: 'painless', + params: fieldUpdates, +}); diff --git a/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts b/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts new file mode 100644 index 0000000000000..30a8a53da8433 --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface TermBoolClause { + term: { [field: string]: string }; +} +export interface RangeBoolClause { + range: { [field: string]: { lte: string | number } | { lt: string | number } }; +} +export interface ExistsBoolClause { + exists: { field: string }; +} + +export interface ShouldClause { + should: Array | T>; +} +export interface MustClause { + must: Array | T>; +} +export interface BoolClause { + bool: MustClause | ShouldClause; +} +export interface SortClause { + _script: { + type: string; + order: string; + script: { + lang: string; + source: string; + }; + }; +} +export interface ScriptClause { + source: string; + lang: string; + params: { + [field: string]: string | number | Date; + }; +} +export interface UpdateByQuery { + query: BoolClause; + sort: SortClause; + seq_no_primary_term: true; + script: ScriptClause; +} + +export function shouldBeOneOf( + ...should: Array | T> +): { + bool: ShouldClause; +} { + return { + bool: { + should, + }, + }; +} + +export function mustBeAllOf( + ...must: Array | T> +): { + bool: MustClause; +} { + return { + bool: { + must, + }, + }; +} + +export function asUpdateByQuery({ + query, + update, + sort, +}: { + query: BoolClause; + update: ScriptClause; + sort: SortClause; +}): UpdateByQuery { + return { + query, + sort, + seq_no_primary_term: true, + script: update, + }; +} diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 58bffd2269eb6..7feff6a993a0e 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -24,6 +24,24 @@ import { TaskInstance, } from './task'; +import { + asUpdateByQuery, + shouldBeOneOf, + mustBeAllOf, + ExistsBoolClause, + TermBoolClause, + RangeBoolClause, +} from './queries/query_clauses'; + +import { + updateFields, + IdleTaskWithExpiredRunAt, + RunningOrClaimingTaskWithExpiredRetryAt, + RecuringTaskWithInterval, + taskWithLessThanMaxAttempts, + SortByRunAtAndRetryAt, +} from './queries/mark_available_tasks_as_claimed'; + export interface StoreOpts { callCluster: ElasticJs; index: string; @@ -174,87 +192,26 @@ export class TaskStore { claimOwnershipUntil, }: OwnershipClaimingOpts): Promise { const { updated } = await this.updateByQuery( - { - query: { - bool: { - must: [ - // Either a task with idle status and runAt <= now or - // status running or claiming with a retryAt <= now. - { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'task.status': 'idle' } }, - { range: { 'task.runAt': { lte: 'now' } } }, - ], - }, - }, - { - bool: { - must: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - }, - }, - { range: { 'task.retryAt': { lte: 'now' } } }, - ], - }, - }, - ], - }, - }, - // Either task has an interval or the attempts < the maximum configured - { - bool: { - should: [ - { exists: { field: 'task.interval' } }, - ...Object.entries(this.definitions).map(([type, definition]) => ({ - bool: { - must: [ - { term: { 'task.taskType': type } }, - { - range: { - 'task.attempts': { - lt: definition.maxAttempts || this.maxAttempts, - }, - }, - }, - ], - }, - })), - ], - }, - }, - ], - }, - }, - sort: { - _script: { - type: 'number', - order: 'asc', - script: { - lang: 'expression', - source: `doc['task.retryAt'].value || doc['task.runAt'].value`, - }, - }, - }, - seq_no_primary_term: true, - script: { - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, - lang: 'painless', - params: { - ownerId: this.taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', - }, - }, - }, + asUpdateByQuery({ + query: mustBeAllOf( + // Either a task with idle status and runAt <= now or + // status running or claiming with a retryAt <= now. + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), + // Either task has an interval or the attempts < the maximum configured + shouldBeOneOf( + RecuringTaskWithInterval, + ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => + taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) + ) + ) + ), + update: updateFields({ + ownerId: this.taskManagerId, + status: 'claiming', + retryAt: claimOwnershipUntil, + }), + sort: SortByRunAtAndRetryAt, + }), { max_docs: size, } From 3ed64edb1310416ea46e13aadc2a598b48cc4984 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 25 Nov 2019 16:24:45 +0000 Subject: [PATCH 02/45] refactor(task-manager): made TM constructor a little more legible --- .../task_manager/lib/fill_pool.test.ts | 12 ++-- .../plugins/task_manager/lib/fill_pool.ts | 4 +- .../plugins/task_manager/task_manager.ts | 57 ++++++++++--------- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/lib/fill_pool.test.ts b/x-pack/legacy/plugins/task_manager/lib/fill_pool.test.ts index c4927475d586b..3863fdaf9da62 100644 --- a/x-pack/legacy/plugins/task_manager/lib/fill_pool.test.ts +++ b/x-pack/legacy/plugins/task_manager/lib/fill_pool.test.ts @@ -20,7 +20,7 @@ describe('fillPool', () => { const run = sinon.spy(async () => TaskPoolRunResult.RunningAllClaimedTasks); const converter = _.identity; - await fillPool(run, fetchAvailableTasks, converter); + await fillPool(fetchAvailableTasks, converter, run); expect(_.flattenDeep(run.args)).toEqual([1, 2, 3, 4, 5]); }); @@ -35,7 +35,7 @@ describe('fillPool', () => { const run = sinon.spy(async () => TaskPoolRunResult.RanOutOfCapacity); const converter = _.identity; - await fillPool(run, fetchAvailableTasks, converter); + await fillPool(fetchAvailableTasks, converter, run); expect(_.flattenDeep(run.args)).toEqual([1, 2, 3]); }); @@ -50,7 +50,7 @@ describe('fillPool', () => { const run = sinon.spy(async () => TaskPoolRunResult.RanOutOfCapacity); const converter = (x: number) => x.toString(); - await fillPool(run, fetchAvailableTasks, converter); + await fillPool(fetchAvailableTasks, converter, run); expect(_.flattenDeep(run.args)).toEqual(['1', '2', '3']); }); @@ -63,7 +63,7 @@ describe('fillPool', () => { try { const fetchAvailableTasks = async () => Promise.reject('fetch is not working'); - await fillPool(run, fetchAvailableTasks, converter); + await fillPool(fetchAvailableTasks, converter, run); } catch (err) { expect(err.toString()).toBe('fetch is not working'); expect(run.called).toBe(false); @@ -82,7 +82,7 @@ describe('fillPool', () => { let index = 0; const fetchAvailableTasks = async () => tasks[index++] || []; - await fillPool(run, fetchAvailableTasks, converter); + await fillPool(fetchAvailableTasks, converter, run); } catch (err) { expect(err.toString()).toBe('run is not working'); } @@ -101,7 +101,7 @@ describe('fillPool', () => { throw new Error(`can not convert ${x}`); }; - await fillPool(run, fetchAvailableTasks, converter); + await fillPool(fetchAvailableTasks, converter, run); } catch (err) { expect(err.toString()).toBe('Error: can not convert 1'); } diff --git a/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts b/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts index 6fe965e048ea5..f2dc37d3c7fdb 100644 --- a/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts +++ b/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts @@ -29,9 +29,9 @@ type Converter = (t: T1) => T2; * @param converter - a function that converts task records to the appropriate task runner */ export async function fillPool( - run: BatchRun, fetchAvailableTasks: Fetcher, - converter: Converter + converter: Converter, + run: BatchRun ): Promise { performance.mark('fillPool.start'); while (true) { diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 269d7ff67384b..a0ff67ce7d2e5 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -90,7 +90,7 @@ export class TaskManager { this.logger.info(`TaskManager is identified by the Kibana UUID: ${taskManagerId}`); } - const store = new TaskStore({ + this.store = new TaskStore({ serializer: opts.serializer, savedObjectsRepository: opts.savedObjectsRepository, callCluster: opts.callWithInternalUser, @@ -100,38 +100,43 @@ export class TaskManager { taskManagerId: `kibana:${taskManagerId}`, }); - const pool = new TaskPool({ + this.pool = new TaskPool({ logger: this.logger, maxWorkers: this.maxWorkers, }); - const createRunner = (instance: ConcreteTaskInstance) => - new TaskManagerRunner({ - logger: this.logger, - instance, - store, - definitions: this.definitions, - beforeRun: this.middleware.beforeRun, - beforeMarkRunning: this.middleware.beforeMarkRunning, - }); - const poller = new TaskPoller({ + + this.poller = new TaskPoller({ logger: this.logger, pollInterval: opts.config.get('xpack.task_manager.poll_interval'), - work: (): Promise => - fillPool( - async tasks => await pool.run(tasks), - () => - claimAvailableTasks( - this.store.claimAvailableTasks.bind(this.store), - this.pool.availableWorkers, - this.logger - ), - createRunner - ), + work: this.work.bind(this), }); + } - this.pool = pool; - this.store = store; - this.poller = poller; + private work(): Promise { + return fillPool( + // claim available tasks + () => + claimAvailableTasks( + this.store.claimAvailableTasks.bind(this.store), + this.pool.availableWorkers, + this.logger + ), + // wrap each task in a Task Runner + this.createTaskRunnerForTask.bind(this), + // place tasks in the Task Pool + async tasks => await this.pool.run(tasks) + ); + } + + private createTaskRunnerForTask(instance: ConcreteTaskInstance) { + return new TaskManagerRunner({ + logger: this.logger, + instance, + store: this.store, + definitions: this.definitions, + beforeRun: this.middleware.beforeRun, + beforeMarkRunning: this.middleware.beforeMarkRunning, + }); } /** From 542735082d4343e404512889f5f01069fe7dc5fb Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 27 Nov 2019 12:31:02 +0000 Subject: [PATCH 03/45] feat(run-now): Added TaskEvent when tasks are claimed in TaskStore --- .../plugins/task_manager/lib/result_type.ts | 50 +++ .../mark_available_tasks_as_claimed.ts | 28 +- .../task_manager/queries/query_clauses.ts | 3 +- .../plugins/task_manager/task_events.ts | 66 ++++ .../plugins/task_manager/task_manager.ts | 42 +- .../plugins/task_manager/task_store.test.ts | 360 ++++++++++++++++++ .../legacy/plugins/task_manager/task_store.ts | 103 +++-- 7 files changed, 603 insertions(+), 49 deletions(-) create mode 100644 x-pack/legacy/plugins/task_manager/lib/result_type.ts create mode 100644 x-pack/legacy/plugins/task_manager/task_events.ts diff --git a/x-pack/legacy/plugins/task_manager/lib/result_type.ts b/x-pack/legacy/plugins/task_manager/lib/result_type.ts new file mode 100644 index 0000000000000..256463251315d --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/lib/result_type.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// There appears to be an unexported implementation of Either in here: src/core/server/saved_objects/service/lib/repository.ts +// Which is basically the Haskel equivalent of Rust/ML/Scala's Result +// I'll reach out to other's in Kibana to see if we can merge these into one type + +export interface Ok { + tag: 'ok'; + value: T; +} + +export interface Err { + tag: 'err'; + error: E; +} +export type Result = Ok | Err; + +export function asOk(value: T): Ok { + return { + tag: 'ok', + value, + }; +} + +export function asErr(error: T): Err { + return { + tag: 'err', + error, + }; +} + +export function isOk(result: Result): result is Ok { + return result.tag === 'ok'; +} + +export function isErr(result: Result): result is Err { + return !isOk(result); +} + +export async function promiseResult(future: Promise): Promise> { + try { + return asOk(await future); + } catch (e) { + return asErr(e); + } +} diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index 63809e9f6d38f..b40bc4cbe51fb 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import { defaultsDeep } from 'lodash'; import { BoolClause, SortClause, @@ -40,6 +40,12 @@ export const IdleTaskWithExpiredRunAt: BoolClause => ({ + bool: { + must: [{ term: { 'task.status': 'idle' } }, { term: { 'task.id': claimTasksById } }], + }, +}); + export const RunningOrClaimingTaskWithExpiredRetryAt: BoolClause< TermBoolClause | RangeBoolClause > = { @@ -66,6 +72,26 @@ export const SortByRunAtAndRetryAt: SortClause = { }, }; +const SORT_VALUE_TO_BE_FIRST = 0; +export const sortByIdsThenByScheduling = (claimTasksById: string[]): SortClause => { + const { + _script: { + script: { source }, + }, + } = SortByRunAtAndRetryAt; + return defaultsDeep( + { + _script: { + script: { + source: `params.ids.contains(doc['task.id']) ? ${SORT_VALUE_TO_BE_FIRST} : (${source})`, + params: { ids: claimTasksById }, + }, + }, + }, + SortByRunAtAndRetryAt + ); +}; + export const updateFields = (fieldUpdates: { [field: string]: string | number | Date; }): ScriptClause => ({ diff --git a/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts b/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts index 30a8a53da8433..ca86354f0f9e4 100644 --- a/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts +++ b/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts @@ -5,7 +5,7 @@ */ export interface TermBoolClause { - term: { [field: string]: string }; + term: { [field: string]: string | string[] }; } export interface RangeBoolClause { range: { [field: string]: { lte: string | number } | { lt: string | number } }; @@ -30,6 +30,7 @@ export interface SortClause { script: { lang: string; source: string; + params?: { [param: string]: string | string[] }; }; }; } diff --git a/x-pack/legacy/plugins/task_manager/task_events.ts b/x-pack/legacy/plugins/task_manager/task_events.ts new file mode 100644 index 0000000000000..8d0fbf00ed065 --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/task_events.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ConcreteTaskInstance } from './task'; + +import { Result } from './lib/result_type'; + +export type TaskRun = Result; +export type TaskRunNow = Result; +export type TaskClaim = Result; + +export enum TaskEventType { + TASK_RUN_NOW, + TASK_RUN, + TASK_CLAIM, +} + +export type TaskEvent = { + id: string; +} & ( + | { + type: TaskEventType.TASK_RUN; + event: TaskRun; + } + | { + type: TaskEventType.TASK_CLAIM; + event: TaskClaim; + } + | { + type: TaskEventType.TASK_RUN_NOW; + event: TaskRunNow; + } +); + +export function asTaskRunEvent( + id: string, + event: Result +): TaskEvent { + return { + id, + type: TaskEventType.TASK_RUN, + event, + }; +} + +export function asTaskRunNowEvent(id: string, event: Result): TaskEvent { + return { + id, + type: TaskEventType.TASK_RUN_NOW, + event, + }; +} + +export function asTaskClaimEvent( + id: string, + event: Result +): TaskEvent { + return { + id, + type: TaskEventType.TASK_CLAIM, + event, + }; +} diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index a0ff67ce7d2e5..2bc9d70282c9a 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -3,9 +3,12 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { Subject } from 'rxjs'; +import { partial } from 'lodash'; import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; import { Logger } from './types'; +import { TaskEvent } from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { addMiddlewareToChain, BeforeSaveMiddlewareParams, Middleware } from './lib/middleware'; import { sanitizeTaskDefinitions } from './lib/sanitize_task_definitions'; @@ -20,7 +23,7 @@ import { } from './task'; import { TaskPoller } from './task_poller'; import { TaskPool } from './task_pool'; -import { TaskManagerRunner } from './task_runner'; +import { TaskManagerRunner, TaskRunner } from './task_runner'; import { FetchOpts, FetchResult, @@ -62,6 +65,7 @@ export class TaskManager { private poller: TaskPoller; private logger: Logger; private pool: TaskPool; + private taskEvent$: Subject; private startQueue: Array<() => void> = []; private middleware = { beforeSave: async (saveOpts: BeforeSaveMiddlewareParams) => saveOpts, @@ -79,6 +83,7 @@ export class TaskManager { this.pollerInterval = opts.config.get('xpack.task_manager.poll_interval'); this.definitions = {}; this.logger = opts.logger; + this.taskEvent$ = new Subject(); const taskManagerId = opts.config.get('server.uuid'); if (!taskManagerId) { @@ -108,27 +113,24 @@ export class TaskManager { this.poller = new TaskPoller({ logger: this.logger, pollInterval: opts.config.get('xpack.task_manager.poll_interval'), - work: this.work.bind(this), + work: partial( + fillPool, + // claim available tasks + () => + claimAvailableTasks( + this.store.claimAvailableTasks, + this.pool.availableWorkers, + this.logger + ), + // wrap each task in a Task Runner + this.createTaskRunnerForTask, + // place tasks in the Task Pool + async (tasks: TaskRunner[]) => await this.pool.run(tasks) + ), }); } - private work(): Promise { - return fillPool( - // claim available tasks - () => - claimAvailableTasks( - this.store.claimAvailableTasks.bind(this.store), - this.pool.availableWorkers, - this.logger - ), - // wrap each task in a Task Runner - this.createTaskRunnerForTask.bind(this), - // place tasks in the Task Pool - async tasks => await this.pool.run(tasks) - ); - } - - private createTaskRunnerForTask(instance: ConcreteTaskInstance) { + private createTaskRunnerForTask = (instance: ConcreteTaskInstance) => { return new TaskManagerRunner({ logger: this.logger, instance, @@ -137,7 +139,7 @@ export class TaskManager { beforeRun: this.middleware.beforeRun, beforeMarkRunning: this.middleware.beforeMarkRunning, }); - } + }; /** * Starts up the task manager and starts picking up tasks. diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index 46efc4bb57ba7..bfc78c7f9bedc 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -7,10 +7,14 @@ import _ from 'lodash'; import sinon from 'sinon'; import uuid from 'uuid'; +import { filter } from 'rxjs/operators'; + import { TaskDictionary, TaskDefinition, TaskInstance, TaskStatus } from './task'; import { FetchOpts, StoreOpts, OwnershipClaimingOpts, TaskStore } from './task_store'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { SavedObjectsSerializer, SavedObjectsSchema, SavedObjectAttributes } from 'src/core/server'; +import { asTaskClaimEvent, TaskEvent } from './task_events'; +import { asOk, asErr } from './lib/result_type'; const taskDefinitions: TaskDictionary = { report: { @@ -505,6 +509,156 @@ describe('TaskStore', () => { }); }); + test('it supports claiming specific tasks by id', async () => { + const maxAttempts = _.random(2, 43); + const customMaxAttempts = _.random(44, 100); + const { + args: { + updateByQuery: { + body: { query, sort }, + }, + }, + } = await testClaimAvailableTasks({ + opts: { + maxAttempts, + definitions: { + foo: { + type: 'foo', + title: '', + createTaskRunner: jest.fn(), + }, + bar: { + type: 'bar', + title: '', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, + }, + }, + claimingOpts: { + claimOwnershipUntil: new Date(), + size: 10, + claimTasksById: [ + '33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], + }, + }); + + expect(query).toMatchObject({ + bool: { + must: [ + { term: { type: 'task' } }, + { + bool: { + should: [ + { + bool: { + must: [ + { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { range: { 'task.runAt': { lte: 'now' } } }, + ], + }, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + }, + }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], + }, + }, + ], + }, + }, + { + bool: { + should: [ + { exists: { field: 'task.interval' } }, + { + bool: { + must: [ + { term: { 'task.taskType': 'foo' } }, + { + range: { + 'task.attempts': { + lt: maxAttempts, + }, + }, + }, + ], + }, + }, + { + bool: { + must: [ + { term: { 'task.taskType': 'bar' } }, + { + range: { + 'task.attempts': { + lt: customMaxAttempts, + }, + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { + term: { + 'task.id': [ + '33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }); + + expect(sort).toMatchObject({ + _script: { + type: 'number', + order: 'asc', + script: { + lang: 'expression', + source: `params.ids.contains(doc['task.id']) ? 0 : (doc['task.retryAt'].value || doc['task.runAt'].value)`, + params: { + ids: ['33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', 'a208b22c-14ec-4fb4-995f-d2ff7a3b03b8'], + }, + }, + }, + }); + }); + test('it claims tasks by setting their ownerId, status and retryAt', async () => { const taskManagerId = uuid.v1(); const claimOwnershipUntil = new Date(Date.now()); @@ -736,6 +890,212 @@ describe('TaskStore', () => { expect(savedObjectsClient.delete).toHaveBeenCalledWith('task', id); }); }); + + describe('task events', () => { + function generateTasks() { + const taskManagerId = uuid.v1(); + const runAt = new Date(); + const tasks = [ + { + _id: 'aaa', + _source: { + type: 'task', + task: { + runAt, + taskType: 'foo', + interval: undefined, + attempts: 0, + status: 'idle', + params: '{ "hello": "world" }', + state: '{ "baby": "Henhen" }', + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + }, + }, + _seq_no: 1, + _primary_term: 2, + sort: ['a', 1], + }, + { + _id: 'bbb', + _source: { + type: 'task', + task: { + runAt, + taskType: 'bar', + interval: '5m', + attempts: 2, + status: 'running', + params: '{ "shazm": 1 }', + state: '{ "henry": "The 8th" }', + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + }, + }, + _seq_no: 3, + _primary_term: 4, + sort: ['b', 2], + }, + ]; + + return { taskManagerId, runAt, tasks }; + } + + test('emits an event when a task is succesfully claimed by id', async done => { + const { taskManagerId, runAt, tasks } = generateTasks(); + const callCluster = sinon.spy(async (name: string, params?: any) => + name === 'updateByQuery' + ? { + total: tasks.length, + updated: tasks.length, + } + : { hits: { hits: tasks } } + ); + const store = new TaskStore({ + callCluster, + maxAttempts: 2, + definitions: taskDefinitions, + serializer, + savedObjectsRepository: savedObjectsClient, + taskManagerId, + index: '', + }); + + const sub = store.events.pipe(filter((event: TaskEvent) => event.id === 'aaa')).subscribe({ + next: (event: TaskEvent) => { + expect(event).toMatchObject( + asTaskClaimEvent( + 'aaa', + asOk({ + id: 'aaa', + runAt, + taskType: 'foo', + interval: undefined, + attempts: 0, + status: 'idle' as TaskStatus, + params: { hello: 'world' }, + state: { baby: 'Henhen' }, + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + }) + ) + ); + sub.unsubscribe(); + done(); + }, + }); + + await store.claimAvailableTasks({ + claimTasksById: ['aaa'], + claimOwnershipUntil: new Date(), + size: 10, + }); + }); + + test('emits an event when a task is succesfully by scheduling', async done => { + const { taskManagerId, runAt, tasks } = generateTasks(); + const callCluster = sinon.spy(async (name: string, params?: any) => + name === 'updateByQuery' + ? { + total: tasks.length, + updated: tasks.length, + } + : { hits: { hits: tasks } } + ); + const store = new TaskStore({ + callCluster, + maxAttempts: 2, + definitions: taskDefinitions, + serializer, + savedObjectsRepository: savedObjectsClient, + taskManagerId, + index: '', + }); + + const sub = store.events.pipe(filter((event: TaskEvent) => event.id === 'bbb')).subscribe({ + next: (event: TaskEvent) => { + expect(event).toMatchObject( + asTaskClaimEvent( + 'bbb', + asOk({ + id: 'bbb', + runAt, + taskType: 'bar', + interval: '5m', + attempts: 2, + status: 'running' as TaskStatus, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + }) + ) + ); + sub.unsubscribe(); + done(); + }, + }); + + await store.claimAvailableTasks({ + claimTasksById: ['aaa'], + claimOwnershipUntil: new Date(), + size: 10, + }); + }); + + test('emits an event when the store fails to claim a required task by id', async done => { + const { taskManagerId, tasks } = generateTasks(); + const callCluster = sinon.spy(async (name: string, params?: any) => + name === 'updateByQuery' + ? { + total: tasks.length, + updated: tasks.length, + } + : { hits: { hits: tasks } } + ); + const store = new TaskStore({ + callCluster, + maxAttempts: 2, + definitions: taskDefinitions, + serializer, + savedObjectsRepository: savedObjectsClient, + taskManagerId, + index: '', + }); + + const sub = store.events.pipe(filter((event: TaskEvent) => event.id === 'ccc')).subscribe({ + next: (event: TaskEvent) => { + expect(event).toMatchObject( + asTaskClaimEvent('ccc', asErr(new Error(`failed to claim task 'ccc'`))) + ); + sub.unsubscribe(); + done(); + }, + }); + + await store.claimAvailableTasks({ + claimTasksById: ['ccc'], + claimOwnershipUntil: new Date(), + size: 10, + }); + }); + }); }); function generateFakeTasks(count: number = 1) { diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 7feff6a993a0e..4982c6de2333a 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -7,8 +7,9 @@ /* * This module contains helpers for managing the task manager storage layer. */ +import { Subject, Observable } from 'rxjs'; +import { omit, difference, indexBy } from 'lodash'; -import { omit } from 'lodash'; import { SavedObjectsClientContract, SavedObject, @@ -16,6 +17,9 @@ import { SavedObjectsSerializer, SavedObjectsRawDoc, } from 'src/core/server'; + +import { asOk, asErr } from './lib/result_type'; + import { ConcreteTaskInstance, ElasticJs, @@ -24,6 +28,8 @@ import { TaskInstance, } from './task'; +import { TaskEvent, asTaskClaimEvent } from './task_events'; + import { asUpdateByQuery, shouldBeOneOf, @@ -40,6 +46,8 @@ import { RecuringTaskWithInterval, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, + idleTaskWithIDs, + sortByIdsThenByScheduling, } from './queries/mark_available_tasks_as_claimed'; export interface StoreOpts { @@ -50,6 +58,7 @@ export interface StoreOpts { definitions: TaskDictionary; savedObjectsRepository: SavedObjectsClientContract; serializer: SavedObjectsSerializer; + onEvent?: (event: TaskEvent) => void; } export interface SearchOpts { @@ -75,6 +84,7 @@ export interface UpdateByQueryOpts extends SearchOpts { export interface OwnershipClaimingOpts { claimOwnershipUntil: Date; + claimTasksById?: string[]; size: number; } @@ -107,10 +117,13 @@ export class TaskStore { public readonly maxAttempts: number; public readonly index: string; public readonly taskManagerId: string; + private callCluster: ElasticJs; private definitions: TaskDictionary; private savedObjectsRepository: SavedObjectsClientContract; private serializer: SavedObjectsSerializer; + private events$: Subject; + private onEvent: (event: TaskEvent) => void; /** * Constructs a new TaskStore. @@ -130,8 +143,18 @@ export class TaskStore { this.definitions = opts.definitions; this.serializer = opts.serializer; this.savedObjectsRepository = opts.savedObjectsRepository; + this.onEvent = opts.onEvent || ((e: TaskEvent) => {}); + this.events$ = new Subject(); + } + + public get events(): Observable { + return this.events$; } + private emitEvent = (event: TaskEvent) => { + this.events$.next(event); + }; + /** * Schedules a task. * @@ -178,39 +201,70 @@ export class TaskStore { * @param {OwnershipClaimingOpts} options * @returns {Promise} */ - public async claimAvailableTasks(opts: OwnershipClaimingOpts): Promise { + public claimAvailableTasks = async ( + opts: OwnershipClaimingOpts + ): Promise => { + const { claimTasksById } = opts; + const claimedTasks = await this.markAvailableTasksAsClaimed(opts); const docs = claimedTasks > 0 ? await this.sweepForClaimedTasks(opts) : []; + + // emit success/fail events for claimed tasks by id + if (claimTasksById && claimTasksById.length) { + docs.map(doc => asTaskClaimEvent(doc.id, asOk(doc))).forEach(this.emitEvent); + + difference( + claimTasksById, + docs.map(doc => doc.id) + ) + .map(id => asTaskClaimEvent(id, asErr(new Error(`failed to claim task '${id}'`)))) + .forEach(this.emitEvent); + } + return { claimedTasks, docs, }; - } + }; private async markAvailableTasksAsClaimed({ size, claimOwnershipUntil, + claimTasksById, }: OwnershipClaimingOpts): Promise { + const queryForScheduledTasks = mustBeAllOf( + // Either a task with idle status and runAt <= now or + // status running or claiming with a retryAt <= now. + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), + // Either task has an interval or the attempts < the maximum configured + shouldBeOneOf( + RecuringTaskWithInterval, + ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => + taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) + ) + ) + ); + + const { query, sort } = + claimTasksById && claimTasksById.length + ? { + query: shouldBeOneOf(queryForScheduledTasks, idleTaskWithIDs(claimTasksById)), + sort: sortByIdsThenByScheduling(claimTasksById), + } + : { + query: queryForScheduledTasks, + sort: SortByRunAtAndRetryAt, + }; + const { updated } = await this.updateByQuery( asUpdateByQuery({ - query: mustBeAllOf( - // Either a task with idle status and runAt <= now or - // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an interval or the attempts < the maximum configured - shouldBeOneOf( - RecuringTaskWithInterval, - ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) - ) - ) - ), + query, update: updateFields({ ownerId: this.taskManagerId, status: 'claiming', retryAt: claimOwnershipUntil, }), - sort: SortByRunAtAndRetryAt, + sort, }), { max_docs: size, @@ -224,6 +278,7 @@ export class TaskStore { */ private async sweepForClaimedTasks({ size, + claimTasksById, }: OwnershipClaimingOpts): Promise { const { docs } = await this.search({ query: { @@ -239,16 +294,10 @@ export class TaskStore { }, }, size, - sort: { - _script: { - type: 'number', - order: 'asc', - script: { - lang: 'expression', - source: `doc['task.retryAt'].value || doc['task.runAt'].value`, - }, - }, - }, + sort: + claimTasksById && claimTasksById.length + ? sortByIdsThenByScheduling(claimTasksById) + : SortByRunAtAndRetryAt, seq_no_primary_term: true, }); @@ -363,7 +412,7 @@ function taskInstanceToAttributes(doc: TaskInstance): SavedObjectAttributes { }; } -function savedObjectToConcreteTaskInstance( +export function savedObjectToConcreteTaskInstance( savedObject: Omit ): ConcreteTaskInstance { return { From 9041b18b542f1a789278e3704b8de3c8447d2c9a Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 28 Nov 2019 13:05:31 +0000 Subject: [PATCH 04/45] feat(run-now): Use streams in TaskPoller to make it easier to push values in async --- .../plugins/task_manager/task_manager.ts | 69 +++--- .../task_poller.intervals.test.ts | 40 ++++ .../plugins/task_manager/task_poller.test.ts | 199 +++++++++--------- .../plugins/task_manager/task_poller.ts | 91 +++----- .../legacy/plugins/task_manager/task_store.ts | 5 +- x-pack/package.json | 1 + yarn.lock | 12 ++ 7 files changed, 219 insertions(+), 198 deletions(-) create mode 100644 x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 2bc9d70282c9a..9747aaa4ad210 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { Subject } from 'rxjs'; -import { partial } from 'lodash'; +import { Option, none } from 'fp-ts/lib/Option'; import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; import { Logger } from './types'; @@ -59,13 +59,16 @@ export interface TaskManagerOpts { export class TaskManager { private isStarted = false; private maxWorkers: number; - private readonly pollerInterval: number; private definitions: TaskDictionary; private store: TaskStore; - private poller: TaskPoller; + // private poller: TaskPoller; private logger: Logger; private pool: TaskPool; + private taskEvent$: Subject; + + private poller: TaskPoller>; + private startQueue: Array<() => void> = []; private middleware = { beforeSave: async (saveOpts: BeforeSaveMiddlewareParams) => saveOpts, @@ -80,7 +83,6 @@ export class TaskManager { */ constructor(opts: TaskManagerOpts) { this.maxWorkers = opts.config.get('xpack.task_manager.max_workers'); - this.pollerInterval = opts.config.get('xpack.task_manager.poll_interval'); this.definitions = {}; this.logger = opts.logger; this.taskEvent$ = new Subject(); @@ -110,26 +112,33 @@ export class TaskManager { maxWorkers: this.maxWorkers, }); - this.poller = new TaskPoller({ + this.poller = new TaskPoller>({ logger: this.logger, pollInterval: opts.config.get('xpack.task_manager.poll_interval'), - work: partial( - fillPool, - // claim available tasks - () => - claimAvailableTasks( - this.store.claimAvailableTasks, - this.pool.availableWorkers, - this.logger - ), - // wrap each task in a Task Runner - this.createTaskRunnerForTask, - // place tasks in the Task Pool - async (tasks: TaskRunner[]) => await this.pool.run(tasks) - ), + work: this.pollForWork, }); } + private pollForWork = async (): Promise => { + return fillPool( + // claim available tasks + () => + claimAvailableTasks( + this.store.claimAvailableTasks, + this.pool.availableWorkers, + this.logger + ), + // wrap each task in a Task Runner + this.createTaskRunnerForTask, + // place tasks in the Task Pool + async (tasks: TaskRunner[]) => await this.pool.run(tasks) + ); + }; + + private attemptWork() { + this.poller.queueWork(none); + } + private createTaskRunnerForTask = (instance: ConcreteTaskInstance) => { return new TaskManagerRunner({ logger: this.logger, @@ -145,24 +154,16 @@ export class TaskManager { * Starts up the task manager and starts picking up tasks. */ public start() { + if (this.isStarted) { + return; + } this.isStarted = true; + // Some calls are waiting until task manager is started this.startQueue.forEach(fn => fn()); this.startQueue = []; - const startPoller = async () => { - try { - await this.poller.start(); - } catch (err) { - // FIXME: check the type of error to make sure it's actually an ES error - this.logger.warn(`PollError ${err.message}`); - - // rety again to initialize store and poller, using the timing of - // task_manager's configurable poll interval - const retryInterval = this.pollerInterval; - setTimeout(() => startPoller(), retryInterval); - } - }; - startPoller(); + + this.poller.start(); } private async waitUntilStarted() { @@ -225,7 +226,7 @@ export class TaskManager { taskInstance, }); const result = await this.store.schedule(modifiedTask); - this.poller.attemptWork(); + this.attemptWork(); return result; } diff --git a/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts new file mode 100644 index 0000000000000..adcf2daa903f7 --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import _ from 'lodash'; +import { Subject } from 'rxjs'; +import { TaskPoller } from './task_poller'; +import { mockLogger, resolvable } from './test_utils'; +import { fakeSchedulers } from 'rxjs-marbles/jest'; + +describe('TaskPoller Intervals', () => { + beforeEach(() => jest.useFakeTimers()); + test( + 'runs the work function on an interval', + fakeSchedulers(async advance => { + jest.useFakeTimers(); + const pollInterval = _.random(10, 20); + const done = resolvable(); + const work = jest.fn(async () => { + done.resolve(); + return true; + }); + const poller = new TaskPoller({ + pollInterval, + work, + logger: mockLogger(), + }); + poller.start(); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(1); + await done; + advance(pollInterval - 1); + expect(work).toHaveBeenCalledTimes(1); + advance(1); + expect(work).toHaveBeenCalledTimes(2); + }) + ); +}); diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts index 88bcf29ec6084..f14402d75c145 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts @@ -5,138 +5,131 @@ */ import _ from 'lodash'; -import sinon from 'sinon'; +import { Subject } from 'rxjs'; import { TaskPoller } from './task_poller'; import { mockLogger, resolvable, sleep } from './test_utils'; describe('TaskPoller', () => { - beforeEach(() => { - const callCluster = sinon.stub(); - callCluster.withArgs('indices.getTemplate').returns(Promise.resolve({ tasky: {} })); - }); - - describe('interval tests', () => { - let clock: sinon.SinonFakeTimers; + describe('lifecycle', () => { + test('logs, but does not crash if the work function fails', async done => { + const logger = mockLogger(); - beforeEach(() => { - clock = sinon.useFakeTimers(); - }); + let count = 0; + const work = jest.fn(async () => { + ++count; + if (count === 1) { + throw new Error('Dang it!'); + } else if (count > 1) { + poller.stop(); - afterEach(() => clock.restore()); + expect(work).toHaveBeenCalledTimes(2); + expect(logger.error.mock.calls[0][0]).toMatchInlineSnapshot( + `"Failed to poll for work: Error: Dang it!"` + ); - test('runs the work function on an interval', async () => { - const pollInterval = _.random(10, 20); - const done = resolvable(); - const work = sinon.spy(() => { - done.resolve(); - return Promise.resolve(); + done(); + } }); - const poller = new TaskPoller({ - pollInterval, + + const poller = new TaskPoller({ + logger, + pollInterval: 1, work, - logger: mockLogger(), }); - await poller.start(); + poller.start(); + }); - sinon.assert.calledOnce(work); - await done; + test('is stoppable', async () => { + const doneWorking = resolvable(); + const work = jest.fn(async () => { + poller.stop(); + doneWorking.resolve(); + }); - clock.tick(pollInterval - 1); - sinon.assert.calledOnce(work); - clock.tick(1); - sinon.assert.calledTwice(work); - }); - }); + const poller = new TaskPoller({ + logger: mockLogger(), + pollInterval: 1, + work, + }); - test('logs, but does not crash if the work function fails', async () => { - let count = 0; - const logger = mockLogger(); - const doneWorking = resolvable(); - const poller = new TaskPoller({ - logger, - pollInterval: 1, - work: async () => { - ++count; - if (count === 1) { - throw new Error('Dang it!'); - } - if (count > 1) { - poller.stop(); - doneWorking.resolve(); - } - }, + poller.start(); + await doneWorking; + expect(work).toHaveBeenCalledTimes(1); + await sleep(100); + expect(work).toHaveBeenCalledTimes(1); }); - poller.start(); - - await doneWorking; + test('disregards duplicate calls to "start"', async () => { + const doneWorking = resolvable(); + const work = jest.fn(async () => { + await doneWorking; + }); + const poller = new TaskPoller({ + pollInterval: 1, + logger: mockLogger(), + work, + }); - expect(count).toEqual(2); - expect(logger.error.mock.calls[0][0]).toMatchInlineSnapshot( - `"Failed to poll for work: Error: Dang it!"` - ); - }); + poller.start(); + await sleep(10); + poller.start(); + poller.start(); + await sleep(10); + poller.start(); + await sleep(10); - test('is stoppable', async () => { - const doneWorking = resolvable(); - const work = sinon.spy(async () => { poller.stop(); + doneWorking.resolve(); - }); - const poller = new TaskPoller({ - logger: mockLogger(), - pollInterval: 1, - work, - }); + await sleep(10); - poller.start(); - await doneWorking; - await sleep(10); + expect(work).toHaveBeenCalledTimes(1); + }); - sinon.assert.calledOnce(work); - }); + test('waits for work before polling', async () => { + const doneWorking = resolvable(); + const work = jest.fn(async () => { + await sleep(10); + poller.stop(); + doneWorking.resolve(); + }); + const poller = new TaskPoller({ + pollInterval: 1, + logger: mockLogger(), + work, + }); - test('disregards duplicate calls to "start"', async () => { - const doneWorking = resolvable(); - const work = sinon.spy(async () => { + poller.start(); await doneWorking; - }); - const poller = new TaskPoller({ - pollInterval: 1, - logger: mockLogger(), - work, - }); - - await poller.start(); - poller.start(); - poller.start(); - poller.start(); - poller.stop(); + expect(work).toHaveBeenCalledTimes(1); + }); - doneWorking.resolve(); + test('queues claim requests while working', async done => { + let count = 0; - sinon.assert.calledOnce(work); - }); + const poller = new TaskPoller({ + pollInterval: 1, + logger: mockLogger(), + work: jest.fn(async (first, second) => { + count++; + if (count === 1) { + poller.queueWork('asd'); + poller.queueWork('123'); + } else if (count === 2) { + expect(first).toEqual('asd'); + expect(second).toEqual('123'); + + done(); + } else { + poller.stop(); + } + }), + }); - test('waits for work before polling', async () => { - const doneWorking = resolvable(); - const work = sinon.spy(async () => { - await sleep(10); - poller.stop(); - doneWorking.resolve(); + poller.start(); }); - const poller = new TaskPoller({ - pollInterval: 1, - logger: mockLogger(), - work, - }); - - poller.start(); - await doneWorking; - - sinon.assert.calledOnce(work); }); }); diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 0f7b49f17872a..c1610f66aa96a 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -8,28 +8,30 @@ * This module contains the logic for polling the task manager index for new work. */ -import { performance } from 'perf_hooks'; +import { Subject, Subscription, Observable, interval } from 'rxjs'; +import { buffer, throttle } from 'rxjs/operators'; +import { Result, asOk, asErr } from './lib/result_type'; import { Logger } from './types'; -type WorkFn = () => Promise; +type WorkFn = (...params: H[]) => Promise; -interface Opts { +interface Opts { pollInterval: number; logger: Logger; - work: WorkFn; + work: WorkFn; } /** * Performs work on a scheduled interval, logging any errors. This waits for work to complete * (or error) prior to attempting another run. */ -export class TaskPoller { - private isStarted = false; - private isWorking = false; - private timeout: any; - private pollInterval: number; +export class TaskPoller { private logger: Logger; - private work: WorkFn; + private work: WorkFn; + private poller$: Observable; + private pollPhaseResults$: Subject>; + private claimRequestQueue$: Subject; + private pollingSubscription: Subscription; /** * Constructs a new TaskPoller. @@ -39,77 +41,52 @@ export class TaskPoller { * @prop {Logger} logger - The task manager logger * @prop {WorkFn} work - An empty, asynchronous function that performs the desired work */ - constructor(opts: Opts) { - this.pollInterval = opts.pollInterval; + constructor(opts: Opts) { this.logger = opts.logger; this.work = opts.work; + + this.pollingSubscription = Subscription.EMPTY; + this.pollPhaseResults$ = new Subject(); + this.claimRequestQueue$ = new Subject(); + this.poller$ = this.claimRequestQueue$.pipe( + buffer(interval(opts.pollInterval).pipe(throttle(ev => this.pollPhaseResults$))) + ); } /** * Starts the poller. If the poller is already running, this has no effect. */ public async start() { - if (this.isStarted) { + if (this.pollingSubscription && !this.pollingSubscription.closed) { return; } - this.isStarted = true; - - const poll = async () => { - await this.attemptWork(); - - performance.mark('TaskPoller.sleep'); - if (this.isStarted) { - this.timeout = setTimeout( - tryAndLogOnError(() => { - performance.mark('TaskPoller.poll'); - performance.measure('TaskPoller.sleepDuration', 'TaskPoller.sleep', 'TaskPoller.poll'); - poll(); - }, this.logger), - this.pollInterval - ); - } - }; - - poll(); + this.pollingSubscription = this.poller$.subscribe(requests => { + // console.log({ requests }); + this.attemptWork(...requests); + }); } /** * Stops the poller. */ public stop() { - this.isStarted = false; - clearTimeout(this.timeout); - this.timeout = undefined; + this.pollingSubscription.unsubscribe(); + } + + public queueWork(request: H) { + this.claimRequestQueue$.next(request); } /** - * Runs the work function. If the work function is currently running, - * this has no effect. + * Runs the work function, this is called in respose to the polling stream */ - public async attemptWork() { - if (!this.isStarted || this.isWorking) { - return; - } - - this.isWorking = true; - + private attemptWork = async (...requests: H[]) => { try { - await this.work(); + this.pollPhaseResults$.next(asOk(await this.work(...requests))); } catch (err) { this.logger.error(`Failed to poll for work: ${err}`); - } finally { - this.isWorking = false; - } - } -} - -function tryAndLogOnError(fn: Function, logger: Logger): Function { - return () => { - try { - fn(); - } catch (err) { - logger.error(`Task Poller polling phase failed: ${err}`); + this.pollPhaseResults$.next(asErr(err)); } }; } diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 4982c6de2333a..a4456e4c89c87 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -8,7 +8,7 @@ * This module contains helpers for managing the task manager storage layer. */ import { Subject, Observable } from 'rxjs'; -import { omit, difference, indexBy } from 'lodash'; +import { omit, difference } from 'lodash'; import { SavedObjectsClientContract, @@ -58,7 +58,6 @@ export interface StoreOpts { definitions: TaskDictionary; savedObjectsRepository: SavedObjectsClientContract; serializer: SavedObjectsSerializer; - onEvent?: (event: TaskEvent) => void; } export interface SearchOpts { @@ -123,7 +122,6 @@ export class TaskStore { private savedObjectsRepository: SavedObjectsClientContract; private serializer: SavedObjectsSerializer; private events$: Subject; - private onEvent: (event: TaskEvent) => void; /** * Constructs a new TaskStore. @@ -143,7 +141,6 @@ export class TaskStore { this.definitions = opts.definitions; this.serializer = opts.serializer; this.savedObjectsRepository = opts.savedObjectsRepository; - this.onEvent = opts.onEvent || ((e: TaskEvent) => {}); this.events$ = new Subject(); } diff --git a/x-pack/package.json b/x-pack/package.json index 84ce92bf8e9e6..eecc4dfc7b4ca 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -328,6 +328,7 @@ "resize-observer-polyfill": "^1.5.0", "rison-node": "0.3.1", "rxjs": "^6.5.3", + "rxjs-marbles": "^5.0.3", "semver": "5.7.0", "squel": "^5.13.0", "stats-lite": "^2.2.0", diff --git a/yarn.lock b/yarn.lock index 64d33426d7aa4..67a2bc00d81ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12038,6 +12038,11 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== +fast-equals@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/fast-equals/-/fast-equals-2.0.0.tgz#bef2c423af3939f2c54310df54c57e64cd2adefc" + integrity sha512-u6RBd8cSiLLxAiC04wVsLV6GBFDOXcTCgWkd3wEoFXgidPSoAJENqC9m7Jb2vewSvjBIfXV6icKeh3GTKfIaXA== + fast-glob@^2.0.2: version "2.0.4" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.0.4.tgz#a4b9f49e36175f5ef1a3456f580226a6e7abcc9e" @@ -24676,6 +24681,13 @@ rx@^4.1.0: resolved "https://registry.yarnpkg.com/rx/-/rx-4.1.0.tgz#a5f13ff79ef3b740fe30aa803fb09f98805d4782" integrity sha1-pfE/957zt0D+MKqAP7CfmIBdR4I= +rxjs-marbles@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/rxjs-marbles/-/rxjs-marbles-5.0.3.tgz#d3ca62a4e02d032b1b4ffd558e93336ad78fd100" + integrity sha512-JK6EvLe9uReJxBmUgdKrpMB2JswV+fDcKDg97x20LErLQ7Gi0FG3YEr2Uq9hvgHJjgZXGCvonpzcxARLzKsT4A== + dependencies: + fast-equals "^2.0.0" + rxjs@^5.0.0-beta.11, rxjs@^5.5.2: version "5.5.12" resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-5.5.12.tgz#6fa61b8a77c3d793dbaf270bee2f43f652d741cc" From 112227e08b70e60160c36218336faceb8c1468ab Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 28 Nov 2019 14:17:34 +0000 Subject: [PATCH 05/45] refactor(run-now): switch to simple mock of rxjs instead of importing whole mocking library --- .../task_poller.intervals.test.ts | 58 +++++++++---------- x-pack/package.json | 3 +- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts index adcf2daa903f7..a370e141a12ec 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts @@ -5,36 +5,36 @@ */ import _ from 'lodash'; -import { Subject } from 'rxjs'; +import { interval } from 'rxjs'; import { TaskPoller } from './task_poller'; -import { mockLogger, resolvable } from './test_utils'; -import { fakeSchedulers } from 'rxjs-marbles/jest'; +import { mockLogger } from './test_utils'; + +jest.mock('rxjs', () => ({ + Subject: jest.fn(() => ({ + pipe: jest.fn(() => ({ + pipe: jest.fn(), + subscribe: jest.fn(), + })), + })), + Subscription: jest.fn(), + Observable: jest.fn(() => ({ + pipe: jest.fn(), + })), + interval: jest.fn(() => ({ + pipe: jest.fn(), + })), +})); describe('TaskPoller Intervals', () => { - beforeEach(() => jest.useFakeTimers()); - test( - 'runs the work function on an interval', - fakeSchedulers(async advance => { - jest.useFakeTimers(); - const pollInterval = _.random(10, 20); - const done = resolvable(); - const work = jest.fn(async () => { - done.resolve(); - return true; - }); - const poller = new TaskPoller({ - pollInterval, - work, - logger: mockLogger(), - }); - poller.start(); - advance(pollInterval); - expect(work).toHaveBeenCalledTimes(1); - await done; - advance(pollInterval - 1); - expect(work).toHaveBeenCalledTimes(1); - advance(1); - expect(work).toHaveBeenCalledTimes(2); - }) - ); + test('intializes with the provided interval', () => { + const pollInterval = _.random(10, 20); + + new TaskPoller({ + pollInterval, + work: async () => {}, + logger: mockLogger(), + }); + + expect(interval).toHaveBeenCalledWith(pollInterval); + }); }); diff --git a/x-pack/package.json b/x-pack/package.json index 59ad697d2cfa5..739228ec27730 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -42,8 +42,8 @@ "@storybook/addon-storyshots": "^5.2.6", "@storybook/react": "^5.2.6", "@storybook/theming": "^5.2.6", - "@testing-library/react": "^9.3.2", "@testing-library/jest-dom": "4.2.0", + "@testing-library/react": "^9.3.2", "@types/angular": "^1.6.56", "@types/archiver": "^3.0.0", "@types/base64-js": "^1.2.5", @@ -326,7 +326,6 @@ "resize-observer-polyfill": "^1.5.0", "rison-node": "0.3.1", "rxjs": "^6.5.3", - "rxjs-marbles": "^5.0.3", "semver": "5.7.0", "squel": "^5.13.0", "stats-lite": "^2.2.0", From a01f110f6b240ba5bb3b7deaa8f660470dbdbaee Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 2 Dec 2019 13:09:11 +0000 Subject: [PATCH 06/45] expone runNow api on TaskManager --- x-pack/legacy/plugins/task_manager/plugin.ts | 2 + .../mark_available_tasks_as_claimed.ts | 25 +- .../task_manager/queries/query_clauses.ts | 13 +- .../plugins/task_manager/task_events.ts | 33 ++- .../plugins/task_manager/task_manager.test.ts | 8 +- .../plugins/task_manager/task_manager.ts | 77 +++++- .../plugins/task_manager/task_poller.test.ts | 1 - .../plugins/task_manager/task_poller.ts | 1 - .../legacy/plugins/task_manager/task_pool.ts | 20 +- .../plugins/task_manager/task_runner.test.ts | 244 ++++++++++++++++-- .../plugins/task_manager/task_runner.ts | 45 +++- .../legacy/plugins/task_manager/task_store.ts | 39 +-- .../plugins/task_manager/index.js | 9 +- .../plugins/task_manager/init_routes.js | 25 ++ .../task_manager/task_manager_integration.js | 104 +++++++- yarn.lock | 12 - 16 files changed, 539 insertions(+), 119 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/plugin.ts b/x-pack/legacy/plugins/task_manager/plugin.ts index 3e1514bd5234f..08382d1d825b6 100644 --- a/x-pack/legacy/plugins/task_manager/plugin.ts +++ b/x-pack/legacy/plugins/task_manager/plugin.ts @@ -11,6 +11,7 @@ export interface PluginSetupContract { fetch: TaskManager['fetch']; remove: TaskManager['remove']; schedule: TaskManager['schedule']; + runNow: TaskManager['runNow']; ensureScheduled: TaskManager['ensureScheduled']; addMiddleware: TaskManager['addMiddleware']; registerTaskDefinitions: TaskManager['registerTaskDefinitions']; @@ -60,6 +61,7 @@ export class Plugin { fetch: (...args) => taskManager.fetch(...args), remove: (...args) => taskManager.remove(...args), schedule: (...args) => taskManager.schedule(...args), + runNow: (...args) => taskManager.runNow(...args), ensureScheduled: (...args) => taskManager.ensureScheduled(...args), addMiddleware: (...args) => taskManager.addMiddleware(...args), registerTaskDefinitions: (...args) => taskManager.registerTaskDefinitions(...args), diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index b40bc4cbe51fb..bc5e80658ba9e 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -6,6 +6,7 @@ import { defaultsDeep } from 'lodash'; import { BoolClause, + IDsClause, SortClause, ScriptClause, ExistsBoolClause, @@ -40,9 +41,9 @@ export const IdleTaskWithExpiredRunAt: BoolClause => ({ - bool: { - must: [{ term: { 'task.status': 'idle' } }, { term: { 'task.id': claimTasksById } }], +export const idleTaskWithIDs = (claimTasksById: string[]): IDsClause => ({ + ids: { + values: claimTasksById, }, }); @@ -66,8 +67,15 @@ export const SortByRunAtAndRetryAt: SortClause = { type: 'number', order: 'asc', script: { - lang: 'expression', - source: `doc['task.retryAt'].value || doc['task.runAt'].value`, + lang: 'painless', + source: ` + if (doc['task.retryAt'].size()!=0) { + return doc['task.retryAt'].value.toInstant().toEpochMilli(); + } + if (doc['task.runAt'].size()!=0) { + return doc['task.runAt'].value.toInstant().toEpochMilli(); + } + `, }, }, }; @@ -83,7 +91,12 @@ export const sortByIdsThenByScheduling = (claimTasksById: string[]): SortClause { _script: { script: { - source: `params.ids.contains(doc['task.id']) ? ${SORT_VALUE_TO_BE_FIRST} : (${source})`, + source: ` + if(params.ids.contains(doc['_id'].value)){ + return ${SORT_VALUE_TO_BE_FIRST}; + } + ${source} + `, params: { ids: claimTasksById }, }, }, diff --git a/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts b/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts index ca86354f0f9e4..1f76ce99e600a 100644 --- a/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts +++ b/x-pack/legacy/plugins/task_manager/queries/query_clauses.ts @@ -14,11 +14,16 @@ export interface ExistsBoolClause { exists: { field: string }; } +export interface IDsClause { + ids: { + values: string[]; + }; +} export interface ShouldClause { - should: Array | T>; + should: Array | IDsClause | T>; } export interface MustClause { - must: Array | T>; + must: Array | IDsClause | T>; } export interface BoolClause { bool: MustClause | ShouldClause; @@ -49,7 +54,7 @@ export interface UpdateByQuery { } export function shouldBeOneOf( - ...should: Array | T> + ...should: Array | IDsClause | T> ): { bool: ShouldClause; } { @@ -61,7 +66,7 @@ export function shouldBeOneOf( } export function mustBeAllOf( - ...must: Array | T> + ...must: Array | IDsClause | T> ): { bool: MustClause; } { diff --git a/x-pack/legacy/plugins/task_manager/task_events.ts b/x-pack/legacy/plugins/task_manager/task_events.ts index 8d0fbf00ed065..54cd49a5449d6 100644 --- a/x-pack/legacy/plugins/task_manager/task_events.ts +++ b/x-pack/legacy/plugins/task_manager/task_events.ts @@ -8,48 +8,45 @@ import { ConcreteTaskInstance } from './task'; import { Result } from './lib/result_type'; -export type TaskRun = Result; -export type TaskRunNow = Result; +export type TaskMarkRunning = Result; +export type TaskRun = Result; export type TaskClaim = Result; export enum TaskEventType { - TASK_RUN_NOW, - TASK_RUN, - TASK_CLAIM, + TASK_CLAIM = 'TASK_CLAIM', + TASK_MARK_RUNNING = 'TASK_MARK_RUNNING', + TASK_RUN = 'TASK_RUN', } export type TaskEvent = { id: string; } & ( - | { - type: TaskEventType.TASK_RUN; - event: TaskRun; - } | { type: TaskEventType.TASK_CLAIM; event: TaskClaim; } | { - type: TaskEventType.TASK_RUN_NOW; - event: TaskRunNow; + type: TaskEventType.TASK_MARK_RUNNING; + event: TaskMarkRunning; + } + | { + type: TaskEventType.TASK_RUN; + event: TaskRun; } ); -export function asTaskRunEvent( - id: string, - event: Result -): TaskEvent { +export function asTaskMarkRunningEvent(id: string, event: Result): TaskEvent { return { id, - type: TaskEventType.TASK_RUN, + type: TaskEventType.TASK_MARK_RUNNING, event, }; } -export function asTaskRunNowEvent(id: string, event: Result): TaskEvent { +export function asTaskRunEvent(id: string, event: Result): TaskEvent { return { id, - type: TaskEventType.TASK_RUN_NOW, + type: TaskEventType.TASK_RUN, event, }; } diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts index 0b4a22910e611..90f3695c5e627 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts @@ -280,18 +280,18 @@ describe('TaskManager', () => { const availableWorkers = 1; - claimAvailableTasks(claim, availableWorkers, logger); + claimAvailableTasks([], claim, availableWorkers, logger); expect(claim).toHaveBeenCalledTimes(1); }); - test('shouldnt claim Available Tasks when there are no available workers', () => { + test('should not claim Available Tasks when there are no available workers', () => { const logger = mockLogger(); const claim = jest.fn(() => Promise.resolve({ docs: [], claimedTasks: 0 })); const availableWorkers = 0; - claimAvailableTasks(claim, availableWorkers, logger); + claimAvailableTasks([], claim, availableWorkers, logger); expect(claim).not.toHaveBeenCalled(); }); @@ -320,7 +320,7 @@ describe('TaskManager', () => { }); }); - claimAvailableTasks(claim, 10, logger); + claimAvailableTasks([], claim, 10, logger); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn.mock.calls[0][0]).toMatchInlineSnapshot( diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 9747aaa4ad210..744422e923822 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -3,12 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Subject } from 'rxjs'; -import { Option, none } from 'fp-ts/lib/Option'; +import { Subject, merge } from 'rxjs'; +import { filter } from 'rxjs/operators'; import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; + +import { Option, none, some, Some, isSome } from 'fp-ts/lib/Option'; +import { isOk, isErr } from './lib/result_type'; + import { Logger } from './types'; -import { TaskEvent } from './task_events'; +import { TaskEvent, TaskEventType } from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { addMiddlewareToChain, BeforeSaveMiddlewareParams, Middleware } from './lib/middleware'; import { sanitizeTaskDefinitions } from './lib/sanitize_task_definitions'; @@ -43,6 +47,15 @@ export interface TaskManagerOpts { serializer: SavedObjectsSerializer; } +type RunNowResult = + | { + id: string; + error: object | Error; + } + | { + id: string; + }; + /* * The TaskManager is the public interface into the task manager system. This glues together * all of the disparate modules in one integration point. The task manager operates in two different ways: @@ -61,11 +74,9 @@ export class TaskManager { private maxWorkers: number; private definitions: TaskDictionary; private store: TaskStore; - // private poller: TaskPoller; private logger: Logger; private pool: TaskPool; - - private taskEvent$: Subject; + private events$: Subject; private poller: TaskPoller>; @@ -85,7 +96,6 @@ export class TaskManager { this.maxWorkers = opts.config.get('xpack.task_manager.max_workers'); this.definitions = {}; this.logger = opts.logger; - this.taskEvent$ = new Subject(); const taskManagerId = opts.config.get('server.uuid'); if (!taskManagerId) { @@ -112,6 +122,10 @@ export class TaskManager { maxWorkers: this.maxWorkers, }); + this.events$ = new Subject(); + // if we end up with only one stream, remove merge + merge(this.store.events).subscribe(event => this.events$.next(event)); + this.poller = new TaskPoller>({ logger: this.logger, pollInterval: opts.config.get('xpack.task_manager.poll_interval'), @@ -119,11 +133,21 @@ export class TaskManager { }); } - private pollForWork = async (): Promise => { + private emitEvent = (event: TaskEvent) => { + this.events$.next(event); + }; + + private pollForWork = async ( + ...optionalTasksToClaim: Array> + ): Promise => { + const tasksToClaim = optionalTasksToClaim + .filter(isSome) + .map((task: Some) => task.value); return fillPool( // claim available tasks () => claimAvailableTasks( + tasksToClaim.splice(0, this.pool.availableWorkers), this.store.claimAvailableTasks, this.pool.availableWorkers, this.logger @@ -135,8 +159,8 @@ export class TaskManager { ); }; - private attemptWork() { - this.poller.queueWork(none); + private attemptToRun(task: Option = none) { + this.poller.queueWork(task); } private createTaskRunnerForTask = (instance: ConcreteTaskInstance) => { @@ -147,6 +171,7 @@ export class TaskManager { definitions: this.definitions, beforeRun: this.middleware.beforeRun, beforeMarkRunning: this.middleware.beforeMarkRunning, + onTaskEvent: this.emitEvent, }); }; @@ -226,10 +251,38 @@ export class TaskManager { taskInstance, }); const result = await this.store.schedule(modifiedTask); - this.attemptWork(); + this.attemptToRun(); return result; } + /** + * Run task. + * + * @param task - The task being scheduled. + * @returns {Promise} + */ + public async runNow(task: string): Promise { + await this.waitUntilStarted(); + return new Promise(resolve => { + const subscription = this.events$ + .pipe(filter(({ id }: TaskEvent) => id === task)) + .subscribe(({ id, event, type }: TaskEvent) => { + if ( + type === TaskEventType.TASK_RUN && + isOk(event) + ) { + subscription.unsubscribe(); + return resolve({ id }); + } else if (isErr(event)) { + subscription.unsubscribe(); + return resolve({ id, error: `${event.error}` }); + } + }); + + this.attemptToRun(some(task)); + }); + } + /** * Schedules a task with an Id * @@ -286,6 +339,7 @@ export class TaskManager { } export async function claimAvailableTasks( + claimTasksById: string[], claim: (opts: OwnershipClaimingOpts) => Promise, availableWorkers: number, logger: Logger @@ -297,6 +351,7 @@ export async function claimAvailableTasks( const { docs, claimedTasks } = await claim({ size: availableWorkers, claimOwnershipUntil: intervalFromNow('30s')!, + claimTasksById, }); if (claimedTasks === 0) { diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts index f14402d75c145..adb4d8040b7ea 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts @@ -5,7 +5,6 @@ */ import _ from 'lodash'; -import { Subject } from 'rxjs'; import { TaskPoller } from './task_poller'; import { mockLogger, resolvable, sleep } from './test_utils'; diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index c1610f66aa96a..5cbb57b7f24bd 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -62,7 +62,6 @@ export class TaskPoller { } this.pollingSubscription = this.poller$.subscribe(requests => { - // console.log({ requests }); this.attemptWork(...requests); }); } diff --git a/x-pack/legacy/plugins/task_manager/task_pool.ts b/x-pack/legacy/plugins/task_manager/task_pool.ts index 5828cb0df4a4d..02048aea7d2b9 100644 --- a/x-pack/legacy/plugins/task_manager/task_pool.ts +++ b/x-pack/legacy/plugins/task_manager/task_pool.ts @@ -85,18 +85,18 @@ export class TaskPool { performance.mark('attemptToRun_start'); await Promise.all( tasksToRun.map( - async task => - await task + async taskRunner => + await taskRunner .markTaskAsRunning() .then((hasTaskBeenMarkAsRunning: boolean) => hasTaskBeenMarkAsRunning - ? this.handleMarkAsRunning(task) - : this.handleFailureOfMarkAsRunning(task, { + ? this.handleMarkAsRunning(taskRunner) + : this.handleFailureOfMarkAsRunning(taskRunner, { name: 'TaskPoolVersionConflictError', message: VERSION_CONFLICT_MESSAGE, }) ) - .catch(ex => this.handleFailureOfMarkAsRunning(task, ex)) + .catch(err => this.handleFailureOfMarkAsRunning(taskRunner, err)) ) ); @@ -113,14 +113,14 @@ export class TaskPool { return TaskPoolRunResult.RunningAllClaimedTasks; } - private handleMarkAsRunning(task: TaskRunner) { - this.running.add(task); - task + private handleMarkAsRunning(taskRunner: TaskRunner) { + this.running.add(taskRunner); + taskRunner .run() .catch(err => { - this.logger.warn(`Task ${task.toString()} failed in attempt to run: ${err.message}`); + this.logger.warn(`Task ${taskRunner.toString()} failed in attempt to run: ${err.message}`); }) - .then(() => this.running.delete(task)); + .then(() => this.running.delete(taskRunner)); } private handleFailureOfMarkAsRunning(task: TaskRunner, err: Error) { diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts index 578b86ba0b3f6..3bdf32b81dce2 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts @@ -7,6 +7,8 @@ import _ from 'lodash'; import sinon from 'sinon'; import { minutesFromNow } from './lib/intervals'; +import { asOk, asErr } from './lib/result_type'; +import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; import { ConcreteTaskInstance } from './task'; import { TaskManagerRunner } from './task_runner'; import { mockLogger } from './test_utils'; @@ -657,45 +659,245 @@ describe('TaskManagerRunner', () => { ); }); + describe('TaskEvents', () => { + test('emits TaskEvent when a task is marked as running', async () => { + const id = _.random(1, 20).toString(); + const onTaskEvent = jest.fn(); + const { runner, instance, store } = testOpts({ + onTaskEvent, + instance: { + id, + }, + definitions: { + bar: { + timeout: `1m`, + getRetry: () => {}, + createTaskRunner: () => ({ + run: async () => undefined, + }), + }, + }, + }); + + store.update.returns(instance); + + await runner.markTaskAsRunning(); + + expect(onTaskEvent).toHaveBeenCalledWith(asTaskMarkRunningEvent(id, asOk(undefined))); + }); + + test('emits TaskEvent when a task fails to be marked as running', async () => { + expect.assertions(2); + + const id = _.random(1, 20).toString(); + const onTaskEvent = jest.fn(); + const { runner, store } = testOpts({ + onTaskEvent, + instance: { + id, + }, + definitions: { + bar: { + timeout: `1m`, + getRetry: () => {}, + createTaskRunner: () => ({ + run: async () => undefined, + }), + }, + }, + }); + + store.update.throws(new Error('cant mark as running')); + + try { + await runner.markTaskAsRunning(); + } catch (err) { + expect(onTaskEvent).toHaveBeenCalledWith(asTaskMarkRunningEvent(id, asErr(err))); + } + expect(onTaskEvent).toHaveBeenCalledTimes(1); + }); + + test('emits TaskEvent when a task is run successfully', async () => { + const id = _.random(1, 20).toString(); + const onTaskEvent = jest.fn(); + const { runner } = testOpts({ + onTaskEvent, + instance: { + id, + }, + definitions: { + bar: { + createTaskRunner: () => ({ + async run() { + return {}; + }, + }), + }, + }, + }); + + await runner.run(); + + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(undefined))); + }); + + test('emits TaskEvent when a recurring task is run successfully', async () => { + const id = _.random(1, 20).toString(); + const runAt = minutesFromNow(_.random(5)); + const onTaskEvent = jest.fn(); + const { runner } = testOpts({ + onTaskEvent, + instance: { + id, + interval: '1m', + }, + definitions: { + bar: { + createTaskRunner: () => ({ + async run() { + return { runAt }; + }, + }), + }, + }, + }); + + await runner.run(); + + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(undefined))); + }); + + test('emits TaskEvent when a task run throws an error', async () => { + const id = _.random(1, 20).toString(); + const error = new Error('Dangit!'); + const onTaskEvent = jest.fn(); + const { runner } = testOpts({ + onTaskEvent, + instance: { + id, + }, + definitions: { + bar: { + createTaskRunner: () => ({ + async run() { + throw error; + }, + }), + }, + }, + }); + await runner.run(); + + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledTimes(1); + }); + + test('emits TaskEvent when a task run returns an error', async () => { + const id = _.random(1, 20).toString(); + const error = new Error('Dangit!'); + const onTaskEvent = jest.fn(); + const { runner } = testOpts({ + onTaskEvent, + instance: { + id, + interval: '1m', + startedAt: new Date(), + }, + definitions: { + bar: { + createTaskRunner: () => ({ + async run() { + return { error }; + }, + }), + }, + }, + }); + + await runner.run(); + + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledTimes(1); + }); + + test('emits TaskEvent when a task returns an error and is marked as failed', async () => { + const id = _.random(1, 20).toString(); + const error = new Error('Dangit!'); + const onTaskEvent = jest.fn(); + const { runner, store } = testOpts({ + onTaskEvent, + instance: { + id, + startedAt: new Date(), + }, + definitions: { + bar: { + getRetry: () => false, + createTaskRunner: () => ({ + async run() { + return { error }; + }, + }), + }, + }, + }); + + await runner.run(); + + const instance = store.update.args[0][0]; + expect(instance.status).toBe('failed'); + + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledTimes(1); + }); + }); + interface TestOpts { instance?: Partial; definitions?: any; + onTaskEvent?: (event: TaskEvent) => void; } function testOpts(opts: TestOpts) { const callCluster = sinon.stub(); const createTaskRunner = sinon.stub(); const logger = mockLogger(); + + const instance = Object.assign( + { + id: 'foo', + taskType: 'bar', + sequenceNumber: 32, + primaryTerm: 32, + runAt: new Date(), + scheduledAt: new Date(), + startedAt: null, + retryAt: null, + attempts: 0, + params: {}, + scope: ['reporting'], + state: {}, + status: 'idle', + user: 'example', + ownerId: null, + }, + opts.instance || {} + ); + const store = { update: sinon.stub(), remove: sinon.stub(), maxAttempts: 5, }; + + store.update.returns(instance); + const runner = new TaskManagerRunner({ beforeRun: context => Promise.resolve(context), beforeMarkRunning: context => Promise.resolve(context), logger, store, - instance: Object.assign( - { - id: 'foo', - taskType: 'bar', - sequenceNumber: 32, - primaryTerm: 32, - runAt: new Date(), - scheduledAt: new Date(), - startedAt: null, - retryAt: null, - attempts: 0, - params: {}, - scope: ['reporting'], - state: {}, - status: 'idle', - user: 'example', - ownerId: null, - }, - opts.instance || {} - ), + instance, definitions: Object.assign(opts.definitions || {}, { testbar: { type: 'bar', @@ -703,6 +905,7 @@ describe('TaskManagerRunner', () => { createTaskRunner, }, }), + onTaskEvent: opts.onTaskEvent, }); return { @@ -711,6 +914,7 @@ describe('TaskManagerRunner', () => { runner, logger, store, + instance, }; } diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index 9d1431ed004e3..48b2819d4ef8a 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -12,6 +12,10 @@ import { performance } from 'perf_hooks'; import Joi from 'joi'; +import { identity } from 'lodash'; + +import { asOk, asErr } from './lib/result_type'; +import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { Logger } from './types'; import { BeforeRunFunction, BeforeMarkRunningFunction } from './lib/middleware'; @@ -33,6 +37,7 @@ export interface TaskRunner { cancel: CancelFunction; markTaskAsRunning: () => Promise; run: () => Promise; + id: string; toString: () => string; } @@ -49,6 +54,7 @@ interface Opts { store: Updatable; beforeRun: BeforeRunFunction; beforeMarkRunning: BeforeMarkRunningFunction; + onTaskEvent?: (event: TaskEvent) => void; } /** @@ -67,6 +73,7 @@ export class TaskManagerRunner implements TaskRunner { private bufferedTaskStore: Updatable; private beforeRun: BeforeRunFunction; private beforeMarkRunning: BeforeMarkRunningFunction; + private onTaskEvent: (event: TaskEvent) => void; /** * Creates an instance of TaskManagerRunner. @@ -78,13 +85,22 @@ export class TaskManagerRunner implements TaskRunner { * @prop {BeforeRunFunction} beforeRun - A function that adjusts the run context prior to running the task * @memberof TaskManagerRunner */ - constructor(opts: Opts) { - this.instance = sanitizeInstance(opts.instance); - this.definitions = opts.definitions; - this.logger = opts.logger; - this.bufferedTaskStore = opts.store; - this.beforeRun = opts.beforeRun; - this.beforeMarkRunning = opts.beforeMarkRunning; + constructor({ + instance, + definitions, + logger, + store, + beforeRun, + beforeMarkRunning, + onTaskEvent = identity, + }: Opts) { + this.instance = sanitizeInstance(instance); + this.definitions = definitions; + this.logger = logger; + this.bufferedTaskStore = store; + this.beforeRun = beforeRun; + this.beforeMarkRunning = beforeMarkRunning; + this.onTaskEvent = onTaskEvent; } /** @@ -143,7 +159,6 @@ export class TaskManagerRunner implements TaskRunner { return this.processResult(validatedResult); } catch (err) { this.logger.error(`Task ${this} failed: ${err}`); - // in error scenario, we can not get the RunResult // re-use modifiedContext's state, which is correct as of beforeRun return this.processResult({ error: err, state: modifiedContext.taskInstance.state }); @@ -209,9 +224,11 @@ export class TaskManagerRunner implements TaskRunner { } performanceStopMarkingTaskAsRunning(); + this.onTaskEvent(asTaskMarkRunningEvent(this.id, asOk(undefined))); return true; } catch (error) { performanceStopMarkingTaskAsRunning(); + this.onTaskEvent(asTaskMarkRunningEvent(this.id, asErr(error))); if (error.statusCode !== VERSION_CONFLICT_STATUS) { throw error; } @@ -264,6 +281,7 @@ export class TaskManagerRunner implements TaskRunner { attempts: this.instance.attempts, error: result.error, }); + if (!newRunAt) { status = 'failed'; runAt = this.instance.runAt; @@ -304,10 +322,16 @@ export class TaskManagerRunner implements TaskRunner { } private async processResult(result: RunResult): Promise { - if (result.runAt || this.instance.interval || result.error) { + if (result.error) { await this.processResultForRecurringTask(result); + this.onTaskEvent(asTaskRunEvent(this.id, asErr(result.error))); } else { - await this.processResultWhenDone(result); + if (result.runAt || this.instance.interval) { + await this.processResultForRecurringTask(result); + } else { + await this.processResultWhenDone(result); + } + this.onTaskEvent(asTaskRunEvent(this.id, asOk(undefined))); } return result; } @@ -347,7 +371,6 @@ export class TaskManagerRunner implements TaskRunner { if (addDuration && result) { result = intervalFromDate(result, addDuration)!; } - return result; } } diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index a4456e4c89c87..25e566ba77d22 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -198,13 +198,22 @@ export class TaskStore { * @param {OwnershipClaimingOpts} options * @returns {Promise} */ - public claimAvailableTasks = async ( - opts: OwnershipClaimingOpts - ): Promise => { - const { claimTasksById } = opts; + public claimAvailableTasks = async ({ + claimOwnershipUntil, + claimTasksById = [], + size, + }: OwnershipClaimingOpts): Promise => { + const claimTasksByIdWithRawIds = claimTasksById.map(id => + this.serializer.generateRawId(undefined, 'task', id) + ); - const claimedTasks = await this.markAvailableTasksAsClaimed(opts); - const docs = claimedTasks > 0 ? await this.sweepForClaimedTasks(opts) : []; + const claimedTasks = await this.markAvailableTasksAsClaimed( + claimOwnershipUntil, + claimTasksByIdWithRawIds, + size + ); + const docs = + claimedTasks > 0 ? await this.sweepForClaimedTasks(claimTasksByIdWithRawIds, size) : []; // emit success/fail events for claimed tasks by id if (claimTasksById && claimTasksById.length) { @@ -224,11 +233,11 @@ export class TaskStore { }; }; - private async markAvailableTasksAsClaimed({ - size, - claimOwnershipUntil, - claimTasksById, - }: OwnershipClaimingOpts): Promise { + private async markAvailableTasksAsClaimed( + claimOwnershipUntil: OwnershipClaimingOpts['claimOwnershipUntil'], + claimTasksById: OwnershipClaimingOpts['claimTasksById'], + size: OwnershipClaimingOpts['size'] + ): Promise { const queryForScheduledTasks = mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. @@ -273,10 +282,10 @@ export class TaskStore { /** * Fetches tasks from the index, which are owned by the current Kibana instance */ - private async sweepForClaimedTasks({ - size, - claimTasksById, - }: OwnershipClaimingOpts): Promise { + private async sweepForClaimedTasks( + claimTasksById: OwnershipClaimingOpts['claimTasksById'], + size: OwnershipClaimingOpts['size'] + ): Promise { const { docs } = await this.search({ query: { bool: { diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js index 73253224bb45d..35f5f648768b2 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js @@ -46,10 +46,15 @@ export default function TaskTestingAPI(kibana) { const { params, state } = taskInstance; const prevState = state || { count: 0 }; + const count = (prevState.count || 0) + 1; + if (params.failWith) { - throw new Error(params.failWith); + if (!params.failOn || (params.failOn && count === params.failOn)) { + throw new Error(params.failWith); + } } + const callCluster = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser; await callCluster('index', { index: '.kibana_task_manager_test_result', @@ -68,7 +73,7 @@ export default function TaskTestingAPI(kibana) { } return { - state: { count: (prevState.count || 0) + 1 }, + state: { count }, runAt: millisecondsFromNow(params.nextRunMilliseconds), }; }, diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index 7b9e265a15d6f..76eefa66bb30b 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -59,6 +59,31 @@ export function initRoutes(server, taskTestingEvents) { }, }); + server.route({ + path: '/api/sample_tasks/run_now', + method: 'POST', + config: { + validate: { + payload: Joi.object({ + task: Joi.object({ + id: Joi.string().optional() + }) + }), + }, + }, + async handler(request) { + try { + const { task } = request.payload; + + const taskResult = await (taskManager.runNow(task.id)); + + return taskResult; + } catch (err) { + return err; + } + }, + }); + server.route({ path: '/api/sample_tasks/ensure_scheduled', method: 'POST', diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index 986648f795da6..53c4f1bd74085 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -65,6 +65,14 @@ export default function ({ getService }) { .then((response) => response.body); } + function runTaskNow(task) { + return supertest.post('/api/sample_tasks/run_now') + .set('kbn-xsrf', 'xxx') + .send({ task }) + .expect(200) + .then((response) => response.body); + } + function scheduleTaskIfNotExists(task) { return supertest.post('/api/sample_tasks/ensure_scheduled') .set('kbn-xsrf', 'xxx') @@ -180,7 +188,7 @@ export default function ({ getService }) { expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(count + 1); - expectReschedule(originalTask, task, nextRunMilliseconds); + expectReschedule(Date.parse(originalTask.runAt), task, nextRunMilliseconds); }); }); @@ -201,12 +209,100 @@ export default function ({ getService }) { expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(1); - expectReschedule(originalTask, task, intervalMilliseconds); + expectReschedule(Date.parse(originalTask.runAt), task, intervalMilliseconds); + }); + }); + + it('should return a task run result when asked to run a task now', async () => { + + const originalTask = await scheduleTask({ + taskType: 'sampleTask', + interval: `30m`, + params: { }, + }); + + await retry.try(async () => { + const docs = await historyDocs(); + expect(docs.filter(taskDoc => taskDoc._source.taskId === originalTask.id).length).to.eql(1); + + const [task] = (await currentTasks()).docs.filter(taskDoc => taskDoc.id === originalTask.id); + + expect(task.state.count).to.eql(1); + + // ensure this task shouldnt run for another half hour + expectReschedule(Date.parse(originalTask.runAt), task, 30 * 60000); + + }); + + const now = Date.now(); + const runNowResult = await runTaskNow({ + id: originalTask.id + }); + + expect(runNowResult).to.eql({ id: originalTask.id }); + + + await retry.try(async () => { + expect((await historyDocs()).filter(taskDoc => taskDoc._source.taskId === originalTask.id).length).to.eql(2); + + const [task] = (await currentTasks()).docs.filter(taskDoc => taskDoc.id === originalTask.id); + expect(task.state.count).to.eql(2); + + // ensure this task shouldnt run for another half hour + expectReschedule(now, task, 30 * 60000); + + }); + }); + + it('should return a task run error result when running a task now fails', async () => { + + + const originalTask = await scheduleTask({ + taskType: 'sampleTask', + interval: `30m`, + params: { failWith: 'error on run now', failOn: 3 }, + }); + + await retry.try(async () => { + const docs = await historyDocs(); + expect(docs.filter(taskDoc => taskDoc._source.taskId === originalTask.id).length).to.eql(1); + + const [task] = (await currentTasks()).docs.filter(taskDoc => taskDoc.id === originalTask.id); + + expect(task.state.count).to.eql(1); + + // ensure this task shouldnt run for another half hour + expectReschedule(Date.parse(originalTask.runAt), task, 30 * 60000); + + }); + + // second run should still be successful + const successfulRunNowResult = await runTaskNow({ + id: originalTask.id + }); + expect(successfulRunNowResult).to.eql({ id: originalTask.id }); + + // third run should fail + const failedRunNowResult = await runTaskNow({ + id: originalTask.id + }); + + expect( + failedRunNowResult + ).to.eql( + { id: originalTask.id, error: `Error: error on run now` } + ); + + await retry.try(async () => { + expect((await historyDocs()).filter(taskDoc => taskDoc._source.taskId === originalTask.id).length).to.eql(2); + + const [task] = (await currentTasks()).docs.filter(taskDoc => taskDoc.id === originalTask.id); + expect(task.attempts).to.eql(1); + }); }); - async function expectReschedule(originalTask, currentTask, expectedDiff) { - const originalRunAt = Date.parse(originalTask.runAt); + async function expectReschedule(originalRunAt, currentTask, expectedDiff) { const buffer = 10000; expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.greaterThan(expectedDiff - buffer); expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.lessThan(expectedDiff + buffer); diff --git a/yarn.lock b/yarn.lock index 2c5f1fae05a7f..7da65ebf8bc81 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12105,11 +12105,6 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== -fast-equals@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-equals/-/fast-equals-2.0.0.tgz#bef2c423af3939f2c54310df54c57e64cd2adefc" - integrity sha512-u6RBd8cSiLLxAiC04wVsLV6GBFDOXcTCgWkd3wEoFXgidPSoAJENqC9m7Jb2vewSvjBIfXV6icKeh3GTKfIaXA== - fast-glob@^2.0.2: version "2.0.4" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.0.4.tgz#a4b9f49e36175f5ef1a3456f580226a6e7abcc9e" @@ -24726,13 +24721,6 @@ rx@^4.1.0: resolved "https://registry.yarnpkg.com/rx/-/rx-4.1.0.tgz#a5f13ff79ef3b740fe30aa803fb09f98805d4782" integrity sha1-pfE/957zt0D+MKqAP7CfmIBdR4I= -rxjs-marbles@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/rxjs-marbles/-/rxjs-marbles-5.0.3.tgz#d3ca62a4e02d032b1b4ffd558e93336ad78fd100" - integrity sha512-JK6EvLe9uReJxBmUgdKrpMB2JswV+fDcKDg97x20LErLQ7Gi0FG3YEr2Uq9hvgHJjgZXGCvonpzcxARLzKsT4A== - dependencies: - fast-equals "^2.0.0" - rxjs@^5.0.0-beta.11, rxjs@^5.5.2: version "5.5.12" resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-5.5.12.tgz#6fa61b8a77c3d793dbaf270bee2f43f652d741cc" From 2a6ea1fbf7293d9f9fd4917bec4c0a2723c8d9c6 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 2 Dec 2019 16:56:38 +0000 Subject: [PATCH 07/45] refactored EvenTypes and how we process the run result of a task --- .../plugins/task_manager/lib/result_type.ts | 31 ++++ .../plugins/task_manager/plugin.test.ts | 1 + .../mark_available_tasks_as_claimed.test.ts | 11 +- .../mark_available_tasks_as_claimed.ts | 37 ++-- x-pack/legacy/plugins/task_manager/task.ts | 9 + .../plugins/task_manager/task_events.ts | 49 +++--- .../plugins/task_manager/task_manager.mock.ts | 1 + .../plugins/task_manager/task_manager.ts | 45 ++--- .../plugins/task_manager/task_pool.test.ts | 20 +-- .../plugins/task_manager/task_runner.test.ts | 12 +- .../plugins/task_manager/task_runner.ts | 139 ++++++++------- .../plugins/task_manager/task_store.test.ts | 161 ++++++++++-------- .../legacy/plugins/task_manager/task_store.ts | 19 ++- 13 files changed, 325 insertions(+), 210 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/lib/result_type.ts b/x-pack/legacy/plugins/task_manager/lib/result_type.ts index 256463251315d..bf57df981a3cb 100644 --- a/x-pack/legacy/plugins/task_manager/lib/result_type.ts +++ b/x-pack/legacy/plugins/task_manager/lib/result_type.ts @@ -48,3 +48,34 @@ export async function promiseResult(future: Promise): Promise( + result: Result, + onOk: (value: T) => void, + onErr: (error: E) => void +): void { + resolve(result, onOk, onErr); +} + +export async function eitherAsync( + result: Result, + onOk: (value: T) => Promise, + onErr: (error: E) => Promise +): Promise | void> { + return await resolve>(result, onOk, onErr); +} + +export function resolve( + result: Result, + onOk: (value: T) => Resolution, + onErr: (error: E) => Resolution +): Resolution { + return isOk(result) ? onOk(result.value) : onErr(result.error); +} + +export function correctError( + result: Result, + correctErr: (error: E) => Result +): Result { + return isOk(result) ? result : correctErr(result.error); +} diff --git a/x-pack/legacy/plugins/task_manager/plugin.test.ts b/x-pack/legacy/plugins/task_manager/plugin.test.ts index 4f2effb5da3a8..f7c5b35da50c2 100644 --- a/x-pack/legacy/plugins/task_manager/plugin.test.ts +++ b/x-pack/legacy/plugins/task_manager/plugin.test.ts @@ -46,6 +46,7 @@ describe('Task Manager Plugin', () => { "fetch": [Function], "registerTaskDefinitions": [Function], "remove": [Function], + "runNow": [Function], "schedule": [Function], } `); diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts index 060f04c1c18d6..b8ea7d2db832e 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts @@ -144,8 +144,15 @@ describe('mark_available_tasks_as_claimed', () => { type: 'number', order: 'asc', script: { - lang: 'expression', - source: `doc['task.retryAt'].value || doc['task.runAt'].value`, + lang: 'painless', + source: ` +if (doc['task.retryAt'].size()!=0) { + return doc['task.retryAt'].value.toInstant().toEpochMilli(); +} +if (doc['task.runAt'].size()!=0) { + return doc['task.runAt'].value.toInstant().toEpochMilli(); +} + `, }, }, }, diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index bc5e80658ba9e..dd81f153f2810 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -41,9 +41,18 @@ export const IdleTaskWithExpiredRunAt: BoolClause ({ - ids: { - values: claimTasksById, +export const idleTaskWithIDs = ( + claimTasksById: string[] +): BoolClause => ({ + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { + ids: { + values: claimTasksById, + }, + }, + ], }, }); @@ -69,12 +78,12 @@ export const SortByRunAtAndRetryAt: SortClause = { script: { lang: 'painless', source: ` - if (doc['task.retryAt'].size()!=0) { - return doc['task.retryAt'].value.toInstant().toEpochMilli(); - } - if (doc['task.runAt'].size()!=0) { - return doc['task.runAt'].value.toInstant().toEpochMilli(); - } +if (doc['task.retryAt'].size()!=0) { + return doc['task.retryAt'].value.toInstant().toEpochMilli(); +} +if (doc['task.runAt'].size()!=0) { + return doc['task.runAt'].value.toInstant().toEpochMilli(); +} `, }, }, @@ -92,11 +101,11 @@ export const sortByIdsThenByScheduling = (claimTasksById: string[]): SortClause _script: { script: { source: ` - if(params.ids.contains(doc['_id'].value)){ - return ${SORT_VALUE_TO_BE_FIRST}; - } - ${source} - `, +if(params.ids.contains(doc['_id'].value)){ + return ${SORT_VALUE_TO_BE_FIRST}; +} +${source} +`, params: { ids: claimTasksById }, }, }, diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index 3eeb23685f377..9f280b13447ab 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -64,6 +64,15 @@ export interface RunResult { state: Record; } +export interface SuccessfulRunResult { + runAt?: Date; + state?: Record; +} + +export interface FailedRunResult extends SuccessfulRunResult { + error: Error; +} + export const validateRunResult = Joi.object({ runAt: Joi.date().optional(), error: Joi.object().optional(), diff --git a/x-pack/legacy/plugins/task_manager/task_events.ts b/x-pack/legacy/plugins/task_manager/task_events.ts index 54cd49a5449d6..d02adaea411fc 100644 --- a/x-pack/legacy/plugins/task_manager/task_events.ts +++ b/x-pack/legacy/plugins/task_manager/task_events.ts @@ -8,34 +8,25 @@ import { ConcreteTaskInstance } from './task'; import { Result } from './lib/result_type'; -export type TaskMarkRunning = Result; -export type TaskRun = Result; -export type TaskClaim = Result; - export enum TaskEventType { TASK_CLAIM = 'TASK_CLAIM', TASK_MARK_RUNNING = 'TASK_MARK_RUNNING', TASK_RUN = 'TASK_RUN', } -export type TaskEvent = { +export interface TaskEvent { id: string; -} & ( - | { - type: TaskEventType.TASK_CLAIM; - event: TaskClaim; - } - | { - type: TaskEventType.TASK_MARK_RUNNING; - event: TaskMarkRunning; - } - | { - type: TaskEventType.TASK_RUN; - event: TaskRun; - } -); - -export function asTaskMarkRunningEvent(id: string, event: Result): TaskEvent { + type: TaskEventType; + event: Result; +} +export type TaskMarkRunning = TaskEvent; +export type TaskRun = TaskEvent; +export type TaskClaim = TaskEvent; + +export function asTaskMarkRunningEvent( + id: string, + event: Result +): TaskMarkRunning { return { id, type: TaskEventType.TASK_MARK_RUNNING, @@ -43,7 +34,7 @@ export function asTaskMarkRunningEvent(id: string, event: Result): }; } -export function asTaskRunEvent(id: string, event: Result): TaskEvent { +export function asTaskRunEvent(id: string, event: Result): TaskRun { return { id, type: TaskEventType.TASK_RUN, @@ -54,10 +45,22 @@ export function asTaskRunEvent(id: string, event: Result): export function asTaskClaimEvent( id: string, event: Result -): TaskEvent { +): TaskClaim { return { id, type: TaskEventType.TASK_CLAIM, event, }; } + +export function isTaskMarkRunningEvent( + taskEvent: TaskEvent +): taskEvent is TaskMarkRunning { + return taskEvent.type === TaskEventType.TASK_MARK_RUNNING; +} +export function isTaskRunEvent(taskEvent: TaskEvent): taskEvent is TaskRun { + return taskEvent.type === TaskEventType.TASK_RUN; +} +export function isTaskClaimEvent(taskEvent: TaskEvent): taskEvent is TaskClaim { + return taskEvent.type === TaskEventType.TASK_CLAIM; +} diff --git a/x-pack/legacy/plugins/task_manager/task_manager.mock.ts b/x-pack/legacy/plugins/task_manager/task_manager.mock.ts index 515099a8bd479..4837e75fd3160 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.mock.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.mock.ts @@ -13,6 +13,7 @@ const createTaskManagerMock = () => { ensureScheduled: jest.fn(), schedule: jest.fn(), fetch: jest.fn(), + runNow: jest.fn(), remove: jest.fn(), start: jest.fn(), stop: jest.fn(), diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 744422e923822..8861f6727818c 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -9,10 +9,10 @@ import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; import { Option, none, some, Some, isSome } from 'fp-ts/lib/Option'; -import { isOk, isErr } from './lib/result_type'; +import { either } from './lib/result_type'; import { Logger } from './types'; -import { TaskEvent, TaskEventType } from './task_events'; +import { TaskMarkRunning, TaskRun, TaskClaim, isTaskRunEvent } from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { addMiddlewareToChain, BeforeSaveMiddlewareParams, Middleware } from './lib/middleware'; import { sanitizeTaskDefinitions } from './lib/sanitize_task_definitions'; @@ -50,12 +50,14 @@ export interface TaskManagerOpts { type RunNowResult = | { id: string; - error: object | Error; + error: string; } | { id: string; }; +type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; + /* * The TaskManager is the public interface into the task manager system. This glues together * all of the disparate modules in one integration point. The task manager operates in two different ways: @@ -76,7 +78,7 @@ export class TaskManager { private store: TaskStore; private logger: Logger; private pool: TaskPool; - private events$: Subject; + private events$: Subject; private poller: TaskPoller>; @@ -122,9 +124,9 @@ export class TaskManager { maxWorkers: this.maxWorkers, }); - this.events$ = new Subject(); + this.events$ = new Subject(); // if we end up with only one stream, remove merge - merge(this.store.events).subscribe(event => this.events$.next(event)); + merge(this.store.events).subscribe(event => this.events$.next(event)); this.poller = new TaskPoller>({ logger: this.logger, @@ -133,7 +135,7 @@ export class TaskManager { }); } - private emitEvent = (event: TaskEvent) => { + private emitEvent = (event: TaskLifecycleEvent) => { this.events$.next(event); }; @@ -265,18 +267,23 @@ export class TaskManager { await this.waitUntilStarted(); return new Promise(resolve => { const subscription = this.events$ - .pipe(filter(({ id }: TaskEvent) => id === task)) - .subscribe(({ id, event, type }: TaskEvent) => { - if ( - type === TaskEventType.TASK_RUN && - isOk(event) - ) { - subscription.unsubscribe(); - return resolve({ id }); - } else if (isErr(event)) { - subscription.unsubscribe(); - return resolve({ id, error: `${event.error}` }); - } + .pipe(filter(({ id }: TaskLifecycleEvent) => id === task)) + .subscribe((taskEvent: TaskLifecycleEvent) => { + const { id, event } = taskEvent; + + either( + event, + (taskInstance: ConcreteTaskInstance) => { + if (isTaskRunEvent(taskEvent)) { + subscription.unsubscribe(); + resolve({ id }); + } + }, + (error: Error) => { + subscription.unsubscribe(); + resolve({ id, error: `${error}` }); + } + ); }); this.attemptToRun(some(task)); diff --git a/x-pack/legacy/plugins/task_manager/task_pool.test.ts b/x-pack/legacy/plugins/task_manager/task_pool.test.ts index 4967f4383294f..a827c1436c9bd 100644 --- a/x-pack/legacy/plugins/task_manager/task_pool.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_pool.test.ts @@ -7,6 +7,7 @@ import sinon from 'sinon'; import { TaskPool, TaskPoolRunResult } from './task_pool'; import { mockLogger, resolvable, sleep } from './test_utils'; +import { asOk } from './lib/result_type'; describe('TaskPool', () => { test('occupiedWorkers are a sum of running tasks', async () => { @@ -128,13 +129,13 @@ describe('TaskPool', () => { const firstRun = sinon.spy(async () => { await sleep(0); firstWork.resolve(); - return { state: {} }; + return asOk({ state: {} }); }); const secondWork = resolvable(); const secondRun = sinon.spy(async () => { await sleep(0); secondWork.resolve(); - return { state: {} }; + return asOk({ state: {} }); }); const result = await pool.run([ @@ -179,9 +180,7 @@ describe('TaskPool', () => { this.isExpired = true; expired.resolve(); await sleep(10); - return { - state: {}, - }; + return asOk({ state: {} }); }, cancel: shouldRun, }, @@ -189,9 +188,7 @@ describe('TaskPool', () => { ...mockTask(), async run() { await sleep(10); - return { - state: {}, - }; + return asOk({ state: {} }); }, cancel: shouldNotRun, }, @@ -225,9 +222,7 @@ describe('TaskPool', () => { async run() { this.isExpired = true; await sleep(10); - return { - state: {}, - }; + return asOk({ state: {} }); }, async cancel() { cancelled.resolve(); @@ -253,13 +248,14 @@ describe('TaskPool', () => { function mockRun() { return jest.fn(async () => { await sleep(0); - return { state: {} }; + return asOk({ state: {} }); }); } function mockTask() { return { isExpired: false, + id: 'foo', cancel: async () => undefined, markTaskAsRunning: jest.fn(async () => true), run: mockRun(), diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts index 3bdf32b81dce2..78b3306b7ffc9 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts @@ -683,7 +683,7 @@ describe('TaskManagerRunner', () => { await runner.markTaskAsRunning(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskMarkRunningEvent(id, asOk(undefined))); + expect(onTaskEvent).toHaveBeenCalledWith(asTaskMarkRunningEvent(id, asOk(instance))); }); test('emits TaskEvent when a task fails to be marked as running', async () => { @@ -720,7 +720,7 @@ describe('TaskManagerRunner', () => { test('emits TaskEvent when a task is run successfully', async () => { const id = _.random(1, 20).toString(); const onTaskEvent = jest.fn(); - const { runner } = testOpts({ + const { runner, instance } = testOpts({ onTaskEvent, instance: { id, @@ -738,14 +738,14 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(undefined))); + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(instance))); }); test('emits TaskEvent when a recurring task is run successfully', async () => { const id = _.random(1, 20).toString(); const runAt = minutesFromNow(_.random(5)); const onTaskEvent = jest.fn(); - const { runner } = testOpts({ + const { runner, instance } = testOpts({ onTaskEvent, instance: { id, @@ -764,7 +764,7 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(undefined))); + expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(instance))); }); test('emits TaskEvent when a task run throws an error', async () => { @@ -855,7 +855,7 @@ describe('TaskManagerRunner', () => { interface TestOpts { instance?: Partial; definitions?: any; - onTaskEvent?: (event: TaskEvent) => void; + onTaskEvent?: (event: TaskEvent) => void; } function testOpts(opts: TestOpts) { diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index 48b2819d4ef8a..86a71fb79d107 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -14,8 +14,8 @@ import { performance } from 'perf_hooks'; import Joi from 'joi'; import { identity } from 'lodash'; -import { asOk, asErr } from './lib/result_type'; -import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; +import { asOk, asErr, isErr, correctError, eitherAsync, resolve, Result } from './lib/result_type'; +import { TaskRun, TaskMarkRunning, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { Logger } from './types'; import { BeforeRunFunction, BeforeMarkRunningFunction } from './lib/middleware'; @@ -24,6 +24,8 @@ import { CancellableTask, ConcreteTaskInstance, RunResult, + SuccessfulRunResult, + FailedRunResult, TaskDefinition, TaskDictionary, validateRunResult, @@ -31,12 +33,13 @@ import { } from './task'; const defaultBackoffPerFailure = 5 * 60 * 1000; +const EMPTY_RUN_RESULT: SuccessfulRunResult = {}; export interface TaskRunner { isExpired: boolean; cancel: CancelFunction; markTaskAsRunning: () => Promise; - run: () => Promise; + run: () => Promise>; id: string; toString: () => string; } @@ -54,7 +57,7 @@ interface Opts { store: Updatable; beforeRun: BeforeRunFunction; beforeMarkRunning: BeforeMarkRunningFunction; - onTaskEvent?: (event: TaskEvent) => void; + onTaskEvent?: (event: TaskRun | TaskMarkRunning) => void; } /** @@ -73,7 +76,7 @@ export class TaskManagerRunner implements TaskRunner { private bufferedTaskStore: Updatable; private beforeRun: BeforeRunFunction; private beforeMarkRunning: BeforeMarkRunningFunction; - private onTaskEvent: (event: TaskEvent) => void; + private onTaskEvent: (event: TaskRun | TaskMarkRunning) => void; /** * Creates an instance of TaskManagerRunner. @@ -144,9 +147,9 @@ export class TaskManagerRunner implements TaskRunner { * into the total timeout time the task in configured with. We may decide to * start the timer after beforeRun resolves * - * @returns {Promise} + * @returns {Promise>} */ - public async run(): Promise { + public async run(): Promise> { this.logger.debug(`Running task ${this}`); const modifiedContext = await this.beforeRun({ taskInstance: this.instance, @@ -161,7 +164,7 @@ export class TaskManagerRunner implements TaskRunner { this.logger.error(`Task ${this} failed: ${err}`); // in error scenario, we can not get the RunResult // re-use modifiedContext's state, which is correct as of beforeRun - return this.processResult({ error: err, state: modifiedContext.taskInstance.state }); + return this.processResult(asErr({ error: err, state: modifiedContext.taskInstance.state })); } } @@ -224,7 +227,7 @@ export class TaskManagerRunner implements TaskRunner { } performanceStopMarkingTaskAsRunning(); - this.onTaskEvent(asTaskMarkRunningEvent(this.id, asOk(undefined))); + this.onTaskEvent(asTaskMarkRunningEvent(this.id, asOk(this.instance))); return true; } catch (error) { performanceStopMarkingTaskAsRunning(); @@ -251,62 +254,78 @@ export class TaskManagerRunner implements TaskRunner { this.logger.warn(`The task ${this} is not cancellable.`); } - private validateResult(result?: RunResult | void): RunResult { + private validateResult(result?: RunResult | void): Result { const { error } = Joi.validate(result, validateRunResult); if (error) { this.logger.warn(`Invalid task result for ${this}: ${error.message}`); + return asErr({ + error: new Error(`Invalid task result for ${this}: ${error.message}`), + state: {}, + }); + } + if (!result) { + return asOk(EMPTY_RUN_RESULT); } - return result || { state: {} }; + return result.error ? asErr({ ...result, error: result.error as Error }) : asOk(result); } - private async processResultForRecurringTask(result: RunResult): Promise { - // recurring task: update the task instance + private retryAtAfterFailure = ( + failureResult: FailedRunResult + ): Result => { + const { runAt, state, error } = failureResult; + const startedAt = this.instance.startedAt!; - const state = result.state || this.instance.state || {}; - let status: TaskStatus = this.getInstanceStatus(); - - let runAt; - if (status === 'failed') { - // task run errored, keep the same runAt - runAt = this.instance.runAt; - } else if (result.runAt) { - runAt = result.runAt; - } else if (result.error) { - // when result.error is truthy, then we're retrying because it failed - const newRunAt = this.instance.interval - ? intervalFromDate(startedAt, this.instance.interval)! - : this.getRetryDelay({ - attempts: this.instance.attempts, - error: result.error, - }); - - if (!newRunAt) { - status = 'failed'; - runAt = this.instance.runAt; - } else { - runAt = newRunAt; - } - } else { - runAt = intervalFromDate(startedAt, this.instance.interval)!; + + if (runAt) { + return asOk({ state, runAt }); + } + + // when result.error is truthy, then we're retrying because it failed + const newRunAt = this.instance.interval + ? intervalFromDate(startedAt, this.instance.interval)! + : this.getRetryDelay({ + attempts: this.instance.attempts, + error, + }); + + if (newRunAt) { + return asOk({ state, runAt: newRunAt }); } + return asErr(new Error(`Task ${this} retry failed: cannot schedule retry`)); + }; + + private async processResultForRecurringTask( + result: Result + ): Promise { + // recurring task: update the task instance + const { startedAt, interval } = this.instance; + const status: TaskStatus = this.getInstanceStatus(); + + const resolution = + status === 'failed' + ? { status: 'failed' as TaskStatus, runAt: this.instance.runAt } + : resolve }>( + correctError(result, this.retryAtAfterFailure), + ({ runAt, state = this.instance.state }: SuccessfulRunResult) => ({ + runAt: runAt || intervalFromDate(startedAt!, interval)!, + state, + }), + (failedResult: Error) => ({ status: 'failed', runAt: this.instance.runAt }) + ); await this.bufferedTaskStore.update({ ...this.instance, - runAt, - state, - status, + ...resolution, startedAt: null, retryAt: null, ownerId: null, - attempts: result.error ? this.instance.attempts : 0, + attempts: isErr(result) ? this.instance.attempts : 0, }); - - return result; } - private async processResultWhenDone(result: RunResult): Promise { + private async processResultWhenDone(): Promise { // not a recurring task: clean up by removing the task instance from store try { await this.bufferedTaskStore.remove(this.instance.id); @@ -317,22 +336,26 @@ export class TaskManagerRunner implements TaskRunner { throw err; } } - - return result; } - private async processResult(result: RunResult): Promise { - if (result.error) { - await this.processResultForRecurringTask(result); - this.onTaskEvent(asTaskRunEvent(this.id, asErr(result.error))); - } else { - if (result.runAt || this.instance.interval) { + private async processResult( + result: Result + ): Promise> { + await eitherAsync( + result, + async ({ runAt }: SuccessfulRunResult) => { + if (runAt || this.instance.interval) { + await this.processResultForRecurringTask(result); + } else { + await this.processResultWhenDone(); + } + this.onTaskEvent(asTaskRunEvent(this.id, asOk(this.instance))); + }, + async ({ error }: FailedRunResult) => { await this.processResultForRecurringTask(result); - } else { - await this.processResultWhenDone(result); + this.onTaskEvent(asTaskRunEvent(this.id, asErr(error))); } - this.onTaskEvent(asTaskRunEvent(this.id, asOk(undefined))); - } + ); return result; } diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index bfc78c7f9bedc..8821e2078a861 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -627,10 +627,10 @@ describe('TaskStore', () => { must: [ { term: { 'task.status': 'idle' } }, { - term: { - 'task.id': [ - '33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', - 'a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ids: { + values: [ + 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', ], }, }, @@ -649,10 +649,25 @@ describe('TaskStore', () => { type: 'number', order: 'asc', script: { - lang: 'expression', - source: `params.ids.contains(doc['task.id']) ? 0 : (doc['task.retryAt'].value || doc['task.runAt'].value)`, + lang: 'painless', + source: ` +if(params.ids.contains(doc['_id'].value)){ + return 0; +} + +if (doc['task.retryAt'].size()!=0) { + return doc['task.retryAt'].value.toInstant().toEpochMilli(); +} +if (doc['task.runAt'].size()!=0) { + return doc['task.runAt'].value.toInstant().toEpochMilli(); +} + +`, params: { - ids: ['33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', 'a208b22c-14ec-4fb4-995f-d2ff7a3b03b8'], + ids: [ + 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], }, }, }, @@ -969,33 +984,35 @@ describe('TaskStore', () => { index: '', }); - const sub = store.events.pipe(filter((event: TaskEvent) => event.id === 'aaa')).subscribe({ - next: (event: TaskEvent) => { - expect(event).toMatchObject( - asTaskClaimEvent( - 'aaa', - asOk({ - id: 'aaa', - runAt, - taskType: 'foo', - interval: undefined, - attempts: 0, - status: 'idle' as TaskStatus, - params: { hello: 'world' }, - state: { baby: 'Henhen' }, - user: 'jimbo', - scope: ['reporting'], - ownerId: taskManagerId, - startedAt: null, - retryAt: null, - scheduledAt: new Date(), - }) - ) - ); - sub.unsubscribe(); - done(); - }, - }); + const sub = store.events + .pipe(filter((event: TaskEvent) => event.id === 'aaa')) + .subscribe({ + next: (event: TaskEvent) => { + expect(event).toMatchObject( + asTaskClaimEvent( + 'aaa', + asOk({ + id: 'aaa', + runAt, + taskType: 'foo', + interval: undefined, + attempts: 0, + status: 'idle' as TaskStatus, + params: { hello: 'world' }, + state: { baby: 'Henhen' }, + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + }) + ) + ); + sub.unsubscribe(); + done(); + }, + }); await store.claimAvailableTasks({ claimTasksById: ['aaa'], @@ -1024,33 +1041,35 @@ describe('TaskStore', () => { index: '', }); - const sub = store.events.pipe(filter((event: TaskEvent) => event.id === 'bbb')).subscribe({ - next: (event: TaskEvent) => { - expect(event).toMatchObject( - asTaskClaimEvent( - 'bbb', - asOk({ - id: 'bbb', - runAt, - taskType: 'bar', - interval: '5m', - attempts: 2, - status: 'running' as TaskStatus, - params: { shazm: 1 }, - state: { henry: 'The 8th' }, - user: 'dabo', - scope: ['reporting', 'ceo'], - ownerId: taskManagerId, - startedAt: null, - retryAt: null, - scheduledAt: new Date(), - }) - ) - ); - sub.unsubscribe(); - done(); - }, - }); + const sub = store.events + .pipe(filter((event: TaskEvent) => event.id === 'bbb')) + .subscribe({ + next: (event: TaskEvent) => { + expect(event).toMatchObject( + asTaskClaimEvent( + 'bbb', + asOk({ + id: 'bbb', + runAt, + taskType: 'bar', + interval: '5m', + attempts: 2, + status: 'running' as TaskStatus, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + }) + ) + ); + sub.unsubscribe(); + done(); + }, + }); await store.claimAvailableTasks({ claimTasksById: ['aaa'], @@ -1079,15 +1098,17 @@ describe('TaskStore', () => { index: '', }); - const sub = store.events.pipe(filter((event: TaskEvent) => event.id === 'ccc')).subscribe({ - next: (event: TaskEvent) => { - expect(event).toMatchObject( - asTaskClaimEvent('ccc', asErr(new Error(`failed to claim task 'ccc'`))) - ); - sub.unsubscribe(); - done(); - }, - }); + const sub = store.events + .pipe(filter((event: TaskEvent) => event.id === 'ccc')) + .subscribe({ + next: (event: TaskEvent) => { + expect(event).toMatchObject( + asTaskClaimEvent('ccc', asErr(new Error(`failed to claim task 'ccc'`))) + ); + sub.unsubscribe(); + done(); + }, + }); await store.claimAvailableTasks({ claimTasksById: ['ccc'], diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 25e566ba77d22..a853ecb3922ff 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -28,7 +28,7 @@ import { TaskInstance, } from './task'; -import { TaskEvent, asTaskClaimEvent } from './task_events'; +import { TaskClaim, asTaskClaimEvent } from './task_events'; import { asUpdateByQuery, @@ -37,6 +37,8 @@ import { ExistsBoolClause, TermBoolClause, RangeBoolClause, + BoolClause, + IDsClause, } from './queries/query_clauses'; import { @@ -121,7 +123,7 @@ export class TaskStore { private definitions: TaskDictionary; private savedObjectsRepository: SavedObjectsClientContract; private serializer: SavedObjectsSerializer; - private events$: Subject; + private events$: Subject; /** * Constructs a new TaskStore. @@ -141,14 +143,14 @@ export class TaskStore { this.definitions = opts.definitions; this.serializer = opts.serializer; this.savedObjectsRepository = opts.savedObjectsRepository; - this.events$ = new Subject(); + this.events$ = new Subject(); } - public get events(): Observable { + public get events(): Observable { return this.events$; } - private emitEvent = (event: TaskEvent) => { + private emitEvent = (event: TaskClaim) => { this.events$.next(event); }; @@ -254,7 +256,12 @@ export class TaskStore { const { query, sort } = claimTasksById && claimTasksById.length ? { - query: shouldBeOneOf(queryForScheduledTasks, idleTaskWithIDs(claimTasksById)), + query: shouldBeOneOf< + | ExistsBoolClause + | TermBoolClause + | RangeBoolClause + | BoolClause + >(queryForScheduledTasks, idleTaskWithIDs(claimTasksById)), sort: sortByIdsThenByScheduling(claimTasksById), } : { From 5e979b320daa1237ab545ac3dcefa8748c007b16 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 3 Dec 2019 12:32:47 +0000 Subject: [PATCH 08/45] refactored task running result processing --- .../alerting/server/alerts_client.test.ts | 9 +- .../server/lib/task_runner_factory.test.ts | 4 +- .../task_manager/lib/middleware.test.ts | 2 +- .../plugins/task_manager/lib/result_type.ts | 45 ++++--- x-pack/legacy/plugins/task_manager/task.ts | 11 +- .../plugins/task_manager/task_manager.ts | 4 +- .../plugins/task_manager/task_runner.test.ts | 4 +- .../plugins/task_manager/task_runner.ts | 116 +++++++++--------- 8 files changed, 111 insertions(+), 84 deletions(-) diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts index 08607f04a5235..e9afdb872b5b8 100644 --- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts +++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts @@ -9,6 +9,7 @@ import { AlertsClient } from './alerts_client'; import { savedObjectsClientMock, loggingServiceMock } from '../../../../../src/core/server/mocks'; import { taskManagerMock } from '../../task_manager/task_manager.mock'; import { alertTypeRegistryMock } from './alert_type_registry.mock'; +import { TaskStatus } from '../../task_manager'; const taskManager = taskManagerMock.create(); const alertTypeRegistry = alertTypeRegistryMock.create(); @@ -106,7 +107,7 @@ describe('create()', () => { taskType: 'alerting:123', scheduledAt: new Date(), attempts: 1, - status: 'idle', + status: 'idle' as TaskStatus, runAt: new Date(), startedAt: null, retryAt: null, @@ -474,7 +475,7 @@ describe('create()', () => { taskType: 'alerting:123', scheduledAt: new Date(), attempts: 1, - status: 'idle', + status: 'idle' as TaskStatus, runAt: new Date(), startedAt: null, retryAt: null, @@ -554,7 +555,7 @@ describe('enable()', () => { id: 'task-123', scheduledAt: new Date(), attempts: 0, - status: 'idle', + status: 'idle' as TaskStatus, runAt: new Date(), state: {}, params: {}, @@ -631,7 +632,7 @@ describe('enable()', () => { id: 'task-123', scheduledAt: new Date(), attempts: 0, - status: 'idle', + status: 'idle' as TaskStatus, runAt: new Date(), state: {}, params: {}, diff --git a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts index 1d91d4a35d588..3e1a23e37b448 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts @@ -7,7 +7,7 @@ import sinon from 'sinon'; import { schema } from '@kbn/config-schema'; import { AlertExecutorOptions } from '../types'; -import { ConcreteTaskInstance } from '../../../task_manager'; +import { ConcreteTaskInstance, TaskStatus } from '../../../task_manager'; import { TaskRunnerContext, TaskRunnerFactory } from './task_runner_factory'; import { encryptedSavedObjectsMock } from '../../../../../plugins/encrypted_saved_objects/server/mocks'; import { @@ -30,7 +30,7 @@ beforeAll(() => { mockedTaskInstance = { id: '', attempts: 0, - status: 'running', + status: 'running' as TaskStatus, version: '123', runAt: new Date(), scheduledAt: new Date(), diff --git a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts index 699765f16d83e..5e3289eaafd48 100644 --- a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts +++ b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts @@ -37,7 +37,7 @@ const getMockConcreteTaskInstance = () => { sequenceNumber: 1, primaryTerm: 1, attempts: 0, - status: 'idle', + status: TaskStatus.IDLE, runAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()), scheduledAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()), startedAt: null, diff --git a/x-pack/legacy/plugins/task_manager/lib/result_type.ts b/x-pack/legacy/plugins/task_manager/lib/result_type.ts index bf57df981a3cb..b0ed2efcfb501 100644 --- a/x-pack/legacy/plugins/task_manager/lib/result_type.ts +++ b/x-pack/legacy/plugins/task_manager/lib/result_type.ts @@ -4,9 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -// There appears to be an unexported implementation of Either in here: src/core/server/saved_objects/service/lib/repository.ts -// Which is basically the Haskel equivalent of Rust/ML/Scala's Result -// I'll reach out to other's in Kibana to see if we can merge these into one type +import { curry } from 'lodash'; export interface Ok { tag: 'ok'; @@ -49,33 +47,46 @@ export async function promiseResult(future: Promise): Promise(result: Result): T | E { + return isOk(result) ? result.value : result.error; +} + export function either( - result: Result, onOk: (value: T) => void, - onErr: (error: E) => void -): void { - resolve(result, onOk, onErr); + onErr: (error: E) => void, + result: Result +): Result { + resolve(onOk, onErr, result); + return result; } export async function eitherAsync( - result: Result, onOk: (value: T) => Promise, - onErr: (error: E) => Promise + onErr: (error: E) => Promise, + result: Result ): Promise | void> { - return await resolve>(result, onOk, onErr); + await resolve>(onOk, onErr, result); + return result; } export function resolve( - result: Result, onOk: (value: T) => Resolution, - onErr: (error: E) => Resolution + onErr: (error: E) => Resolution, + result: Result ): Resolution { return isOk(result) ? onOk(result.value) : onErr(result.error); } -export function correctError( - result: Result, - correctErr: (error: E) => Result +export const mapOk = curry(function( + onOk: (value: T) => Result, + result: Result +): Result { + return isOk(result) ? onOk(result.value) : result; +}); + +export const mapErr = curry(function( + onErr: (error: E) => Result, + result: Result ): Result { - return isOk(result) ? result : correctErr(result.error); -} + return isOk(result) ? result : onErr(result.error); +}); diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index 9f280b13447ab..fbf2d55e1f589 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -73,6 +73,10 @@ export interface FailedRunResult extends SuccessfulRunResult { error: Error; } +export interface FailedTaskResult { + status: TaskStatus.FAILED; +} + export const validateRunResult = Joi.object({ runAt: Joi.date().optional(), error: Joi.object().optional(), @@ -159,7 +163,12 @@ export interface TaskDictionary { [taskType: string]: T; } -export type TaskStatus = 'idle' | 'claiming' | 'running' | 'failed'; +export enum TaskStatus { + IDLE = 'idle', + CLAIMING = 'claiming', + RUNNING = 'running', + FAILED = 'failed', +} /* * A task instance represents all of the data required to store, fetch, diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 8861f6727818c..ccdca227e3063 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -272,7 +272,6 @@ export class TaskManager { const { id, event } = taskEvent; either( - event, (taskInstance: ConcreteTaskInstance) => { if (isTaskRunEvent(taskEvent)) { subscription.unsubscribe(); @@ -282,7 +281,8 @@ export class TaskManager { (error: Error) => { subscription.unsubscribe(); resolve({ id, error: `${error}` }); - } + }, + event ); }); diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts index 78b3306b7ffc9..af262afd4961a 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts @@ -9,7 +9,7 @@ import sinon from 'sinon'; import { minutesFromNow } from './lib/intervals'; import { asOk, asErr } from './lib/result_type'; import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; -import { ConcreteTaskInstance } from './task'; +import { ConcreteTaskInstance, TaskStatus } from './task'; import { TaskManagerRunner } from './task_runner'; import { mockLogger } from './test_utils'; import { SavedObjectsErrorHelpers } from '../../../../src/core/server/saved_objects/service/lib/errors'; @@ -90,7 +90,7 @@ describe('TaskManagerRunner', () => { const { runner, store } = testOpts({ instance: { interval: '10m', - status: 'running', + status: TaskStatus.RUNNING, startedAt: new Date(), }, definitions: { diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index 86a71fb79d107..29e3c4055fcf6 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -12,9 +12,9 @@ import { performance } from 'perf_hooks'; import Joi from 'joi'; -import { identity } from 'lodash'; +import { identity, defaults, flow } from 'lodash'; -import { asOk, asErr, isErr, correctError, eitherAsync, resolve, Result } from './lib/result_type'; +import { asOk, asErr, mapErr, eitherAsync, unwrap, mapOk, Result } from './lib/result_type'; import { TaskRun, TaskMarkRunning, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { Logger } from './types'; @@ -26,6 +26,7 @@ import { RunResult, SuccessfulRunResult, FailedRunResult, + FailedTaskResult, TaskDefinition, TaskDictionary, validateRunResult, @@ -201,7 +202,7 @@ export class TaskManagerRunner implements TaskRunner { this.instance = await this.bufferedTaskStore.update({ ...taskInstance, - status: 'running', + status: TaskStatus.RUNNING, startedAt: now, attempts, retryAt: this.instance.interval @@ -271,58 +272,72 @@ export class TaskManagerRunner implements TaskRunner { return result.error ? asErr({ ...result, error: result.error as Error }) : asOk(result); } - private retryAtAfterFailure = ( - failureResult: FailedRunResult - ): Result => { - const { runAt, state, error } = failureResult; - - const startedAt = this.instance.startedAt!; - - if (runAt) { - return asOk({ state, runAt }); + private shouldTryToScheduleRetry(): boolean { + if (this.instance.interval) { + return true; } - // when result.error is truthy, then we're retrying because it failed - const newRunAt = this.instance.interval - ? intervalFromDate(startedAt, this.instance.interval)! - : this.getRetryDelay({ - attempts: this.instance.attempts, + const maxAttempts = this.definition.maxAttempts || this.bufferedTaskStore.maxAttempts; + return this.instance.attempts < maxAttempts; + } + + private rescheduleFailedRun = ( + failureResult: FailedRunResult + ): Result => { + if (this.shouldTryToScheduleRetry()) { + const { runAt, state, error } = failureResult; + // if we're retrying, keep the number of attempts + const { interval, attempts } = this.instance; + if (runAt || interval) { + return asOk({ state, attempts, runAt }); + } else { + // when result.error is truthy, then we're retrying because it failed + const newRunAt = this.getRetryDelay({ + attempts, error, }); - if (newRunAt) { - return asOk({ state, runAt: newRunAt }); + if (newRunAt) { + return asOk({ state, attempts, runAt: newRunAt }); + } + } } - return asErr(new Error(`Task ${this} retry failed: cannot schedule retry`)); + // scheduling a retry isn't possible,mark task as failed + return asErr({ status: TaskStatus.FAILED }); }; private async processResultForRecurringTask( result: Result ): Promise { - // recurring task: update the task instance - const { startedAt, interval } = this.instance; - const status: TaskStatus = this.getInstanceStatus(); - - const resolution = - status === 'failed' - ? { status: 'failed' as TaskStatus, runAt: this.instance.runAt } - : resolve }>( - correctError(result, this.retryAtAfterFailure), - ({ runAt, state = this.instance.state }: SuccessfulRunResult) => ({ - runAt: runAt || intervalFromDate(startedAt!, interval)!, - state, - }), - (failedResult: Error) => ({ status: 'failed', runAt: this.instance.runAt }) - ); - - await this.bufferedTaskStore.update({ - ...this.instance, - ...resolution, - startedAt: null, - retryAt: null, - ownerId: null, - attempts: isErr(result) ? this.instance.attempts : 0, - }); + const fieldUpdates = flow( + // if running the task has failed ,try to correct by scheduling a retry in the near future + mapErr(this.rescheduleFailedRun), + // if retrying is possible (new runAt) or this is simply an interval + // based task - reschedule + mapOk(({ runAt, state, attempts = 0 }: Partial) => { + const { startedAt, interval } = this.instance; + return asOk({ + runAt: runAt || intervalFromDate(startedAt!, interval)!, + state, + attempts, + status: TaskStatus.IDLE, + }); + }), + unwrap + )(result); + + await this.bufferedTaskStore.update( + defaults( + { + ...fieldUpdates, + // reset fields that track the lifecycle of the concluded `task run` + startedAt: null, + retryAt: null, + ownerId: null, + }, + this.instance + ) + ); } private async processResultWhenDone(): Promise { @@ -342,7 +357,6 @@ export class TaskManagerRunner implements TaskRunner { result: Result ): Promise> { await eitherAsync( - result, async ({ runAt }: SuccessfulRunResult) => { if (runAt || this.instance.interval) { await this.processResultForRecurringTask(result); @@ -354,20 +368,12 @@ export class TaskManagerRunner implements TaskRunner { async ({ error }: FailedRunResult) => { await this.processResultForRecurringTask(result); this.onTaskEvent(asTaskRunEvent(this.id, asErr(error))); - } + }, + result ); return result; } - private getInstanceStatus() { - if (this.instance.interval) { - return 'idle'; - } - - const maxAttempts = this.definition.maxAttempts || this.bufferedTaskStore.maxAttempts; - return this.instance.attempts < maxAttempts ? 'idle' : 'failed'; - } - private getRetryDelay({ error, attempts, From 6a6b3c844a9d884806ea96e93fac675fdbcdf1be Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 3 Dec 2019 16:49:00 +0000 Subject: [PATCH 09/45] handle runNow of nonexistent tasks --- .../task_manager/lib/middleware.test.ts | 2 +- x-pack/legacy/plugins/task_manager/task.ts | 13 +- .../plugins/task_manager/task_manager.ts | 31 +++- .../plugins/task_manager/task_runner.test.ts | 2 +- .../plugins/task_manager/task_runner.ts | 6 +- .../plugins/task_manager/task_store.test.ts | 140 +++++++++++++++++- .../legacy/plugins/task_manager/task_store.ts | 30 ++++ .../task_manager/task_manager_integration.js | 65 +++++++- 8 files changed, 274 insertions(+), 15 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts index 5e3289eaafd48..3aa39eb3db513 100644 --- a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts +++ b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts @@ -37,7 +37,7 @@ const getMockConcreteTaskInstance = () => { sequenceNumber: 1, primaryTerm: 1, attempts: 0, - status: TaskStatus.IDLE, + status: TaskStatus.Idle, runAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()), scheduledAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()), startedAt: null, diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index fbf2d55e1f589..553fa55d39d01 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -74,7 +74,7 @@ export interface FailedRunResult extends SuccessfulRunResult { } export interface FailedTaskResult { - status: TaskStatus.FAILED; + status: TaskStatus.Failed; } export const validateRunResult = Joi.object({ @@ -164,10 +164,13 @@ export interface TaskDictionary { } export enum TaskStatus { - IDLE = 'idle', - CLAIMING = 'claiming', - RUNNING = 'running', - FAILED = 'failed', + Idle = 'idle', + Claiming = 'claiming', + Running = 'running', + Failed = 'failed', +} +export enum TaskLifecycle { + NotFound = 'notFound', } /* diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index ccdca227e3063..2e9f174b412c7 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -12,7 +12,13 @@ import { Option, none, some, Some, isSome } from 'fp-ts/lib/Option'; import { either } from './lib/result_type'; import { Logger } from './types'; -import { TaskMarkRunning, TaskRun, TaskClaim, isTaskRunEvent } from './task_events'; +import { + TaskMarkRunning, + TaskRun, + TaskClaim, + isTaskRunEvent, + isTaskClaimEvent, +} from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { addMiddlewareToChain, BeforeSaveMiddlewareParams, Middleware } from './lib/middleware'; import { sanitizeTaskDefinitions } from './lib/sanitize_task_definitions'; @@ -24,6 +30,8 @@ import { RunContext, TaskInstanceWithId, TaskInstance, + TaskLifecycle, + TaskStatus, } from './task'; import { TaskPoller } from './task_poller'; import { TaskPool } from './task_pool'; @@ -278,8 +286,27 @@ export class TaskManager { resolve({ id }); } }, - (error: Error) => { + async (error: Error) => { subscription.unsubscribe(); + + try { + if (isTaskClaimEvent(taskEvent)) { + const taskLifecycleStatus = await this.store.getLifecycle(id); + if (taskLifecycleStatus === TaskLifecycle.NotFound) { + resolve({ + id, + error: `Error: failed to run task "${id}" as it does not exist`, + }); + } else if (taskLifecycleStatus !== TaskStatus.Idle) { + resolve({ + id, + error: `Error: failed to run task "${id}" as it is currently running`, + }); + } + } + } catch (err) { + this.logger.error(`Failed to get Task "${id}" as part of runNow: ${err}`); + } resolve({ id, error: `${error}` }); }, event diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts index af262afd4961a..c9fc010ca7912 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts @@ -90,7 +90,7 @@ describe('TaskManagerRunner', () => { const { runner, store } = testOpts({ instance: { interval: '10m', - status: TaskStatus.RUNNING, + status: TaskStatus.Running, startedAt: new Date(), }, definitions: { diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index 29e3c4055fcf6..59dedefe8690c 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -202,7 +202,7 @@ export class TaskManagerRunner implements TaskRunner { this.instance = await this.bufferedTaskStore.update({ ...taskInstance, - status: TaskStatus.RUNNING, + status: TaskStatus.Running, startedAt: now, attempts, retryAt: this.instance.interval @@ -303,7 +303,7 @@ export class TaskManagerRunner implements TaskRunner { } } // scheduling a retry isn't possible,mark task as failed - return asErr({ status: TaskStatus.FAILED }); + return asErr({ status: TaskStatus.Failed }); }; private async processResultForRecurringTask( @@ -320,7 +320,7 @@ export class TaskManagerRunner implements TaskRunner { runAt: runAt || intervalFromDate(startedAt!, interval)!, state, attempts, - status: TaskStatus.IDLE, + status: TaskStatus.Idle, }); }), unwrap diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index 8821e2078a861..fca4295dff89a 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -9,12 +9,13 @@ import sinon from 'sinon'; import uuid from 'uuid'; import { filter } from 'rxjs/operators'; -import { TaskDictionary, TaskDefinition, TaskInstance, TaskStatus } from './task'; +import { TaskDictionary, TaskDefinition, TaskInstance, TaskStatus, TaskLifecycle } from './task'; import { FetchOpts, StoreOpts, OwnershipClaimingOpts, TaskStore } from './task_store'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { SavedObjectsSerializer, SavedObjectsSchema, SavedObjectAttributes } from 'src/core/server'; import { asTaskClaimEvent, TaskEvent } from './task_events'; import { asOk, asErr } from './lib/result_type'; +import { SavedObjectsErrorHelpers } from '../../../../src/core/server/saved_objects/service/lib/errors'; const taskDefinitions: TaskDictionary = { report: { @@ -906,6 +907,143 @@ if (doc['task.runAt'].size()!=0) { }); }); + describe('get', () => { + test('gets the task with the specified id', async () => { + const id = `id-${_.random(1, 20)}`; + const task = { + runAt: mockedDate, + scheduledAt: mockedDate, + startedAt: null, + retryAt: null, + id, + params: { hello: 'world' }, + state: { foo: 'bar' }, + taskType: 'report', + attempts: 3, + status: 'idle' as TaskStatus, + version: '123', + ownerId: null, + }; + + const callCluster = jest.fn(); + savedObjectsClient.get.mockImplementation(async (type: string) => ({ + id, + type, + attributes: { + ..._.omit(task, 'id'), + ..._.mapValues(_.pick(task, 'params', 'state'), value => JSON.stringify(value)), + }, + references: [], + version: '123', + })); + + const store = new TaskStore({ + index: 'tasky', + taskManagerId: '', + serializer, + callCluster, + maxAttempts: 2, + definitions: taskDefinitions, + savedObjectsRepository: savedObjectsClient, + }); + + const result = await store.get(id); + + expect(result).toEqual(task); + + expect(savedObjectsClient.get).toHaveBeenCalledWith('task', id); + }); + }); + + describe('getLifecycle', () => { + test('returns the task status if the task exists ', async () => { + expect.assertions(4); + return Promise.all( + Object.values(TaskStatus).map(async status => { + const id = `id-${_.random(1, 20)}`; + const task = { + runAt: mockedDate, + scheduledAt: mockedDate, + startedAt: null, + retryAt: null, + id, + params: { hello: 'world' }, + state: { foo: 'bar' }, + taskType: 'report', + attempts: 3, + status: status as TaskStatus, + version: '123', + ownerId: null, + }; + + const callCluster = jest.fn(); + savedObjectsClient.get.mockImplementation(async (type: string) => ({ + id, + type, + attributes: { + ..._.omit(task, 'id'), + ..._.mapValues(_.pick(task, 'params', 'state'), value => JSON.stringify(value)), + }, + references: [], + version: '123', + })); + + const store = new TaskStore({ + index: 'tasky', + taskManagerId: '', + serializer, + callCluster, + maxAttempts: 2, + definitions: taskDefinitions, + savedObjectsRepository: savedObjectsClient, + }); + + expect(await store.getLifecycle(id)).toEqual(status); + }) + ); + }); + + test('returns NotFound status if the task doesnt exists ', async () => { + const id = `id-${_.random(1, 20)}`; + + savedObjectsClient.get.mockRejectedValueOnce( + SavedObjectsErrorHelpers.createGenericNotFoundError('type', 'id') + ); + + const store = new TaskStore({ + index: 'tasky', + taskManagerId: '', + serializer, + callCluster: jest.fn(), + maxAttempts: 2, + definitions: taskDefinitions, + savedObjectsRepository: savedObjectsClient, + }); + + expect(await store.getLifecycle(id)).toEqual(TaskLifecycle.NotFound); + }); + + test('throws if an unknown error takes place ', async () => { + const id = `id-${_.random(1, 20)}`; + + savedObjectsClient.get.mockRejectedValueOnce( + SavedObjectsErrorHelpers.createBadRequestError() + ); + + const store = new TaskStore({ + index: 'tasky', + taskManagerId: '', + serializer, + callCluster: jest.fn(), + maxAttempts: 2, + definitions: taskDefinitions, + savedObjectsRepository: savedObjectsClient, + }); + + return expect(store.getLifecycle(id)).rejects.toThrow('Bad Request'); + }); + }); + describe('task events', () => { function generateTasks() { const taskManagerId = uuid.v1(); diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index a853ecb3922ff..733e1ff6492bb 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -26,6 +26,8 @@ import { TaskDefinition, TaskDictionary, TaskInstance, + TaskLifecycle, + TaskStatus, } from './task'; import { TaskClaim, asTaskClaimEvent } from './task_events'; @@ -348,6 +350,34 @@ export class TaskStore { await this.savedObjectsRepository.delete('task', id); } + /** + * Gets a task by id + * + * @param {string} id + * @returns {Promise} + */ + public async get(id: string): Promise { + return savedObjectToConcreteTaskInstance(await this.savedObjectsRepository.get('task', id)); + } + + /** + * Gets task lifecycle step by id + * + * @param {string} id + * @returns {Promise} + */ + public async getLifecycle(id: string): Promise { + try { + const task = await this.get(id); + return task.status; + } catch (err) { + if (err.output && err.output.statusCode === 404) { + return TaskLifecycle.NotFound; + } + throw err; + } + } + private async search(opts: SearchOpts = {}): Promise { const { query } = ensureQueryOnlyReturnsTaskObjects(opts); diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index 53c4f1bd74085..a54f2ac44ae6e 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -255,8 +255,6 @@ export default function ({ getService }) { }); it('should return a task run error result when running a task now fails', async () => { - - const originalTask = await scheduleTask({ taskType: 'sampleTask', interval: `30m`, @@ -302,6 +300,69 @@ export default function ({ getService }) { }); }); + it('should return a task run error result when trying to run a non-existent task', async () => { + // runNow should fail + const failedRunNowResult = await runTaskNow({ + id: 'i-dont-exist' + }); + expect(failedRunNowResult).to.eql({ error: `Error: failed to run task "i-dont-exist" as it does not exist`, id: 'i-dont-exist' }); + }); + + it('should return a task run error result when trying to run a task now which is already running', async () => { + const longRunningTask = await scheduleTask({ + taskType: 'sampleTask', + interval: `30m`, + params: { + waitForEvent: 'rescheduleHasHappened' + }, + }); + + function getTaskById(tasks, id) { + return tasks.filter(task => task.id === id)[0]; + } + + // wait for task to run and stall + await retry.try(async () => { + const docs = await historyDocs(); + expect(docs.filter(taskDoc => taskDoc._source.taskId === longRunningTask.id).length).to.eql(1); + }); + + // first runNow should fail + const failedRunNowResult = await runTaskNow({ + id: longRunningTask.id + }); + + expect( + failedRunNowResult + ).to.eql( + { error: `Error: failed to run task "${longRunningTask.id}" as it is currently running`, id: longRunningTask.id } + ); + + // finish first run + await releaseTasksWaitingForEventToComplete('rescheduleHasHappened'); + await retry.try(async () => { + const tasks = (await currentTasks()).docs; + expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); + }); + + // second runNow should be successful + const successfulRunNowResult = runTaskNow({ + id: longRunningTask.id + }); + + // wait for task to run and stall + await retry.try(async () => { + const docs = await historyDocs(); + expect(docs.filter(taskDoc => taskDoc._source.taskId === longRunningTask.id).length).to.eql(2); + }); + // release it + await releaseTasksWaitingForEventToComplete('rescheduleHasHappened'); + + return successfulRunNowResult.then (successfulRunNowResult => { + expect(successfulRunNowResult).to.eql({ id: longRunningTask.id }); + }); + }); + async function expectReschedule(originalRunAt, currentTask, expectedDiff) { const buffer = 10000; expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.greaterThan(expectedDiff - buffer); From 8e83283f23e278160e60264c1679205b2dda1278 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 4 Dec 2019 09:45:28 +0000 Subject: [PATCH 10/45] added waitOnceForEvent in SampleTask to clean up tests --- .../plugins/task_manager/index.js | 9 ++++++--- .../task_manager/task_manager_integration.js | 20 +++++-------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js index 35f5f648768b2..98b8be463bdea 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js @@ -46,10 +46,10 @@ export default function TaskTestingAPI(kibana) { const { params, state } = taskInstance; const prevState = state || { count: 0 }; - const count = (prevState.count || 0) + 1; + const stateUpdate = { count: (prevState.count || 0) + 1 }; if (params.failWith) { - if (!params.failOn || (params.failOn && count === params.failOn)) { + if (!params.failOn || (params.failOn && stateUpdate.count === params.failOn)) { throw new Error(params.failWith); } } @@ -70,10 +70,13 @@ export default function TaskTestingAPI(kibana) { if (params.waitForEvent) { await once(taskTestingEvents, params.waitForEvent); + } else if (params.waitOnceForEvent && !state.hasWaitedOnce) { + await once(taskTestingEvents, params.waitOnceForEvent); + stateUpdate.hasWaitedOnce = true; } return { - state: { count }, + state: stateUpdate, runAt: millisecondsFromNow(params.nextRunMilliseconds), }; }, diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index a54f2ac44ae6e..aa87d7a7f87bb 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -313,7 +313,7 @@ export default function ({ getService }) { taskType: 'sampleTask', interval: `30m`, params: { - waitForEvent: 'rescheduleHasHappened' + waitOnceForEvent: 'runNowHasBeenCalled' }, }); @@ -328,7 +328,7 @@ export default function ({ getService }) { }); // first runNow should fail - const failedRunNowResult = await runTaskNow({ + const failedRunNowResult = await runTaskNow({ id: longRunningTask.id }); @@ -339,28 +339,18 @@ export default function ({ getService }) { ); // finish first run - await releaseTasksWaitingForEventToComplete('rescheduleHasHappened'); + await releaseTasksWaitingForEventToComplete('runNowHasBeenCalled'); await retry.try(async () => { const tasks = (await currentTasks()).docs; expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); }); // second runNow should be successful - const successfulRunNowResult = runTaskNow({ + const successfulRunNowResult = await runTaskNow({ id: longRunningTask.id }); - // wait for task to run and stall - await retry.try(async () => { - const docs = await historyDocs(); - expect(docs.filter(taskDoc => taskDoc._source.taskId === longRunningTask.id).length).to.eql(2); - }); - // release it - await releaseTasksWaitingForEventToComplete('rescheduleHasHappened'); - - return successfulRunNowResult.then (successfulRunNowResult => { - expect(successfulRunNowResult).to.eql({ id: longRunningTask.id }); - }); + expect(successfulRunNowResult).to.eql({ id: longRunningTask.id }); }); async function expectReschedule(originalRunAt, currentTask, expectedDiff) { From 08bd67fc7d55e9ef38ef1b0a4077b3b3b06aa228 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 4 Dec 2019 12:43:23 +0000 Subject: [PATCH 11/45] enable running a failed task using runNow --- .../mark_available_tasks_as_claimed.ts | 8 +- .../plugins/task_manager/task_manager.ts | 12 +- .../legacy/plugins/task_manager/task_store.ts | 4 +- .../plugins/task_manager/index.js | 111 ++++++++++-------- .../plugins/task_manager/init_routes.js | 7 +- .../task_manager/task_manager_integration.js | 79 ++++++++++--- 6 files changed, 145 insertions(+), 76 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index dd81f153f2810..1da4550998885 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -41,12 +41,16 @@ export const IdleTaskWithExpiredRunAt: BoolClause => ({ bool: { must: [ - { term: { 'task.status': 'idle' } }, + { + bool: { + should: [{ term: { 'task.status': 'idle' } }, { term: { 'task.status': 'failed' } }], + }, + }, { ids: { values: claimTasksById, diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 2e9f174b412c7..e2a5b6bd3876c 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -55,14 +55,10 @@ export interface TaskManagerOpts { serializer: SavedObjectsSerializer; } -type RunNowResult = - | { - id: string; - error: string; - } - | { - id: string; - }; +interface RunNowResult { + id: string; + error?: string; +} type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 733e1ff6492bb..297fff7f16a54 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -50,7 +50,7 @@ import { RecuringTaskWithInterval, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, - idleTaskWithIDs, + taskWithIDsAndRunnableStatus, sortByIdsThenByScheduling, } from './queries/mark_available_tasks_as_claimed'; @@ -263,7 +263,7 @@ export class TaskStore { | TermBoolClause | RangeBoolClause | BoolClause - >(queryForScheduledTasks, idleTaskWithIDs(claimTasksById)), + >(queryForScheduledTasks, taskWithIDsAndRunnableStatus(claimTasksById)), sort: sortByIdsThenByScheduling(claimTasksById), } : { diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js index 98b8be463bdea..59977e8f6fce9 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/index.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/index.js @@ -11,7 +11,7 @@ import { initRoutes } from './init_routes'; const once = function (emitter, event) { return new Promise(resolve => { - emitter.once(event, resolve); + emitter.once(event, (data) => resolve(data || {})); }); }; @@ -28,59 +28,76 @@ export default function TaskTestingAPI(kibana) { }).default(); }, + init(server) { const taskManager = server.plugins.task_manager; + const defaultSampleTaskConfig = { + timeout: '1m', + // This task allows tests to specify its behavior (whether it reschedules itself, whether it errors, etc) + // taskInstance.params has the following optional fields: + // nextRunMilliseconds: number - If specified, the run method will return a runAt that is now + nextRunMilliseconds + // failWith: string - If specified, the task will throw an error with the specified message + // failOn: number - If specified, the task will only throw the `failWith` error when `count` equals to the failOn value + // waitForParams : boolean - should the task stall ands wait to receive params asynchronously before using the default params + // waitForEvent : string - if provided, the task will stall (after completing the run) and wait for an asyn event before completing + createTaskRunner: ({ taskInstance }) => ({ + async run() { + const { params, state, id } = taskInstance; + const prevState = state || { count: 0 }; + + const count = (prevState.count || 0) + 1; + + const runParams = { + ...params, + // if this task requires custom params provided async - wait for them + ...(params.waitForParams ? await once(taskTestingEvents, id) : {}) + }; + + if (runParams.failWith) { + if (!runParams.failOn || (runParams.failOn && count === runParams.failOn)) { + throw new Error(runParams.failWith); + } + } + + const callCluster = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser; + await callCluster('index', { + index: '.kibana_task_manager_test_result', + body: { + type: 'task', + taskId: taskInstance.id, + params: JSON.stringify(runParams), + state: JSON.stringify(state), + ranAt: new Date(), + }, + refresh: true, + }); + + // Stall task run until a certain event is triggered + if (runParams.waitForEvent) { + await once(taskTestingEvents, runParams.waitForEvent); + } + + return { + state: { count }, + runAt: millisecondsFromNow(runParams.nextRunMilliseconds), + }; + }, + }), + }; + taskManager.registerTaskDefinitions({ sampleTask: { + ...defaultSampleTaskConfig, title: 'Sample Task', description: 'A sample task for testing the task_manager.', - timeout: '1m', - - // This task allows tests to specify its behavior (whether it reschedules itself, whether it errors, etc) - // taskInstance.params has the following optional fields: - // nextRunMilliseconds: number - If specified, the run method will return a runAt that is now + nextRunMilliseconds - // failWith: string - If specified, the task will throw an error with the specified message - createTaskRunner: ({ taskInstance }) => ({ - async run() { - const { params, state } = taskInstance; - const prevState = state || { count: 0 }; - - const stateUpdate = { count: (prevState.count || 0) + 1 }; - - if (params.failWith) { - if (!params.failOn || (params.failOn && stateUpdate.count === params.failOn)) { - throw new Error(params.failWith); - } - } - - - const callCluster = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser; - await callCluster('index', { - index: '.kibana_task_manager_test_result', - body: { - type: 'task', - taskId: taskInstance.id, - params: JSON.stringify(params), - state: JSON.stringify(state), - ranAt: new Date(), - }, - refresh: true, - }); - - if (params.waitForEvent) { - await once(taskTestingEvents, params.waitForEvent); - } else if (params.waitOnceForEvent && !state.hasWaitedOnce) { - await once(taskTestingEvents, params.waitOnceForEvent); - stateUpdate.hasWaitedOnce = true; - } - - return { - state: stateUpdate, - runAt: millisecondsFromNow(params.nextRunMilliseconds), - }; - }, - }), + }, + singleAttemptSampleTask: { + ...defaultSampleTaskConfig, + title: 'Failing Sample Task', + description: 'A sample task for testing the task_manager that fails on the first attempt to run.', + // fail after the first failed run + maxAttempts: 1, }, }); diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index 76eefa66bb30b..591172ef3deee 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -123,14 +123,15 @@ export function initRoutes(server, taskTestingEvents) { config: { validate: { payload: Joi.object({ - event: Joi.string().required() + event: Joi.string().required(), + data: Joi.object().optional().default({}) }), }, }, async handler(request) { try { - const { event } = request.payload; - taskTestingEvents.emit(event); + const { event, data } = request.payload; + taskTestingEvents.emit(event, data); return { event }; } catch (err) { return err; diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index aa87d7a7f87bb..80240e2faf9de 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -88,6 +88,23 @@ export default function ({ getService }) { .expect(200); } + function getTaskById(tasks, id) { + return tasks.filter(task => task.id === id)[0]; + } + + async function provideParamsToTasksWaitingForParams(taskId, data = {}) { + // wait for task to start running and stall on waitForParams + await retry.try(async () => { + const tasks = (await currentTasks()).docs; + expect(getTaskById(tasks, taskId).status).to.eql('running'); + }); + + return supertest.post('/api/sample_tasks/event') + .set('kbn-xsrf', 'xxx') + .send({ event: taskId, data }) + .expect(200); + } + it('should support middleware', async () => { const historyItem = _.random(1, 100); @@ -311,17 +328,17 @@ export default function ({ getService }) { it('should return a task run error result when trying to run a task now which is already running', async () => { const longRunningTask = await scheduleTask({ taskType: 'sampleTask', - interval: `30m`, + interval: '30m', params: { - waitOnceForEvent: 'runNowHasBeenCalled' + waitForParams: true }, }); - function getTaskById(tasks, id) { - return tasks.filter(task => task.id === id)[0]; - } + // tell the task to wait for the 'runNowHasBeenAttempted' event + await provideParamsToTasksWaitingForParams(longRunningTask.id, { + waitForEvent: 'runNowHasBeenAttempted' + }); - // wait for task to run and stall await retry.try(async () => { const docs = await historyDocs(); expect(docs.filter(taskDoc => taskDoc._source.taskId === longRunningTask.id).length).to.eql(1); @@ -338,19 +355,57 @@ export default function ({ getService }) { { error: `Error: failed to run task "${longRunningTask.id}" as it is currently running`, id: longRunningTask.id } ); - // finish first run - await releaseTasksWaitingForEventToComplete('runNowHasBeenCalled'); + // finish first run by emitting 'runNowHasBeenAttempted' event + await releaseTasksWaitingForEventToComplete('runNowHasBeenAttempted'); await retry.try(async () => { const tasks = (await currentTasks()).docs; expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); }); // second runNow should be successful - const successfulRunNowResult = await runTaskNow({ + const successfulRunNowResult = runTaskNow({ id: longRunningTask.id }); - expect(successfulRunNowResult).to.eql({ id: longRunningTask.id }); + await provideParamsToTasksWaitingForParams(longRunningTask.id); + + expect(await successfulRunNowResult).to.eql({ id: longRunningTask.id }); + }); + + it('should allow a failed task to be rerun using runNow', async () => { + + const taskThatFailsBeforeRunNow = await scheduleTask({ + taskType: 'singleAttemptSampleTask', + params: { + waitForParams: true + }, + }); + + // tell the task to fail on its next run + await provideParamsToTasksWaitingForParams( + taskThatFailsBeforeRunNow.id, + { failWith: 'error on first run' } + ); + + // wait for task to fail + await retry.try(async () => { + const tasks = (await currentTasks()).docs; + expect(getTaskById(tasks, taskThatFailsBeforeRunNow.id).status).to.eql('failed'); + }); + + // runNow should be successfully run the failing task + const runNowResultWithExpectedFailure = runTaskNow({ + id: taskThatFailsBeforeRunNow.id + }); + + // release the task without failing this time + await provideParamsToTasksWaitingForParams(taskThatFailsBeforeRunNow.id); + + expect( + await runNowResultWithExpectedFailure + ).to.eql( + { id: taskThatFailsBeforeRunNow.id } + ); }); async function expectReschedule(originalRunAt, currentTask, expectedDiff) { @@ -380,10 +435,6 @@ export default function ({ getService }) { }, }); - function getTaskById(tasks, id) { - return tasks.filter(task => task.id === id)[0]; - } - await retry.try(async () => { const tasks = (await currentTasks()).docs; expect(getTaskById(tasks, fastTask.id).state.count).to.eql(2); From 3a9f82f1842e8ade6245a52553fc010523d62c6e Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 4 Dec 2019 14:44:43 +0000 Subject: [PATCH 12/45] added documentation --- .../plugins/task_manager/task_manager.ts | 12 +++++---- .../plugins/task_manager/task_poller.ts | 25 ++++++++++++++++--- .../plugins/task_manager/task_store.test.ts | 9 ++++++- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index e2a5b6bd3876c..4cbcd7c170538 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -5,6 +5,7 @@ */ import { Subject, merge } from 'rxjs'; import { filter } from 'rxjs/operators'; +import { uniq } from 'lodash'; import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; @@ -146,14 +147,11 @@ export class TaskManager { private pollForWork = async ( ...optionalTasksToClaim: Array> ): Promise => { - const tasksToClaim = optionalTasksToClaim - .filter(isSome) - .map((task: Some) => task.value); return fillPool( // claim available tasks () => claimAvailableTasks( - tasksToClaim.splice(0, this.pool.availableWorkers), + rejectDuplicateAndEmptyValues(optionalTasksToClaim), this.store.claimAvailableTasks, this.pool.availableWorkers, this.logger @@ -368,6 +366,10 @@ export class TaskManager { } } +function rejectDuplicateAndEmptyValues(values: Array>) { + return uniq(values.filter(isSome).map((optional: Some) => optional.value)); +} + export async function claimAvailableTasks( claimTasksById: string[], claim: (opts: OwnershipClaimingOpts) => Promise, @@ -381,7 +383,7 @@ export async function claimAvailableTasks( const { docs, claimedTasks } = await claim({ size: availableWorkers, claimOwnershipUntil: intervalFromNow('30s')!, - claimTasksById, + claimTasksById: claimTasksById.splice(0, availableWorkers), }); if (claimedTasks === 0) { diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 5cbb57b7f24bd..e69d0c605eb74 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -48,9 +48,25 @@ export class TaskPoller { this.pollingSubscription = Subscription.EMPTY; this.pollPhaseResults$ = new Subject(); this.claimRequestQueue$ = new Subject(); - this.poller$ = this.claimRequestQueue$.pipe( - buffer(interval(opts.pollInterval).pipe(throttle(ev => this.pollPhaseResults$))) - ); + this.poller$ = + // queue of requests for work to be done, each request can provide a single + // argument of type `H` + this.claimRequestQueue$.pipe( + // buffer these requests (so that multiple requests get flattened into + // a single request which receives an array of arguments of type `H`) + buffer( + // flush this buffer at the fixed interval specified under `pollInterval` + interval(opts.pollInterval).pipe( + throttle( + // only emit one interval event and then ignore every subsequent interval + // event until a corresponding completion of `attemptWork`. This prevents + // us from flushing the request buffer until the work has been completed + // the previous calls to `attemptWork` + () => this.pollPhaseResults$ + ) + ) + ) + ); } /** @@ -82,7 +98,8 @@ export class TaskPoller { */ private attemptWork = async (...requests: H[]) => { try { - this.pollPhaseResults$.next(asOk(await this.work(...requests))); + const workResult = await this.work(...requests); + this.pollPhaseResults$.next(asOk(workResult)); } catch (err) { this.logger.error(`Failed to poll for work: ${err}`); this.pollPhaseResults$.next(asErr(err)); diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index fca4295dff89a..a5d395821ea3d 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -626,7 +626,14 @@ describe('TaskStore', () => { { bool: { must: [ - { term: { 'task.status': 'idle' } }, + { + bool: { + should: [ + { term: { 'task.status': 'idle' } }, + { term: { 'task.status': 'failed' } }, + ], + }, + }, { ids: { values: [ From f4a4d285706ee16d3738b88ef435a82d8be247aa Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 5 Dec 2019 19:23:02 +0000 Subject: [PATCH 13/45] remodeled poller using streams of push events --- x-pack/legacy/plugins/task_manager/README.md | 30 ++ .../plugins/task_manager/lib/pull_from_set.ts | 21 + .../plugins/task_manager/lib/result_type.ts | 4 +- .../plugins/task_manager/task_manager.ts | 97 ++--- .../task_poller.intervals.test.ts | 40 -- .../plugins/task_manager/task_poller.test.ts | 363 +++++++++++++----- .../plugins/task_manager/task_poller.ts | 158 ++++---- .../legacy/plugins/task_manager/task_pool.ts | 7 + .../plugins/task_manager/task_store.test.ts | 8 +- x-pack/package.json | 1 + yarn.lock | 18 +- 11 files changed, 459 insertions(+), 288 deletions(-) create mode 100644 x-pack/legacy/plugins/task_manager/lib/pull_from_set.ts delete mode 100644 x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index 744643458e136..aba784b5044d2 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -305,6 +305,36 @@ server.plugins.task_manager.addMiddleware({ }); ``` +## Task Poller: polling for work +TaskManager used to work in a `pull` model, but it now needs to support both `push` and `pull`, so it has been remodeled internally to support a single `push` model. + +Task Manager's lifecycle is pushed by the following operations: + +1. A polling interval has been reached. +2. A new Task is scheduled. +3. A Task is run using `runNow`. + +The polling interval straight forward: TaskPoller is configured to emit an event at a fixed interval. +We wish to ignore any polling interval that goes off when there are no workers available, so we'll throttle that on workerAvailability + +Every time a Task is scheduled we want to trigger an early polling in order to respond to the newly scheduled task asap, but this too we only wish to do if there are available workers, so we can throttle this too. + +When a runNow call is made we need to force a poll as the user will now be waiting on the result of the runNow call, but +there is a complexity here- we don't want to force polling as there might not be any worker capacity, but we also can't throttle, as we can't afford to "drop" these requests (as we are bypassing normal scheduling), so we'll have to buffer these. + +We now want to respond to all three of these push events, but we still need to balance against our worker capacity, so if there are too many requests buffered, we only want to `take` as many requests as we have capacity top handle. +Luckily, `Polling Interval` and `Task Scheduled` simply denote a request to "poll for work as soon as possible", unlike `Run Task Now` which also means "poll for these specific tasks", so our capacity only needs to be applied to `Run Task Now`. + +We achieve this model by maintaining a queue using a Set (which removes duplicated). +TODO: We don't want an unbounded queue, sobest to add a configurable cap and return an error to the `runNow` call when this cap is reached. + +Our current model, then, is this: +``` + Polling Interval --> filter(workerAvailability > 0) -- [] --\ + Task Scheduled --> filter(workerAvailability > 0) -- [] ---|==> Set([] + [] + [`ID`]) ==> work([`ID`]) + Run Task `ID` Now --> buffer(workerAvailability > 0) -- [`ID`] --/ +``` + ## Limitations in v1.0 In v1, the system only understands 1 minute increments (e.g. '1m', '7m'). Tasks which need something more robust will need to specify their own "runAt" in their run method's return value. diff --git a/x-pack/legacy/plugins/task_manager/lib/pull_from_set.ts b/x-pack/legacy/plugins/task_manager/lib/pull_from_set.ts new file mode 100644 index 0000000000000..ea77ea8dbd55e --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/lib/pull_from_set.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export function pullFromSet(set: Set, capacity: number) { + if (capacity > 0 && set.size > 0) { + const values = []; + for (const value of set) { + if (set.delete(value)) { + values.push(value); + if (values.length === capacity) { + return values; + } + } + } + return values; + } + return []; +} diff --git a/x-pack/legacy/plugins/task_manager/lib/result_type.ts b/x-pack/legacy/plugins/task_manager/lib/result_type.ts index b0ed2efcfb501..eafaff6c31bc6 100644 --- a/x-pack/legacy/plugins/task_manager/lib/result_type.ts +++ b/x-pack/legacy/plugins/task_manager/lib/result_type.ts @@ -51,14 +51,14 @@ export function unwrap(result: Result): T | E { return isOk(result) ? result.value : result.error; } -export function either( +export const either = curry(function( onOk: (value: T) => void, onErr: (error: E) => void, result: Result ): Result { resolve(onOk, onErr, result); return result; -} +}); export async function eitherAsync( onOk: (value: T) => Promise, diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 4cbcd7c170538..e0b8c605beccd 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Subject, merge } from 'rxjs'; +import { Subject, Observable, Subscription, merge } from 'rxjs'; import { filter } from 'rxjs/operators'; -import { uniq } from 'lodash'; + import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; -import { Option, none, some, Some, isSome } from 'fp-ts/lib/Option'; -import { either } from './lib/result_type'; +import { Option, none, some } from 'fp-ts/lib/Option'; +import { Result, either, mapErr } from './lib/result_type'; import { Logger } from './types'; import { @@ -34,7 +34,7 @@ import { TaskLifecycle, TaskStatus, } from './task'; -import { TaskPoller } from './task_poller'; +import { createTaskPoller } from './task_poller'; import { TaskPool } from './task_pool'; import { TaskManagerRunner, TaskRunner } from './task_runner'; import { @@ -77,15 +77,15 @@ type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; * The public interface into the task manager system. */ export class TaskManager { - private isStarted = false; private maxWorkers: number; private definitions: TaskDictionary; private store: TaskStore; private logger: Logger; private pool: TaskPool; private events$: Subject; - - private poller: TaskPoller>; + private claimRequests$: Subject>; + private pollingSubscription: Subscription; + private poller$: Observable>; private startQueue: Array<() => void> = []; private middleware = { @@ -130,12 +130,15 @@ export class TaskManager { }); this.events$ = new Subject(); + this.claimRequests$ = new Subject(); // if we end up with only one stream, remove merge merge(this.store.events).subscribe(event => this.events$.next(event)); - this.poller = new TaskPoller>({ - logger: this.logger, + this.pollingSubscription = Subscription.EMPTY; + this.poller$ = createTaskPoller({ pollInterval: opts.config.get('xpack.task_manager.poll_interval'), + getCapacity: () => this.pool.availableWorkers, + pollRequests$: this.claimRequests$, work: this.pollForWork, }); } @@ -144,27 +147,8 @@ export class TaskManager { this.events$.next(event); }; - private pollForWork = async ( - ...optionalTasksToClaim: Array> - ): Promise => { - return fillPool( - // claim available tasks - () => - claimAvailableTasks( - rejectDuplicateAndEmptyValues(optionalTasksToClaim), - this.store.claimAvailableTasks, - this.pool.availableWorkers, - this.logger - ), - // wrap each task in a Task Runner - this.createTaskRunnerForTask, - // place tasks in the Task Pool - async (tasks: TaskRunner[]) => await this.pool.run(tasks) - ); - }; - private attemptToRun(task: Option = none) { - this.poller.queueWork(task); + this.claimRequests$.next(task); } private createTaskRunnerForTask = (instance: ConcreteTaskInstance) => { @@ -179,20 +163,40 @@ export class TaskManager { }); }; + public get isStarted() { + return this.pollingSubscription && !this.pollingSubscription.closed; + } + + private pollForWork = async (...tasksToClaim: string[]): Promise => { + return fillPool( + // claim available tasks + () => + claimAvailableTasks( + tasksToClaim.splice(0, this.pool.availableWorkers), + this.store.claimAvailableTasks, + this.pool.availableWorkers, + this.logger + ), + // wrap each task in a Task Runner + this.createTaskRunnerForTask, + // place tasks in the Task Pool + async (tasks: TaskRunner[]) => await this.pool.run(tasks) + ); + }; + /** * Starts up the task manager and starts picking up tasks. */ public start() { - if (this.isStarted) { - return; - } - this.isStarted = true; - - // Some calls are waiting until task manager is started - this.startQueue.forEach(fn => fn()); - this.startQueue = []; + if (!this.isStarted) { + // Some calls are waiting until task manager is started + this.startQueue.forEach(fn => fn()); + this.startQueue = []; - this.poller.start(); + this.pollingSubscription = this.poller$.subscribe( + mapErr((error: string) => this.logger.error(error)) + ); + } } private async waitUntilStarted() { @@ -207,8 +211,10 @@ export class TaskManager { * Stops the task manager and cancels running tasks. */ public stop() { - this.poller.stop(); - this.pool.cancelRunningTasks(); + if (this.isStarted) { + this.pollingSubscription.unsubscribe(); + this.pool.cancelRunningTasks(); + } } /** @@ -272,8 +278,7 @@ export class TaskManager { .pipe(filter(({ id }: TaskLifecycleEvent) => id === task)) .subscribe((taskEvent: TaskLifecycleEvent) => { const { id, event } = taskEvent; - - either( + either( (taskInstance: ConcreteTaskInstance) => { if (isTaskRunEvent(taskEvent)) { subscription.unsubscribe(); @@ -366,10 +371,6 @@ export class TaskManager { } } -function rejectDuplicateAndEmptyValues(values: Array>) { - return uniq(values.filter(isSome).map((optional: Some) => optional.value)); -} - export async function claimAvailableTasks( claimTasksById: string[], claim: (opts: OwnershipClaimingOpts) => Promise, @@ -383,7 +384,7 @@ export async function claimAvailableTasks( const { docs, claimedTasks } = await claim({ size: availableWorkers, claimOwnershipUntil: intervalFromNow('30s')!, - claimTasksById: claimTasksById.splice(0, availableWorkers), + claimTasksById, }); if (claimedTasks === 0) { diff --git a/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts deleted file mode 100644 index a370e141a12ec..0000000000000 --- a/x-pack/legacy/plugins/task_manager/task_poller.intervals.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import _ from 'lodash'; -import { interval } from 'rxjs'; -import { TaskPoller } from './task_poller'; -import { mockLogger } from './test_utils'; - -jest.mock('rxjs', () => ({ - Subject: jest.fn(() => ({ - pipe: jest.fn(() => ({ - pipe: jest.fn(), - subscribe: jest.fn(), - })), - })), - Subscription: jest.fn(), - Observable: jest.fn(() => ({ - pipe: jest.fn(), - })), - interval: jest.fn(() => ({ - pipe: jest.fn(), - })), -})); - -describe('TaskPoller Intervals', () => { - test('intializes with the provided interval', () => { - const pollInterval = _.random(10, 20); - - new TaskPoller({ - pollInterval, - work: async () => {}, - logger: mockLogger(), - }); - - expect(interval).toHaveBeenCalledWith(pollInterval); - }); -}); diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts index adb4d8040b7ea..dbc51fcd6516b 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts @@ -5,130 +5,281 @@ */ import _ from 'lodash'; -import { TaskPoller } from './task_poller'; -import { mockLogger, resolvable, sleep } from './test_utils'; +import { Subject } from 'rxjs'; +import { Option, none, some } from 'fp-ts/lib/Option'; +import { createTaskPoller } from './task_poller'; +import { fakeSchedulers } from 'rxjs-marbles/jest'; +import { sleep, resolvable } from './test_utils'; +import { asOk, asErr } from './lib/result_type'; describe('TaskPoller', () => { - describe('lifecycle', () => { - test('logs, but does not crash if the work function fails', async done => { - const logger = mockLogger(); - - let count = 0; - const work = jest.fn(async () => { - ++count; - if (count === 1) { - throw new Error('Dang it!'); - } else if (count > 1) { - poller.stop(); - - expect(work).toHaveBeenCalledTimes(2); - expect(logger.error.mock.calls[0][0]).toMatchInlineSnapshot( - `"Failed to poll for work: Error: Dang it!"` - ); - - done(); - } - }); - - const poller = new TaskPoller({ - logger, - pollInterval: 1, + beforeEach(() => jest.useFakeTimers()); + + test( + 'intializes the poller with the provided interval', + fakeSchedulers(async advance => { + const pollInterval = 100; + const halfInterval = Math.floor(pollInterval / 2); + + const work = jest.fn(async () => true); + createTaskPoller({ + pollInterval, + getCapacity: () => 1, work, - }); + pollRequests$: new Subject>(), + }).subscribe(() => {}); + + // `work` is async, we have to force a node `tick` + await sleep(0); + advance(halfInterval); + expect(work).toHaveBeenCalledTimes(0); + advance(halfInterval); + + await sleep(0); + expect(work).toHaveBeenCalledTimes(1); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(2); + }) + ); - poller.start(); - }); + test( + 'filters interval polling on capacity', + fakeSchedulers(async advance => { + const pollInterval = 100; - test('is stoppable', async () => { - const doneWorking = resolvable(); - const work = jest.fn(async () => { - poller.stop(); - doneWorking.resolve(); - }); + const work = jest.fn(async () => true); - const poller = new TaskPoller({ - logger: mockLogger(), - pollInterval: 1, + let hasCapacity = true; + createTaskPoller({ + pollInterval, work, - }); + getCapacity: () => (hasCapacity ? 1 : 0), + pollRequests$: new Subject>(), + }).subscribe(() => {}); - poller.start(); - await doneWorking; + expect(work).toHaveBeenCalledTimes(0); + + await sleep(0); + advance(pollInterval); expect(work).toHaveBeenCalledTimes(1); - await sleep(100); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(2); + + hasCapacity = false; + + await sleep(0); + advance(pollInterval); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(2); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(2); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(2); + + hasCapacity = true; + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(3); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(4); + }) + ); + + test( + 'requests with no arguments (nudge requests) are queued on-demand in between intervals', + fakeSchedulers(async advance => { + const pollInterval = 100; + const querterInterval = Math.floor(pollInterval / 4); + const halfInterval = querterInterval * 2; + + const work = jest.fn(async () => true); + const pollRequests$ = new Subject>(); + createTaskPoller({ + pollInterval, + work, + getCapacity: () => 1, + pollRequests$, + }).subscribe(jest.fn()); + + expect(work).toHaveBeenCalledTimes(0); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(1); + + advance(querterInterval); + await sleep(0); expect(work).toHaveBeenCalledTimes(1); - }); - - test('disregards duplicate calls to "start"', async () => { - const doneWorking = resolvable(); - const work = jest.fn(async () => { - await doneWorking; - }); - const poller = new TaskPoller({ - pollInterval: 1, - logger: mockLogger(), + + pollRequests$.next(none); + + expect(work).toHaveBeenCalledTimes(2); + expect(work).toHaveBeenNthCalledWith(2); + + await sleep(0); + advance(querterInterval); + expect(work).toHaveBeenCalledTimes(2); + + await sleep(0); + advance(halfInterval); + expect(work).toHaveBeenCalledTimes(3); + }) + ); + + test( + 'requests with no arguments (nudge requests) are dropped when there is no capacity', + fakeSchedulers(async advance => { + const pollInterval = 100; + const querterInterval = Math.floor(pollInterval / 4); + const halfInterval = querterInterval * 2; + + let hasCapacity = true; + const work = jest.fn(async () => true); + const pollRequests$ = new Subject>(); + createTaskPoller({ + pollInterval, work, - }); + getCapacity: () => (hasCapacity ? 1 : 0), + pollRequests$, + }).subscribe(() => {}); - poller.start(); - await sleep(10); - poller.start(); - poller.start(); - await sleep(10); - poller.start(); - await sleep(10); + expect(work).toHaveBeenCalledTimes(0); - poller.stop(); + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(1); + hasCapacity = false; - doneWorking.resolve(); + await sleep(0); + advance(querterInterval); - await sleep(10); + pollRequests$.next(none); expect(work).toHaveBeenCalledTimes(1); - }); - - test('waits for work before polling', async () => { - const doneWorking = resolvable(); - const work = jest.fn(async () => { - await sleep(10); - poller.stop(); - doneWorking.resolve(); - }); - const poller = new TaskPoller({ - pollInterval: 1, - logger: mockLogger(), + + await sleep(0); + advance(querterInterval); + + hasCapacity = true; + advance(halfInterval); + expect(work).toHaveBeenCalledTimes(2); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledTimes(3); + }) + ); + + test( + 'requests with arguments are emitted', + fakeSchedulers(async advance => { + const pollInterval = 100; + + const work = jest.fn(async () => true); + const pollRequests$ = new Subject>(); + createTaskPoller({ + pollInterval, work, - }); + getCapacity: () => 1, + pollRequests$, + }).subscribe(() => {}); - poller.start(); - await doneWorking; + advance(pollInterval); - expect(work).toHaveBeenCalledTimes(1); - }); - - test('queues claim requests while working', async done => { - let count = 0; - - const poller = new TaskPoller({ - pollInterval: 1, - logger: mockLogger(), - work: jest.fn(async (first, second) => { - count++; - if (count === 1) { - poller.queueWork('asd'); - poller.queueWork('123'); - } else if (count === 2) { - expect(first).toEqual('asd'); - expect(second).toEqual('123'); - - done(); - } else { - poller.stop(); - } - }), - }); - - poller.start(); - }); - }); + pollRequests$.next(some('one')); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledWith('one'); + + pollRequests$.next(some('two')); + + await sleep(0); + advance(pollInterval); + + expect(work).toHaveBeenCalledWith('two'); + }) + ); + + test( + 'waits for work to complete before emitting the next event', + fakeSchedulers(async advance => { + const pollInterval = 100; + + const worker = resolvable(); + + const handler = jest.fn(); + const pollRequests$ = new Subject>(); + createTaskPoller({ + pollInterval, + work: async (...args) => { + await worker; + return args; + }, + getCapacity: () => 5, + pollRequests$, + }).subscribe(handler); + + pollRequests$.next(some('one')); + + advance(pollInterval); + + // work should now be in progress + pollRequests$.next(none); + pollRequests$.next(some('two')); + pollRequests$.next(some('three')); + + advance(pollInterval); + await sleep(pollInterval); + + expect(handler).toHaveBeenCalledTimes(0); + + worker.resolve(); + + advance(pollInterval); + await sleep(pollInterval); + + expect(handler).toHaveBeenCalledWith(asOk(['one'])); + + advance(pollInterval); + + expect(handler).toHaveBeenCalledWith(asOk(['two', 'three'])); + }) + ); + + test( + 'returns an error when polling for work fails', + fakeSchedulers(async advance => { + const pollInterval = 100; + + const handler = jest.fn(); + const pollRequests$ = new Subject>(); + createTaskPoller({ + pollInterval, + work: async (...args) => { + throw new Error('failed to work'); + }, + getCapacity: () => 5, + pollRequests$, + }).subscribe(handler); + + advance(pollInterval); + await sleep(0); + + expect(handler).toHaveBeenCalledWith(asErr('Failed to poll for work: Error: failed to work')); + }) + ); }); diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index e69d0c605eb74..8b080745a6294 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -8,101 +8,89 @@ * This module contains the logic for polling the task manager index for new work. */ -import { Subject, Subscription, Observable, interval } from 'rxjs'; -import { buffer, throttle } from 'rxjs/operators'; +import { Subject, merge, partition, interval, of, Observable } from 'rxjs'; +import { mapTo, buffer, filter, mergeScan, concatMap } from 'rxjs/operators'; + +import { pipe } from 'fp-ts/lib/pipeable'; +import { Option, map as mapOptional, isSome } from 'fp-ts/lib/Option'; +import { pullFromSet } from './lib/pull_from_set'; import { Result, asOk, asErr } from './lib/result_type'; -import { Logger } from './types'; -type WorkFn = (...params: H[]) => Promise; +type WorkFn = (...params: T[]) => Promise; interface Opts { pollInterval: number; - logger: Logger; - work: WorkFn; + getCapacity: () => number; + pollRequests$: Subject>; + work: WorkFn; } /** - * Performs work on a scheduled interval, logging any errors. This waits for work to complete - * (or error) prior to attempting another run. + * constructs a new TaskPoller stream, which emits events on demand and on a scheduled interval, waiting for capacity to be available before emitting more events. + * + * @param opts + * @prop {number} pollInterval - How often, in milliseconds, we will an event be emnitted, assuming there's capacity to do so + * @prop {() => number} getCapacity - A function specifying whether there is capacity to emit new events + * @prop {Observable>} pollRequests$ - A stream of requests for polling which can provide an optional argument for the polling phase + * + * @returns {Observable>} - An observable which emits an event whenever a polling event is due to take place, providing access to a singleton Set representing a queue + * of unique request argumets of type T. The queue holds all the buffered request arguments streamed in via pollRequests$ */ -export class TaskPoller { - private logger: Logger; - private work: WorkFn; - private poller$: Observable; - private pollPhaseResults$: Subject>; - private claimRequestQueue$: Subject; - private pollingSubscription: Subscription; - - /** - * Constructs a new TaskPoller. - * - * @param opts - * @prop {number} pollInterval - How often, in milliseconds, we will run the work function - * @prop {Logger} logger - The task manager logger - * @prop {WorkFn} work - An empty, asynchronous function that performs the desired work - */ - constructor(opts: Opts) { - this.logger = opts.logger; - this.work = opts.work; - - this.pollingSubscription = Subscription.EMPTY; - this.pollPhaseResults$ = new Subject(); - this.claimRequestQueue$ = new Subject(); - this.poller$ = - // queue of requests for work to be done, each request can provide a single - // argument of type `H` - this.claimRequestQueue$.pipe( - // buffer these requests (so that multiple requests get flattened into - // a single request which receives an array of arguments of type `H`) - buffer( - // flush this buffer at the fixed interval specified under `pollInterval` - interval(opts.pollInterval).pipe( - throttle( - // only emit one interval event and then ignore every subsequent interval - // event until a corresponding completion of `attemptWork`. This prevents - // us from flushing the request buffer until the work has been completed - // the previous calls to `attemptWork` - () => this.pollPhaseResults$ - ) - ) - ) - ); - } +export function createTaskPoller({ + pollRequests$, + pollInterval, + getCapacity, + work, +}: Opts): Observable> { + const [ + // requests have arguments to be passed to polling events + requests$, + // nudge rquests try to cause a polling event early (prior to an interval expiring) + // but if there is no capacity, they are ignored + nudgeRequests$, + ] = partition(pollRequests$, req => isSome(req)); - /** - * Starts the poller. If the poller is already running, this has no effect. - */ - public async start() { - if (this.pollingSubscription && !this.pollingSubscription.closed) { - return; - } + const hasCapacity = () => getCapacity() > 0; - this.pollingSubscription = this.poller$.subscribe(requests => { - this.attemptWork(...requests); - }); - } - - /** - * Stops the poller. - */ - public stop() { - this.pollingSubscription.unsubscribe(); - } - - public queueWork(request: H) { - this.claimRequestQueue$.next(request); - } + // emit an event on a fixed interval, but only if there's capacity + const pollOnInterval$ = interval(pollInterval).pipe(filter(hasCapacity)); + return merge( + // buffer all requests, releasing them whenever an interval expires & there's capacity + requests$.pipe(buffer(pollOnInterval$)), + // emit an event when we're nudged to poll for work, as long as there's capacity + nudgeRequests$.pipe(filter(hasCapacity), mapTo([])) + ).pipe( + // buffer all requests in a single set (to remove duplicates) as we don't want + // work to take place in parallel (it could cause Task Manager to pull in the same + // task twice) + mergeScan((queue, requests) => of(pushOptionalValuesIntoSet(queue, requests)), new Set()), + // take as many argumented calls as we have capacity for and call `work` with + // those arguments. If the queue is empty this will still trigger work to be done + concatMap(async (set: Set) => { + const workArguments = pullFromSet(set, getCapacity()); + try { + const workResult = await work(...workArguments); + return asOk(workResult); + } catch (err) { + return asErr( + `Failed to poll for work${ + workArguments.length ? ` [${workArguments.join()}]` : `` + }: ${err}` + ); + } + }) + ); +} - /** - * Runs the work function, this is called in respose to the polling stream - */ - private attemptWork = async (...requests: H[]) => { - try { - const workResult = await this.work(...requests); - this.pollPhaseResults$.next(asOk(workResult)); - } catch (err) { - this.logger.error(`Failed to poll for work: ${err}`); - this.pollPhaseResults$.next(asErr(err)); - } - }; +function pushOptionalValuesIntoSet(set: Set, values: Array>): Set { + values.forEach(optionalValue => { + pipe( + optionalValue, + mapOptional(req => { + set.add(req); + return req; + }) + ); + }); + return set; } diff --git a/x-pack/legacy/plugins/task_manager/task_pool.ts b/x-pack/legacy/plugins/task_manager/task_pool.ts index 02048aea7d2b9..90f1880a159da 100644 --- a/x-pack/legacy/plugins/task_manager/task_pool.ts +++ b/x-pack/legacy/plugins/task_manager/task_pool.ts @@ -59,6 +59,13 @@ export class TaskPool { return this.maxWorkers - this.occupiedWorkers; } + /** + * Gets how many workers are currently available. + */ + public get hasAvailableWorkers() { + return this.availableWorkers > 0; + } + /** * Attempts to run the specified list of tasks. Returns true if it was able * to start every task in the list, false if there was not enough capacity diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index a5d395821ea3d..09fd55f0a6fc8 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -933,8 +933,8 @@ if (doc['task.runAt'].size()!=0) { }; const callCluster = jest.fn(); - savedObjectsClient.get.mockImplementation(async (type: string) => ({ - id, + savedObjectsClient.get.mockImplementation(async (type: string, objectId: string) => ({ + id: objectId, type, attributes: { ..._.omit(task, 'id'), @@ -984,8 +984,8 @@ if (doc['task.runAt'].size()!=0) { }; const callCluster = jest.fn(); - savedObjectsClient.get.mockImplementation(async (type: string) => ({ - id, + savedObjectsClient.get.mockImplementation(async (type: string, objectId: string) => ({ + id: objectId, type, attributes: { ..._.omit(task, 'id'), diff --git a/x-pack/package.json b/x-pack/package.json index cd023ff87a185..4d64932d4877c 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -155,6 +155,7 @@ "proxyquire": "1.8.0", "react-docgen-typescript-loader": "^3.1.1", "react-test-renderer": "^16.12.0", + "rxjs-marbles": "^5.0.3", "sass-loader": "^7.3.1", "sass-resources-loader": "^2.0.1", "simple-git": "1.116.0", diff --git a/yarn.lock b/yarn.lock index bda9f056bdf5c..5bd24ec7e9205 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3913,9 +3913,9 @@ "@types/react" "*" "@types/react@*", "@types/react@^16.8.23", "@types/react@^16.9.11", "@types/react@^16.9.13": - version "16.9.13" - resolved "https://registry.yarnpkg.com/@types/react/-/react-16.9.13.tgz#b3ea5dd443f4a680599e2abba8cc66f5e1ce0059" - integrity sha512-LikzRslbiufJYHyzbHSW0GrAiff8QYLMBFeZmSxzCYGXKxi8m/1PHX+rsVOwhr7mJNq+VIu2Dhf7U6mjFERK6w== + version "16.9.15" + resolved "https://registry.yarnpkg.com/@types/react/-/react-16.9.15.tgz#aeabb7a50f96c9e31a16079ada20ede9ed602977" + integrity sha512-WsmM1b6xQn1tG3X2Hx4F3bZwc2E82pJXt5OPs2YJgg71IzvUoKOSSSYOvLXYCg1ttipM+UuA4Lj3sfvqjVxyZw== dependencies: "@types/prop-types" "*" csstype "^2.2.0" @@ -12110,6 +12110,11 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== +fast-equals@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/fast-equals/-/fast-equals-2.0.0.tgz#bef2c423af3939f2c54310df54c57e64cd2adefc" + integrity sha512-u6RBd8cSiLLxAiC04wVsLV6GBFDOXcTCgWkd3wEoFXgidPSoAJENqC9m7Jb2vewSvjBIfXV6icKeh3GTKfIaXA== + fast-glob@^2.0.2: version "2.0.4" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.0.4.tgz#a4b9f49e36175f5ef1a3456f580226a6e7abcc9e" @@ -24726,6 +24731,13 @@ rx@^4.1.0: resolved "https://registry.yarnpkg.com/rx/-/rx-4.1.0.tgz#a5f13ff79ef3b740fe30aa803fb09f98805d4782" integrity sha1-pfE/957zt0D+MKqAP7CfmIBdR4I= +rxjs-marbles@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/rxjs-marbles/-/rxjs-marbles-5.0.3.tgz#d3ca62a4e02d032b1b4ffd558e93336ad78fd100" + integrity sha512-JK6EvLe9uReJxBmUgdKrpMB2JswV+fDcKDg97x20LErLQ7Gi0FG3YEr2Uq9hvgHJjgZXGCvonpzcxARLzKsT4A== + dependencies: + fast-equals "^2.0.0" + rxjs@^5.0.0-beta.11, rxjs@^5.5.2: version "5.5.12" resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-5.5.12.tgz#6fa61b8a77c3d793dbaf270bee2f43f652d741cc" From a4669400d5a8ca39852d6836c162923c269ec779 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Fri, 6 Dec 2019 11:43:57 +0000 Subject: [PATCH 14/45] cleaned up runNow method --- x-pack/legacy/plugins/task_manager/README.md | 23 +++++++ .../plugins/task_manager/lib/result_type.ts | 31 +++++---- x-pack/legacy/plugins/task_manager/task.ts | 5 +- .../plugins/task_manager/task_manager.ts | 65 ++++++++++--------- .../plugins/task_manager/task_runner.ts | 4 +- .../legacy/plugins/task_manager/task_store.ts | 6 +- .../plugins/task_manager/init_routes.js | 9 +-- .../task_manager/task_manager_integration.js | 4 +- 8 files changed, 91 insertions(+), 56 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index aba784b5044d2..8b85ae4264f8d 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -265,6 +265,29 @@ The danger is that in such a situation, a Task with that same `id` might already To achieve this you should use the `ensureScheduling` api which has the exact same behavior as `schedule`, except it allows the scheduling of a Task with an `id` that's already in assigned to another Task and it will assume that the existing Task is the one you wished to `schedule`, treating this as a successful operation. +### runNow +Using `eunNow` you can instruct TaskManger to run an existing task on-demand, without waiting for its scheduled time to be reached. + +```js +const taskManager = server.plugins.task_manager; + + +if(taskRunResult.error){ + try { + const taskRunResult = await taskManager.runNow('91760f10-1799-11ea-ba42-698eedde9799'); + // if no error is thrown, the task has completed successfully. + // we don't expose internal state for security reasons, but rest assured the task has completed + // if no error is thrown and a `RunNowResult` ({ id: "91760f10-1799-11ea-ba42-698eedde9799" }) + } catch(err: Error) { + // error happened, so err will have the folllowing messages + // when the task doesnt exist: `Error: failed to run task "${id}" as it does not exist` + // when the task is already running:`Error: failed to run task "${id}" as it is currently running` + } + +} +``` + + ### more options More custom access to the tasks can be done directly via Elasticsearch, though that won't be officially supported, as we can change the document structure at any time. diff --git a/x-pack/legacy/plugins/task_manager/lib/result_type.ts b/x-pack/legacy/plugins/task_manager/lib/result_type.ts index eafaff6c31bc6..df2935dd6494f 100644 --- a/x-pack/legacy/plugins/task_manager/lib/result_type.ts +++ b/x-pack/legacy/plugins/task_manager/lib/result_type.ts @@ -51,32 +51,39 @@ export function unwrap(result: Result): T | E { return isOk(result) ? result.value : result.error; } -export const either = curry(function( +export function either( + result: Result, onOk: (value: T) => void, - onErr: (error: E) => void, - result: Result + onErr: (error: E) => void ): Result { - resolve(onOk, onErr, result); + map(result, onOk, onErr); return result; -}); +} export async function eitherAsync( + result: Result, onOk: (value: T) => Promise, - onErr: (error: E) => Promise, - result: Result + onErr: (error: E) => Promise ): Promise | void> { - await resolve>(onOk, onErr, result); - return result; + return await map>(result, onOk, onErr); } -export function resolve( +export function map( + result: Result, onOk: (value: T) => Resolution, - onErr: (error: E) => Resolution, - result: Result + onErr: (error: E) => Resolution ): Resolution { return isOk(result) ? onOk(result.value) : onErr(result.error); } +export const mapR = curry(function( + onOk: (value: T) => Resolution, + onErr: (error: E) => Resolution, + result: Result +): Resolution { + return map(result, onOk, onErr); +}); + export const mapOk = curry(function( onOk: (value: T) => Result, result: Result diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index 553fa55d39d01..2bde8a9a76544 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -169,10 +169,13 @@ export enum TaskStatus { Running = 'running', Failed = 'failed', } -export enum TaskLifecycle { + +export enum TaskLifecycleResult { NotFound = 'notFound', } +export type TaskLifecycle = TaskStatus | TaskLifecycleResult; + /* * A task instance represents all of the data required to store, fetch, * and execute a task. diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index e0b8c605beccd..8d84ab9faf078 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -10,7 +10,7 @@ import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; import { Option, none, some } from 'fp-ts/lib/Option'; -import { Result, either, mapErr } from './lib/result_type'; +import { Result, asErr, either, map, mapErr, promiseResult } from './lib/result_type'; import { Logger } from './types'; import { @@ -32,6 +32,7 @@ import { TaskInstanceWithId, TaskInstance, TaskLifecycle, + TaskLifecycleResult, TaskStatus, } from './task'; import { createTaskPoller } from './task_poller'; @@ -58,7 +59,6 @@ export interface TaskManagerOpts { interface RunNowResult { id: string; - error?: string; } type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; @@ -268,51 +268,56 @@ export class TaskManager { /** * Run task. * - * @param task - The task being scheduled. + * @param taskId - The task being scheduled. * @returns {Promise} */ - public async runNow(task: string): Promise { + public async runNow(taskId: string): Promise { await this.waitUntilStarted(); - return new Promise(resolve => { + return new Promise((resolve, reject) => { const subscription = this.events$ - .pipe(filter(({ id }: TaskLifecycleEvent) => id === task)) + // listen for all events related to the current task + .pipe(filter(({ id }: TaskLifecycleEvent) => id === taskId)) .subscribe((taskEvent: TaskLifecycleEvent) => { - const { id, event } = taskEvent; either( + taskEvent.event, (taskInstance: ConcreteTaskInstance) => { + // resolve if the task has run sucessfully if (isTaskRunEvent(taskEvent)) { subscription.unsubscribe(); - resolve({ id }); + resolve({ id: taskInstance.id }); } }, async (error: Error) => { + // reject if any error event takes place for the requested task subscription.unsubscribe(); - - try { - if (isTaskClaimEvent(taskEvent)) { - const taskLifecycleStatus = await this.store.getLifecycle(id); - if (taskLifecycleStatus === TaskLifecycle.NotFound) { - resolve({ - id, - error: `Error: failed to run task "${id}" as it does not exist`, - }); - } else if (taskLifecycleStatus !== TaskStatus.Idle) { - resolve({ - id, - error: `Error: failed to run task "${id}" as it is currently running`, - }); + reject( + map( + // if the error happened in the Claim phase - we try to provide better insight + // into why we failed to claim by getting the task's current lifecycle status + isTaskClaimEvent(taskEvent) + ? await promiseResult(this.store.getLifecycle(taskId)) + : asErr(error), + (taskLifecycleStatus: TaskLifecycle) => { + if (taskLifecycleStatus === TaskLifecycleResult.NotFound) { + return new Error(`Failed to run task "${taskId}" as it does not exist`); + } else if ( + taskLifecycleStatus === TaskStatus.Running || + taskLifecycleStatus === TaskStatus.Claiming + ) { + return new Error(`Failed to run task "${taskId}" as it is currently running`); + } + }, + (err: Error) => { + this.logger.error(`Failed to get Task "${taskId}" as part of runNow: ${err}`); + return err; } - } - } catch (err) { - this.logger.error(`Failed to get Task "${id}" as part of runNow: ${err}`); - } - resolve({ id, error: `${error}` }); - }, - event + ) + ); + } ); }); - this.attemptToRun(some(task)); + this.attemptToRun(some(taskId)); }); } diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index 59dedefe8690c..4384f1cbbf5c0 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -357,6 +357,7 @@ export class TaskManagerRunner implements TaskRunner { result: Result ): Promise> { await eitherAsync( + result, async ({ runAt }: SuccessfulRunResult) => { if (runAt || this.instance.interval) { await this.processResultForRecurringTask(result); @@ -368,8 +369,7 @@ export class TaskManagerRunner implements TaskRunner { async ({ error }: FailedRunResult) => { await this.processResultForRecurringTask(result); this.onTaskEvent(asTaskRunEvent(this.id, asErr(error))); - }, - result + } ); return result; } diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 297fff7f16a54..023afb03ace5d 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -27,7 +27,7 @@ import { TaskDictionary, TaskInstance, TaskLifecycle, - TaskStatus, + TaskLifecycleResult, } from './task'; import { TaskClaim, asTaskClaimEvent } from './task_events'; @@ -366,13 +366,13 @@ export class TaskStore { * @param {string} id * @returns {Promise} */ - public async getLifecycle(id: string): Promise { + public async getLifecycle(id: string): Promise { try { const task = await this.get(id); return task.status; } catch (err) { if (err.output && err.output.statusCode === 404) { - return TaskLifecycle.NotFound; + return TaskLifecycleResult.NotFound; } throw err; } diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index 591172ef3deee..42692575e555b 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -72,14 +72,11 @@ export function initRoutes(server, taskTestingEvents) { }, }, async handler(request) { + const { task: { id } } = request.payload; try { - const { task } = request.payload; - - const taskResult = await (taskManager.runNow(task.id)); - - return taskResult; + return await (taskManager.runNow(id)); } catch (err) { - return err; + return { id, error: `${err}` }; } }, }); diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index 80240e2faf9de..9e90f54a844ee 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -322,7 +322,7 @@ export default function ({ getService }) { const failedRunNowResult = await runTaskNow({ id: 'i-dont-exist' }); - expect(failedRunNowResult).to.eql({ error: `Error: failed to run task "i-dont-exist" as it does not exist`, id: 'i-dont-exist' }); + expect(failedRunNowResult).to.eql({ error: `Error: Failed to run task "i-dont-exist" as it does not exist`, id: 'i-dont-exist' }); }); it('should return a task run error result when trying to run a task now which is already running', async () => { @@ -352,7 +352,7 @@ export default function ({ getService }) { expect( failedRunNowResult ).to.eql( - { error: `Error: failed to run task "${longRunningTask.id}" as it is currently running`, id: longRunningTask.id } + { error: `Error: Failed to run task "${longRunningTask.id}" as it is currently running`, id: longRunningTask.id } ); // finish first run by emitting 'runNowHasBeenAttempted' event From 4de72c15f6404dee153858ad8d5bab35b1db8589 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Fri, 6 Dec 2019 12:36:57 +0000 Subject: [PATCH 15/45] extracted runNow handler for improved testing --- .../plugins/task_manager/task_manager.test.ts | 154 +++++++++++++++++- .../plugins/task_manager/task_manager.ts | 102 +++++++----- 2 files changed, 210 insertions(+), 46 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts index 90f3695c5e627..982c66e68e1bb 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts @@ -6,10 +6,20 @@ import _ from 'lodash'; import sinon from 'sinon'; -import { TaskManager, claimAvailableTasks } from './task_manager'; +import { Subject } from 'rxjs'; + +import { asTaskMarkRunningEvent, asTaskRunEvent, asTaskClaimEvent } from './task_events'; +import { + TaskManager, + claimAvailableTasks, + awaitTaskRunResult, + TaskLifecycleEvent, +} from './task_manager'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { SavedObjectsSerializer, SavedObjectsSchema } from 'src/core/server'; import { mockLogger } from './test_utils'; +import { asErr, asOk } from './lib/result_type'; +import { ConcreteTaskInstance, TaskLifecycleResult, TaskStatus } from './task'; const savedObjectsClient = savedObjectsClientMock.create(); const serializer = new SavedObjectsSerializer(new SavedObjectsSchema()); @@ -273,6 +283,148 @@ describe('TaskManager', () => { ); }); + describe('runNow', () => { + describe('awaitTaskRunResult', () => { + test('resolves when the task run succeeds', () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + const task = { id } as ConcreteTaskInstance; + events$.next(asTaskRunEvent(id, asOk(task))); + + return expect(result).resolves.toEqual({ id }); + }); + + test('rejects when the task run fails', () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + const task = { id } as ConcreteTaskInstance; + events$.next(asTaskClaimEvent(id, asOk(task))); + events$.next(asTaskMarkRunningEvent(id, asOk(task))); + events$.next(asTaskRunEvent(id, asErr(new Error('some thing gone wrong')))); + + return expect(result).rejects.toEqual(new Error('some thing gone wrong')); + }); + + test('rejects when the task mark as running fails', () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + const task = { id } as ConcreteTaskInstance; + events$.next(asTaskClaimEvent(id, asOk(task))); + events$.next(asTaskMarkRunningEvent(id, asErr(new Error('some thing gone wrong')))); + + return expect(result).rejects.toEqual(new Error('some thing gone wrong')); + }); + + test('when a task claim fails we ensure the task exists', async () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(async () => TaskLifecycleResult.NotFound); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + events$.next(asTaskClaimEvent(id, asErr(new Error('failed to claim')))); + + await expect(result).rejects.toEqual( + new Error(`Failed to run task "${id}" as it does not exist`) + ); + + expect(getLifecycle).toHaveBeenCalledWith(id); + }); + + test('when a task claim fails we ensure the task isnt already claimed', async () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(async () => TaskStatus.Claiming); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + events$.next(asTaskClaimEvent(id, asErr(new Error('failed to claim')))); + + await expect(result).rejects.toEqual( + new Error(`Failed to run task "${id}" as it is currently running`) + ); + + expect(getLifecycle).toHaveBeenCalledWith(id); + }); + + test('when a task claim fails we ensure the task isnt already running', async () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(async () => TaskStatus.Running); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + events$.next(asTaskClaimEvent(id, asErr(new Error('failed to claim')))); + + await expect(result).rejects.toEqual( + new Error(`Failed to run task "${id}" as it is currently running`) + ); + + expect(getLifecycle).toHaveBeenCalledWith(id); + }); + + test('when a task claim fails we return the underlying error if the task is idle', async () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(async () => TaskStatus.Idle); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + events$.next(asTaskClaimEvent(id, asErr(new Error('failed to claim')))); + + await expect(result).rejects.toEqual(new Error('failed to claim')); + + expect(getLifecycle).toHaveBeenCalledWith(id); + }); + + test('when a task claim fails we return the underlying error if the task is failed', async () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(async () => TaskStatus.Failed); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + events$.next(asTaskClaimEvent(id, asErr(new Error('failed to claim')))); + + await expect(result).rejects.toEqual(new Error('failed to claim')); + + expect(getLifecycle).toHaveBeenCalledWith(id); + }); + + test('ignores task run success of other tasks', () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const differentTask = '4bebf429-181b-4518-bb7d-b4246d8a35f0'; + const getLifecycle = jest.fn(); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + const task = { id } as ConcreteTaskInstance; + const otherTask = { id: differentTask } as ConcreteTaskInstance; + events$.next(asTaskClaimEvent(id, asOk(task))); + events$.next(asTaskClaimEvent(differentTask, asOk(otherTask))); + + events$.next(asTaskRunEvent(differentTask, asOk(task))); + + events$.next(asTaskRunEvent(id, asErr(new Error('some thing gone wrong')))); + + return expect(result).rejects.toEqual(new Error('some thing gone wrong')); + }); + }); + }); + describe('claimAvailableTasks', () => { test('should claim Available Tasks when there are available workers', () => { const logger = mockLogger(); diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 8d84ab9faf078..8a1580fba26ed 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -10,7 +10,7 @@ import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; import { Option, none, some } from 'fp-ts/lib/Option'; -import { Result, asErr, either, map, mapErr, promiseResult } from './lib/result_type'; +import { Result, either, map, mapErr, promiseResult } from './lib/result_type'; import { Logger } from './types'; import { @@ -61,7 +61,7 @@ interface RunNowResult { id: string; } -type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; +export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; /* * The TaskManager is the public interface into the task manager system. This glues together @@ -273,49 +273,10 @@ export class TaskManager { */ public async runNow(taskId: string): Promise { await this.waitUntilStarted(); - return new Promise((resolve, reject) => { - const subscription = this.events$ - // listen for all events related to the current task - .pipe(filter(({ id }: TaskLifecycleEvent) => id === taskId)) - .subscribe((taskEvent: TaskLifecycleEvent) => { - either( - taskEvent.event, - (taskInstance: ConcreteTaskInstance) => { - // resolve if the task has run sucessfully - if (isTaskRunEvent(taskEvent)) { - subscription.unsubscribe(); - resolve({ id: taskInstance.id }); - } - }, - async (error: Error) => { - // reject if any error event takes place for the requested task - subscription.unsubscribe(); - reject( - map( - // if the error happened in the Claim phase - we try to provide better insight - // into why we failed to claim by getting the task's current lifecycle status - isTaskClaimEvent(taskEvent) - ? await promiseResult(this.store.getLifecycle(taskId)) - : asErr(error), - (taskLifecycleStatus: TaskLifecycle) => { - if (taskLifecycleStatus === TaskLifecycleResult.NotFound) { - return new Error(`Failed to run task "${taskId}" as it does not exist`); - } else if ( - taskLifecycleStatus === TaskStatus.Running || - taskLifecycleStatus === TaskStatus.Claiming - ) { - return new Error(`Failed to run task "${taskId}" as it is currently running`); - } - }, - (err: Error) => { - this.logger.error(`Failed to get Task "${taskId}" as part of runNow: ${err}`); - return err; - } - ) - ); - } - ); - }); + return new Promise(async (resolve, reject) => { + awaitTaskRunResult(taskId, this.events$, this.store.getLifecycle.bind(this.store)) + .then(resolve) + .catch(reject); this.attemptToRun(some(taskId)); }); @@ -425,3 +386,54 @@ export async function claimAvailableTasks( } return []; } + +export async function awaitTaskRunResult( + taskId: string, + events$: Subject, + getLifecycle: (id: string) => Promise +): Promise { + return new Promise((resolve, reject) => { + const subscription = events$ + // listen for all events related to the current task + .pipe(filter(({ id }: TaskLifecycleEvent) => id === taskId)) + .subscribe((taskEvent: TaskLifecycleEvent) => { + either( + taskEvent.event, + (taskInstance: ConcreteTaskInstance) => { + // resolve if the task has run sucessfully + if (isTaskRunEvent(taskEvent)) { + subscription.unsubscribe(); + resolve({ id: taskInstance.id }); + } + }, + async (error: Error) => { + // reject if any error event takes place for the requested task + subscription.unsubscribe(); + return reject( + isTaskClaimEvent(taskEvent) + ? map( + // if the error happened in the Claim phase - we try to provide better insight + // into why we failed to claim by getting the task's current lifecycle status + await promiseResult(getLifecycle(taskId)), + (taskLifecycleStatus: TaskLifecycle) => { + if (taskLifecycleStatus === TaskLifecycleResult.NotFound) { + return new Error(`Failed to run task "${taskId}" as it does not exist`); + } else if ( + taskLifecycleStatus === TaskStatus.Running || + taskLifecycleStatus === TaskStatus.Claiming + ) { + return new Error( + `Failed to run task "${taskId}" as it is currently running` + ); + } + return error; + }, + () => error + ) + : error + ); + } + ); + }); + }); +} From 92a2ad44ab0a001440ee4b1202a5211f6f5d2067 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Fri, 6 Dec 2019 13:46:25 +0000 Subject: [PATCH 16/45] fixed broken types --- x-pack/legacy/plugins/task_manager/task_store.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index 09fd55f0a6fc8..cde78cf9660c6 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -9,7 +9,13 @@ import sinon from 'sinon'; import uuid from 'uuid'; import { filter } from 'rxjs/operators'; -import { TaskDictionary, TaskDefinition, TaskInstance, TaskStatus, TaskLifecycle } from './task'; +import { + TaskDictionary, + TaskDefinition, + TaskInstance, + TaskStatus, + TaskLifecycleResult, +} from './task'; import { FetchOpts, StoreOpts, OwnershipClaimingOpts, TaskStore } from './task_store'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { SavedObjectsSerializer, SavedObjectsSchema, SavedObjectAttributes } from 'src/core/server'; @@ -1027,7 +1033,7 @@ if (doc['task.runAt'].size()!=0) { savedObjectsRepository: savedObjectsClient, }); - expect(await store.getLifecycle(id)).toEqual(TaskLifecycle.NotFound); + expect(await store.getLifecycle(id)).toEqual(TaskLifecycleResult.NotFound); }); test('throws if an unknown error takes place ', async () => { From bacbf4a2de8aeadaf2562dd11a18531b1f87fd7c Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 9 Dec 2019 12:21:30 +0000 Subject: [PATCH 17/45] improved doc and type signature --- x-pack/legacy/plugins/task_manager/task_poller.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 8b080745a6294..1460eabc271ff 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -63,7 +63,10 @@ export function createTaskPoller({ // buffer all requests in a single set (to remove duplicates) as we don't want // work to take place in parallel (it could cause Task Manager to pull in the same // task twice) - mergeScan((queue, requests) => of(pushOptionalValuesIntoSet(queue, requests)), new Set()), + mergeScan>, Set>( + (queue, requests) => of(pushOptionalValuesIntoSet(queue, requests)), + new Set() + ), // take as many argumented calls as we have capacity for and call `work` with // those arguments. If the queue is empty this will still trigger work to be done concatMap(async (set: Set) => { @@ -82,6 +85,12 @@ export function createTaskPoller({ ); } +/** + * Cycles through an array of optionals and any optional that contains a value in unwrapped and its value + * is pushed into the Set + * @param set A Set of generic type T + * @param values An array of either empty optionals or optionals contianing a generic type T + */ function pushOptionalValuesIntoSet(set: Set, values: Array>): Set { values.forEach(optionalValue => { pipe( From e794ff64d77164629c8434bd947c351a65821c40 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 9 Dec 2019 13:30:54 +0000 Subject: [PATCH 18/45] brought back perf marking in pooler --- x-pack/legacy/plugins/task_manager/index.ts | 2 +- .../legacy/plugins/task_manager/task_poller.ts | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/index.ts b/x-pack/legacy/plugins/task_manager/index.ts index 2d9183bdcd797..0ccab806a1a57 100644 --- a/x-pack/legacy/plugins/task_manager/index.ts +++ b/x-pack/legacy/plugins/task_manager/index.ts @@ -36,7 +36,7 @@ export function taskManager(kibana: any) { .default(3), poll_interval: Joi.number() .description('How often, in milliseconds, the task manager will look for more work.') - .min(1000) + .min(100) .default(3000), index: Joi.string() .description('The name of the index used to store task information.') diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 1460eabc271ff..0af39f37abe28 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -8,8 +8,10 @@ * This module contains the logic for polling the task manager index for new work. */ +import { performance } from 'perf_hooks'; +import { after } from 'lodash'; import { Subject, merge, partition, interval, of, Observable } from 'rxjs'; -import { mapTo, buffer, filter, mergeScan, concatMap } from 'rxjs/operators'; +import { mapTo, buffer, filter, mergeScan, concatMap, tap } from 'rxjs/operators'; import { pipe } from 'fp-ts/lib/pipeable'; import { Option, map as mapOptional, isSome } from 'fp-ts/lib/Option'; @@ -70,6 +72,7 @@ export function createTaskPoller({ // take as many argumented calls as we have capacity for and call `work` with // those arguments. If the queue is empty this will still trigger work to be done concatMap(async (set: Set) => { + closeSleepPerf(); const workArguments = pullFromSet(set, getCapacity()); try { const workResult = await work(...workArguments); @@ -81,10 +84,21 @@ export function createTaskPoller({ }: ${err}` ); } - }) + }), + tap(openSleepPerf) ); } +const openSleepPerf = () => { + performance.mark('TaskPoller.sleep'); +}; +// we only want to close after an open has been called, as we're counting the time *between* work cycles +// so we'll ignore the first call to `closeSleepPerf` but we will run every subsequent call +const closeSleepPerf = after(2, () => { + performance.mark('TaskPoller.poll'); + performance.measure('TaskPoller.sleepDuration', 'TaskPoller.sleep', 'TaskPoller.poll'); +}); + /** * Cycles through an array of optionals and any optional that contains a value in unwrapped and its value * is pushed into the Set From f0f4703ab807139c97ee5dd53caccf12b929ad60 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 9 Dec 2019 14:16:57 +0000 Subject: [PATCH 19/45] removed unneeded second buffer --- .../plugins/task_manager/lib/fill_pool.ts | 6 ++- .../plugins/task_manager/task_poller.ts | 39 +++++++++---------- .../task_manager/task_manager_integration.js | 5 +++ .../plugins/task_manager_performance/index.js | 4 ++ .../task_manager_perf_integration.ts | 22 ++++++++--- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts b/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts index f2dc37d3c7fdb..60470b22c00a9 100644 --- a/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts +++ b/x-pack/legacy/plugins/task_manager/lib/fill_pool.ts @@ -5,6 +5,7 @@ */ import { performance } from 'perf_hooks'; +import { after } from 'lodash'; import { TaskPoolRunResult } from '../task_pool'; export enum FillPoolResult { @@ -34,6 +35,9 @@ export async function fillPool( run: BatchRun ): Promise { performance.mark('fillPool.start'); + const markClaimedTasksOnRerunCycle = after(2, () => + performance.mark('fillPool.claimedOnRerunCycle') + ); while (true) { const instances = await fetchAvailableTasks(); @@ -46,7 +50,7 @@ export async function fillPool( ); return FillPoolResult.NoTasksClaimed; } - + markClaimedTasksOnRerunCycle(); const tasks = instances.map(converter); if ((await run(tasks)) === TaskPoolRunResult.RanOutOfCapacity) { diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 0af39f37abe28..bf48930f7be7b 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -11,10 +11,10 @@ import { performance } from 'perf_hooks'; import { after } from 'lodash'; import { Subject, merge, partition, interval, of, Observable } from 'rxjs'; -import { mapTo, buffer, filter, mergeScan, concatMap, tap } from 'rxjs/operators'; +import { mapTo, filter, mergeScan, concatMap, tap } from 'rxjs/operators'; import { pipe } from 'fp-ts/lib/pipeable'; -import { Option, map as mapOptional, isSome } from 'fp-ts/lib/Option'; +import { Option, none, map as mapOptional, isSome, getOrElse } from 'fp-ts/lib/Option'; import { pullFromSet } from './lib/pull_from_set'; import { Result, asOk, asErr } from './lib/result_type'; @@ -55,18 +55,18 @@ export function createTaskPoller({ const hasCapacity = () => getCapacity() > 0; // emit an event on a fixed interval, but only if there's capacity - const pollOnInterval$ = interval(pollInterval).pipe(filter(hasCapacity)); + const pollOnInterval$ = interval(pollInterval).pipe(mapTo(none)); return merge( // buffer all requests, releasing them whenever an interval expires & there's capacity - requests$.pipe(buffer(pollOnInterval$)), + requests$, // emit an event when we're nudged to poll for work, as long as there's capacity - nudgeRequests$.pipe(filter(hasCapacity), mapTo([])) + merge(pollOnInterval$, nudgeRequests$).pipe(filter(hasCapacity)) ).pipe( // buffer all requests in a single set (to remove duplicates) as we don't want // work to take place in parallel (it could cause Task Manager to pull in the same // task twice) - mergeScan>, Set>( - (queue, requests) => of(pushOptionalValuesIntoSet(queue, requests)), + mergeScan, Set>( + (queue, request) => of(pushOptionalIntoSet(queue, request)), new Set() ), // take as many argumented calls as we have capacity for and call `work` with @@ -100,20 +100,17 @@ const closeSleepPerf = after(2, () => { }); /** - * Cycles through an array of optionals and any optional that contains a value in unwrapped and its value - * is pushed into the Set + * Unwraps optional values and pushes them into a set * @param set A Set of generic type T - * @param values An array of either empty optionals or optionals contianing a generic type T + * @param value An optional T to push into the set if it is there */ -function pushOptionalValuesIntoSet(set: Set, values: Array>): Set { - values.forEach(optionalValue => { - pipe( - optionalValue, - mapOptional(req => { - set.add(req); - return req; - }) - ); - }); - return set; +function pushOptionalIntoSet(set: Set, value: Option): Set { + return pipe( + value, + mapOptional>(req => { + set.add(req); + return set; + }), + getOrElse(() => set) + ); } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index 9e90f54a844ee..f92a25db99b38 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -297,6 +297,11 @@ export default function ({ getService }) { }); expect(successfulRunNowResult).to.eql({ id: originalTask.id }); + await retry.try(async () => { + const [task] = (await currentTasks()).docs.filter(taskDoc => taskDoc.id === originalTask.id); + expect(task.state.count).to.eql(2); + }); + // third run should fail const failedRunNowResult = await runTaskNow({ id: originalTask.id diff --git a/x-pack/test/plugin_api_perf/plugins/task_manager_performance/index.js b/x-pack/test/plugin_api_perf/plugins/task_manager_performance/index.js index 17ae9b26fa3b3..4eb4cebe271bd 100644 --- a/x-pack/test/plugin_api_perf/plugins/task_manager_performance/index.js +++ b/x-pack/test/plugin_api_perf/plugins/task_manager_performance/index.js @@ -263,6 +263,7 @@ function resetPerfState(target) { fillPoolStarts: 0, fillPoolCycles: 0, fillPoolBail: 0, + claimedOnRerunCycle: 0, fillPoolBailNoTasks: 0, }, claimAvailableTasksNoTasks: 0, @@ -320,6 +321,9 @@ function resetPerfState(target) { case 'fillPool.bailExhaustedCapacity': performanceState.performance.cycles.fillPoolBail++; break; + case 'fillPool.claimedOnRerunCycle': + performanceState.performance.cycles.claimedOnRerunCycle++; + break; case 'fillPool.bailNoTasks': performanceState.performance.cycles.fillPoolBail++; performanceState.performance.cycles.fillPoolBailNoTasks++; diff --git a/x-pack/test/plugin_api_perf/test_suites/task_manager/task_manager_perf_integration.ts b/x-pack/test/plugin_api_perf/test_suites/task_manager/task_manager_perf_integration.ts index d5caff5f9d10b..fed3a59ab5a0c 100644 --- a/x-pack/test/plugin_api_perf/test_suites/task_manager/task_manager_perf_integration.ts +++ b/x-pack/test/plugin_api_perf/test_suites/task_manager/task_manager_perf_integration.ts @@ -17,7 +17,13 @@ export default function({ getService }: { getService: (service: string) => any } runningAverageTasksPerSecond, runningAverageLeadTime, // how often things happen in Task Manager - cycles: { fillPoolStarts, fillPoolCycles, fillPoolBail, fillPoolBailNoTasks }, + cycles: { + fillPoolStarts, + fillPoolCycles, + claimedOnRerunCycle, + fillPoolBail, + fillPoolBailNoTasks, + }, claimAvailableTasksNoTasks, claimAvailableTasksNoAvailableWorkers, numberOfTasksRanOverall, @@ -70,9 +76,11 @@ export default function({ getService }: { getService: (service: string) => any } )}]---> next markAsRunning` ); log.info(`Duration of Perf Test: ${bright(perfTestDuration)}`); - log.info(`Activity within Task Poller: ${bright(activityDuration)}`); + log.info(`Activity within Task Pool: ${bright(activityDuration)}`); log.info(`Inactivity due to Sleep: ${bright(sleepDuration)}`); - log.info(`Polling Cycles: ${colorizeCycles(fillPoolStarts, fillPoolCycles, fillPoolBail)}`); + log.info( + `Polling Cycles: ${colorizeCycles(fillPoolStarts, fillPoolCycles, claimedOnRerunCycle)}` + ); if (fillPoolBail > 0) { log.info(` ⮑ Bailed due to:`); if (fillPoolBailNoTasks > 0) { @@ -127,7 +135,11 @@ function colorize(avg: number) { return avg < 500 ? green(`${avg}`) : avg < 1000 ? cyan(`${avg}`) : red(`${avg}`); } -function colorizeCycles(fillPoolStarts: number, fillPoolCycles: number, fillPoolBail: number) { +function colorizeCycles( + fillPoolStarts: number, + fillPoolCycles: number, + claimedOnRerunCycle: number +) { const perc = (fillPoolCycles * 100) / fillPoolStarts; const colorFunc = perc >= 100 ? green : perc >= 50 ? cyan : red; return ( @@ -135,7 +147,7 @@ function colorizeCycles(fillPoolStarts: number, fillPoolCycles: number, fillPool bright(`${fillPoolStarts}`) + ` cycles, of which ` + colorFunc(`${fillPoolCycles}`) + - ` were reran before bailing` + ` were reran (of which ${claimedOnRerunCycle} resulted in claiming) before bailing` ); } From f40378103c3d12916fb2cde7f4b1635fab92a967 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 11:38:29 +0000 Subject: [PATCH 20/45] throw error when a requests comes in but the request buffer is at capacity --- x-pack/legacy/plugins/task_manager/index.ts | 5 + .../plugins/task_manager/task_events.ts | 19 ++- .../plugins/task_manager/task_manager.test.ts | 24 +++- .../plugins/task_manager/task_manager.ts | 76 ++++++---- .../plugins/task_manager/task_poller.test.ts | 78 ++++++++++- .../plugins/task_manager/task_poller.ts | 131 +++++++++++------- 6 files changed, 251 insertions(+), 82 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/index.ts b/x-pack/legacy/plugins/task_manager/index.ts index 0ccab806a1a57..6f2bc3704bb67 100644 --- a/x-pack/legacy/plugins/task_manager/index.ts +++ b/x-pack/legacy/plugins/task_manager/index.ts @@ -38,6 +38,11 @@ export function taskManager(kibana: any) { .description('How often, in milliseconds, the task manager will look for more work.') .min(100) .default(3000), + request_capacity: Joi.number() + .description('How many requests can Task Manager buffer before it rejects new requests.') + .min(1) + // a nice round contrived number, feel free to change as we learn how it behaves + .default(1000), index: Joi.string() .description('The name of the index used to store task information.') .default('.kibana_task_manager') diff --git a/x-pack/legacy/plugins/task_manager/task_events.ts b/x-pack/legacy/plugins/task_manager/task_events.ts index d02adaea411fc..063ac2499471f 100644 --- a/x-pack/legacy/plugins/task_manager/task_events.ts +++ b/x-pack/legacy/plugins/task_manager/task_events.ts @@ -6,12 +6,13 @@ import { ConcreteTaskInstance } from './task'; -import { Result } from './lib/result_type'; +import { Result, Err } from './lib/result_type'; export enum TaskEventType { TASK_CLAIM = 'TASK_CLAIM', TASK_MARK_RUNNING = 'TASK_MARK_RUNNING', TASK_RUN = 'TASK_RUN', + TASK_RUN_REQUEST = 'TASK_RUN_REQUEST', } export interface TaskEvent { @@ -22,6 +23,7 @@ export interface TaskEvent { export type TaskMarkRunning = TaskEvent; export type TaskRun = TaskEvent; export type TaskClaim = TaskEvent; +export type TaskRunRequest = TaskEvent; export function asTaskMarkRunningEvent( id: string, @@ -53,6 +55,18 @@ export function asTaskClaimEvent( }; } +export function asTaskRunRequestEvent( + id: string, + // we only emit a TaskRunRequest event when it fails + event: Err +): TaskRunRequest { + return { + id, + type: TaskEventType.TASK_RUN_REQUEST, + event, + }; +} + export function isTaskMarkRunningEvent( taskEvent: TaskEvent ): taskEvent is TaskMarkRunning { @@ -64,3 +78,6 @@ export function isTaskRunEvent(taskEvent: TaskEvent): taskEvent is Tas export function isTaskClaimEvent(taskEvent: TaskEvent): taskEvent is TaskClaim { return taskEvent.type === TaskEventType.TASK_CLAIM; } +export function isTaskRunRequestEvent(taskEvent: TaskEvent): taskEvent is TaskRunRequest { + return taskEvent.type === TaskEventType.TASK_RUN_REQUEST; +} diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts index 982c66e68e1bb..521d4a0f323ac 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts @@ -8,7 +8,12 @@ import _ from 'lodash'; import sinon from 'sinon'; import { Subject } from 'rxjs'; -import { asTaskMarkRunningEvent, asTaskRunEvent, asTaskClaimEvent } from './task_events'; +import { + asTaskMarkRunningEvent, + asTaskRunEvent, + asTaskClaimEvent, + asTaskRunRequestEvent, +} from './task_events'; import { TaskManager, claimAvailableTasks, @@ -375,6 +380,23 @@ describe('TaskManager', () => { expect(getLifecycle).toHaveBeenCalledWith(id); }); + test('rejects when the task run fails due to capacity', async () => { + const events$ = new Subject(); + const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; + const getLifecycle = jest.fn(async () => TaskStatus.Idle); + + const result = awaitTaskRunResult(id, events$, getLifecycle); + + events$.next(asTaskRunRequestEvent(id, asErr(new Error('failed to buffer request')))); + + await expect(result).rejects.toEqual( + new Error( + `Failed to run task "${id}" as Task Manager is at capacity, please try again later` + ) + ); + expect(getLifecycle).not.toHaveBeenCalled(); + }); + test('when a task claim fails we return the underlying error if the task is idle', async () => { const events$ = new Subject(); const id = '01ddff11-e88a-4d13-bc4e-256164e755e2'; diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 8a1580fba26ed..637052cb40317 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -9,16 +9,20 @@ import { filter } from 'rxjs/operators'; import { performance } from 'perf_hooks'; import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server'; -import { Option, none, some } from 'fp-ts/lib/Option'; -import { Result, either, map, mapErr, promiseResult } from './lib/result_type'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { Option, none, some, map as mapOptional } from 'fp-ts/lib/Option'; +import { Result, asErr, either, map, mapErr, promiseResult } from './lib/result_type'; import { Logger } from './types'; import { TaskMarkRunning, TaskRun, TaskClaim, + TaskRunRequest, isTaskRunEvent, isTaskClaimEvent, + isTaskRunRequestEvent, + asTaskRunRequestEvent, } from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { addMiddlewareToChain, BeforeSaveMiddlewareParams, Middleware } from './lib/middleware'; @@ -35,7 +39,7 @@ import { TaskLifecycleResult, TaskStatus, } from './task'; -import { createTaskPoller } from './task_poller'; +import { createTaskPoller, PollingError, PollingErrorType } from './task_poller'; import { TaskPool } from './task_pool'; import { TaskManagerRunner, TaskRunner } from './task_runner'; import { @@ -61,7 +65,7 @@ interface RunNowResult { id: string; } -export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim; +export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim | TaskRunRequest; /* * The TaskManager is the public interface into the task manager system. This glues together @@ -85,7 +89,7 @@ export class TaskManager { private events$: Subject; private claimRequests$: Subject>; private pollingSubscription: Subscription; - private poller$: Observable>; + private poller$: Observable>>; private startQueue: Array<() => void> = []; private middleware = { @@ -137,6 +141,7 @@ export class TaskManager { this.pollingSubscription = Subscription.EMPTY; this.poller$ = createTaskPoller({ pollInterval: opts.config.get('xpack.task_manager.poll_interval'), + bufferCapacity: opts.config.get('xpack.task_manager.request_capacity'), getCapacity: () => this.pool.availableWorkers, pollRequests$: this.claimRequests$, work: this.pollForWork, @@ -194,7 +199,15 @@ export class TaskManager { this.startQueue = []; this.pollingSubscription = this.poller$.subscribe( - mapErr((error: string) => this.logger.error(error)) + mapErr((error: PollingError) => { + if (error.type === PollingErrorType.RequestCapacityReached) { + pipe( + error.data, + mapOptional(id => this.emitEvent(asTaskRunRequestEvent(id, asErr(error)))) + ); + } + this.logger.error(error.message); + }) ); } } @@ -409,29 +422,34 @@ export async function awaitTaskRunResult( async (error: Error) => { // reject if any error event takes place for the requested task subscription.unsubscribe(); - return reject( - isTaskClaimEvent(taskEvent) - ? map( - // if the error happened in the Claim phase - we try to provide better insight - // into why we failed to claim by getting the task's current lifecycle status - await promiseResult(getLifecycle(taskId)), - (taskLifecycleStatus: TaskLifecycle) => { - if (taskLifecycleStatus === TaskLifecycleResult.NotFound) { - return new Error(`Failed to run task "${taskId}" as it does not exist`); - } else if ( - taskLifecycleStatus === TaskStatus.Running || - taskLifecycleStatus === TaskStatus.Claiming - ) { - return new Error( - `Failed to run task "${taskId}" as it is currently running` - ); - } - return error; - }, - () => error - ) - : error - ); + if (isTaskRunRequestEvent(taskEvent)) { + return reject( + new Error( + `Failed to run task "${taskId}" as Task Manager is at capacity, please try again later` + ) + ); + } else if (isTaskClaimEvent(taskEvent)) { + reject( + map( + // if the error happened in the Claim phase - we try to provide better insight + // into why we failed to claim by getting the task's current lifecycle status + await promiseResult(getLifecycle(taskId)), + (taskLifecycleStatus: TaskLifecycle) => { + if (taskLifecycleStatus === TaskLifecycleResult.NotFound) { + return new Error(`Failed to run task "${taskId}" as it does not exist`); + } else if ( + taskLifecycleStatus === TaskStatus.Running || + taskLifecycleStatus === TaskStatus.Claiming + ) { + return new Error(`Failed to run task "${taskId}" as it is currently running`); + } + return error; + }, + () => error + ) + ); + } + return reject(error); } ); }); diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts index dbc51fcd6516b..077a11d0e1ac1 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts @@ -7,7 +7,7 @@ import _ from 'lodash'; import { Subject } from 'rxjs'; import { Option, none, some } from 'fp-ts/lib/Option'; -import { createTaskPoller } from './task_poller'; +import { createTaskPoller, PollingError, PollingErrorType } from './task_poller'; import { fakeSchedulers } from 'rxjs-marbles/jest'; import { sleep, resolvable } from './test_utils'; import { asOk, asErr } from './lib/result_type'; @@ -19,11 +19,13 @@ describe('TaskPoller', () => { 'intializes the poller with the provided interval', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 5; const halfInterval = Math.floor(pollInterval / 2); const work = jest.fn(async () => true); createTaskPoller({ pollInterval, + bufferCapacity, getCapacity: () => 1, work, pollRequests$: new Subject>(), @@ -48,12 +50,14 @@ describe('TaskPoller', () => { 'filters interval polling on capacity', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 2; const work = jest.fn(async () => true); let hasCapacity = true; createTaskPoller({ pollInterval, + bufferCapacity, work, getCapacity: () => (hasCapacity ? 1 : 0), pollRequests$: new Subject>(), @@ -102,6 +106,7 @@ describe('TaskPoller', () => { 'requests with no arguments (nudge requests) are queued on-demand in between intervals', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 2; const querterInterval = Math.floor(pollInterval / 4); const halfInterval = querterInterval * 2; @@ -109,6 +114,7 @@ describe('TaskPoller', () => { const pollRequests$ = new Subject>(); createTaskPoller({ pollInterval, + bufferCapacity, work, getCapacity: () => 1, pollRequests$, @@ -143,6 +149,7 @@ describe('TaskPoller', () => { 'requests with no arguments (nudge requests) are dropped when there is no capacity', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 2; const querterInterval = Math.floor(pollInterval / 4); const halfInterval = querterInterval * 2; @@ -151,6 +158,7 @@ describe('TaskPoller', () => { const pollRequests$ = new Subject>(); createTaskPoller({ pollInterval, + bufferCapacity, work, getCapacity: () => (hasCapacity ? 1 : 0), pollRequests$, @@ -187,11 +195,13 @@ describe('TaskPoller', () => { 'requests with arguments are emitted', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 2; const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ pollInterval, + bufferCapacity, work, getCapacity: () => 1, pollRequests$, @@ -218,6 +228,7 @@ describe('TaskPoller', () => { 'waits for work to complete before emitting the next event', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 2; const worker = resolvable(); @@ -225,6 +236,7 @@ describe('TaskPoller', () => { const pollRequests$ = new Subject>(); createTaskPoller({ pollInterval, + bufferCapacity, work: async (...args) => { await worker; return args; @@ -264,11 +276,13 @@ describe('TaskPoller', () => { 'returns an error when polling for work fails', fakeSchedulers(async advance => { const pollInterval = 100; + const bufferCapacity = 2; const handler = jest.fn(); const pollRequests$ = new Subject>(); createTaskPoller({ pollInterval, + bufferCapacity, work: async (...args) => { throw new Error('failed to work'); }, @@ -279,7 +293,67 @@ describe('TaskPoller', () => { advance(pollInterval); await sleep(0); - expect(handler).toHaveBeenCalledWith(asErr('Failed to poll for work: Error: failed to work')); + const expectedError = new PollingError( + 'Failed to poll for work: Error: failed to work', + PollingErrorType.WorkError, + none + ); + expect(handler).toHaveBeenCalledWith(asErr(expectedError)); + expect(handler.mock.calls[0][0].error.type).toEqual(PollingErrorType.WorkError); + }) + ); + + test( + 'returns a request capcity error when new request is emitted but the poller is at buffer capacity', + fakeSchedulers(async advance => { + const pollInterval = 1000; + const bufferCapacity = 2; + + const handler = jest.fn(); + const work = jest.fn(async () => {}); + const pollRequests$ = new Subject>(); + createTaskPoller({ + pollInterval, + bufferCapacity, + work, + getCapacity: () => 5, + pollRequests$, + }).subscribe(handler); + + // advance(pollInterval); + + pollRequests$.next(some('one')); + + await sleep(0); + advance(pollInterval); + + expect(work).toHaveBeenCalledWith('one'); + + pollRequests$.next(some('two')); + pollRequests$.next(some('three')); + // three consecutive should cause us to go above capacity + pollRequests$.next(some('four')); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledWith('two', 'three'); + + pollRequests$.next(some('five')); + pollRequests$.next(some('six')); + + await sleep(0); + advance(pollInterval); + expect(work).toHaveBeenCalledWith('five', 'six'); + + expect(handler).toHaveBeenCalledWith( + asErr( + new PollingError( + 'Failed to poll for work: request capacity reached', + PollingErrorType.RequestCapacityReached, + some('four') + ) + ) + ); }) ); }); diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index bf48930f7be7b..ba70b28d78153 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -10,18 +10,19 @@ import { performance } from 'perf_hooks'; import { after } from 'lodash'; -import { Subject, merge, partition, interval, of, Observable } from 'rxjs'; -import { mapTo, filter, mergeScan, concatMap, tap } from 'rxjs/operators'; +import { Subject, merge, interval, of, Observable } from 'rxjs'; +import { mapTo, filter, scan, concatMap, tap, catchError } from 'rxjs/operators'; import { pipe } from 'fp-ts/lib/pipeable'; -import { Option, none, map as mapOptional, isSome, getOrElse } from 'fp-ts/lib/Option'; +import { Option, none, map as mapOptional, getOrElse } from 'fp-ts/lib/Option'; import { pullFromSet } from './lib/pull_from_set'; -import { Result, asOk, asErr } from './lib/result_type'; +import { Result, Err, map as mapResult, asOk, asErr } from './lib/result_type'; type WorkFn = (...params: T[]) => Promise; interface Opts { pollInterval: number; + bufferCapacity: number; getCapacity: () => number; pollRequests$: Subject>; work: WorkFn; @@ -40,77 +41,109 @@ interface Opts { */ export function createTaskPoller({ pollRequests$, + bufferCapacity, pollInterval, getCapacity, work, -}: Opts): Observable> { - const [ - // requests have arguments to be passed to polling events - requests$, - // nudge rquests try to cause a polling event early (prior to an interval expiring) - // but if there is no capacity, they are ignored - nudgeRequests$, - ] = partition(pollRequests$, req => isSome(req)); - +}: Opts): Observable>> { const hasCapacity = () => getCapacity() > 0; - // emit an event on a fixed interval, but only if there's capacity - const pollOnInterval$ = interval(pollInterval).pipe(mapTo(none)); - return merge( - // buffer all requests, releasing them whenever an interval expires & there's capacity - requests$, - // emit an event when we're nudged to poll for work, as long as there's capacity - merge(pollOnInterval$, nudgeRequests$).pipe(filter(hasCapacity)) + const errors$ = new Subject>>(); + + const requestWorkProcessing$ = merge( + // emit a polling event on demand + pollRequests$, + // emit a polling event on a fixed interval + interval(pollInterval).pipe(mapTo(none)) ).pipe( // buffer all requests in a single set (to remove duplicates) as we don't want // work to take place in parallel (it could cause Task Manager to pull in the same // task twice) - mergeScan, Set>( - (queue, request) => of(pushOptionalIntoSet(queue, request)), + scan, Set>( + (queue, request) => + mapResult( + pushOptionalIntoSet(queue, bufferCapacity, request), + // value has been successfully pushed into buffer + () => queue, + // value wasnt pushed into buffer, we must be at capacity + () => { + errors$.next( + asPollingError( + `request capacity reached`, + PollingErrorType.RequestCapacityReached, + request + ) + ); + return queue; + } + ), new Set() ), + // only emit polling events when there's capacity to handle them + filter(hasCapacity), // take as many argumented calls as we have capacity for and call `work` with // those arguments. If the queue is empty this will still trigger work to be done concatMap(async (set: Set) => { closeSleepPerf(); - const workArguments = pullFromSet(set, getCapacity()); - try { - const workResult = await work(...workArguments); - return asOk(workResult); - } catch (err) { - return asErr( - `Failed to poll for work${ - workArguments.length ? ` [${workArguments.join()}]` : `` - }: ${err}` - ); - } + return asOk(await work(...pullFromSet(set, getCapacity()))); }), - tap(openSleepPerf) + tap(openSleepPerf), + // catch + catchError((err: Error) => of(asPollingError(err, PollingErrorType.WorkError))) ); -} - -const openSleepPerf = () => { - performance.mark('TaskPoller.sleep'); -}; -// we only want to close after an open has been called, as we're counting the time *between* work cycles -// so we'll ignore the first call to `closeSleepPerf` but we will run every subsequent call -const closeSleepPerf = after(2, () => { - performance.mark('TaskPoller.poll'); - performance.measure('TaskPoller.sleepDuration', 'TaskPoller.sleep', 'TaskPoller.poll'); -}); + return merge(requestWorkProcessing$, errors$); +} /** * Unwraps optional values and pushes them into a set * @param set A Set of generic type T + * @param maxCapacity How many values are we allowed to push into the set * @param value An optional T to push into the set if it is there */ -function pushOptionalIntoSet(set: Set, value: Option): Set { +function pushOptionalIntoSet( + set: Set, + maxCapacity: number, + value: Option +): Result, Set> { return pipe( value, - mapOptional>(req => { + mapOptional, Set>>(req => { + if (set.size >= maxCapacity) { + return asErr(set); + } set.add(req); - return set; + return asOk(set); }), - getOrElse(() => set) + getOrElse(() => asOk(set) as Result, Set>) ); } + +export enum PollingErrorType { + WorkError, + RequestCapacityReached, +} + +function asPollingError(err: string | Error, type: PollingErrorType, data: Option = none) { + return asErr(new PollingError(`Failed to poll for work: ${err}`, type, data)); +} + +export class PollingError extends Error { + public readonly type: PollingErrorType; + public readonly data: Option; + constructor(message: string, type: PollingErrorType, data: Option) { + super(message); + Object.setPrototypeOf(this, new.target.prototype); + this.type = type; + this.data = data; + } +} + +const openSleepPerf = () => { + performance.mark('TaskPoller.sleep'); +}; +// we only want to close after an open has been called, as we're counting the time *between* work cycles +// so we'll ignore the first call to `closeSleepPerf` but we will run every subsequent call +const closeSleepPerf = after(2, () => { + performance.mark('TaskPoller.poll'); + performance.measure('TaskPoller.sleepDuration', 'TaskPoller.sleep', 'TaskPoller.poll'); +}); From 56baa6dc50b8613cdd3749c4b1a9c815b56dfc78 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 15:34:06 +0000 Subject: [PATCH 21/45] fixed types and typo --- .../plugins/alerting/server/alerts_client.test.ts | 10 +++++----- x-pack/legacy/plugins/task_manager/README.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts index 714edc22e0971..37eb6a9b21d44 100644 --- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts +++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts @@ -120,7 +120,7 @@ describe('create()', () => { taskType: 'alerting:123', scheduledAt: new Date(), attempts: 1, - status: 'idle' as TaskStatus, + status: TaskStatus.Idle, runAt: new Date(), startedAt: null, retryAt: null, @@ -352,7 +352,7 @@ describe('create()', () => { taskType: 'alerting:123', scheduledAt: new Date(), attempts: 1, - status: 'idle', + status: TaskStatus.Idle, runAt: new Date(), startedAt: null, retryAt: null, @@ -750,7 +750,7 @@ describe('create()', () => { taskType: 'alerting:123', scheduledAt: new Date(), attempts: 1, - status: 'idle' as TaskStatus, + status: TaskStatus.Idle, runAt: new Date(), startedAt: null, retryAt: null, @@ -831,7 +831,7 @@ describe('enable()', () => { id: 'task-123', scheduledAt: new Date(), attempts: 0, - status: 'idle' as TaskStatus, + status: TaskStatus.Idle, runAt: new Date(), state: {}, params: {}, @@ -908,7 +908,7 @@ describe('enable()', () => { id: 'task-123', scheduledAt: new Date(), attempts: 0, - status: 'idle' as TaskStatus, + status: TaskStatus.Idle, runAt: new Date(), state: {}, params: {}, diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index 8b85ae4264f8d..da9acb3bbd207 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -266,7 +266,7 @@ The danger is that in such a situation, a Task with that same `id` might already To achieve this you should use the `ensureScheduling` api which has the exact same behavior as `schedule`, except it allows the scheduling of a Task with an `id` that's already in assigned to another Task and it will assume that the existing Task is the one you wished to `schedule`, treating this as a successful operation. ### runNow -Using `eunNow` you can instruct TaskManger to run an existing task on-demand, without waiting for its scheduled time to be reached. +Using `runNow` you can instruct TaskManger to run an existing task on-demand, without waiting for its scheduled time to be reached. ```js const taskManager = server.plugins.task_manager; From 73043bce567c49b18a2142ad8b1b2c76903cc851 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 15:40:51 +0000 Subject: [PATCH 22/45] fixed TaskStatus usage --- .../plugins/actions/server/lib/task_runner_factory.test.ts | 2 +- .../plugins/alerting/server/lib/task_runner_factory.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/actions/server/lib/task_runner_factory.test.ts b/x-pack/legacy/plugins/actions/server/lib/task_runner_factory.test.ts index 41a7c17a02c5a..eb183f1f1d06a 100644 --- a/x-pack/legacy/plugins/actions/server/lib/task_runner_factory.test.ts +++ b/x-pack/legacy/plugins/actions/server/lib/task_runner_factory.test.ts @@ -34,7 +34,7 @@ beforeAll(() => { state: {}, attempts: 0, ownerId: '', - status: 'running' as TaskStatus, + status: TaskStatus.Running, startedAt: new Date(), scheduledAt: new Date(), retryAt: new Date(Date.now() + 5 * 60 * 1000), diff --git a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts index 3e1a23e37b448..c21c419977bbe 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts @@ -30,7 +30,7 @@ beforeAll(() => { mockedTaskInstance = { id: '', attempts: 0, - status: 'running' as TaskStatus, + status: TaskStatus.Running, version: '123', runAt: new Date(), scheduledAt: new Date(), From 3a4f5886704eb7eb4478feb5db589b8db2584dd5 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 15:43:54 +0000 Subject: [PATCH 23/45] corrected doc --- x-pack/legacy/plugins/task_manager/README.md | 22 ++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index da9acb3bbd207..f22b6a92c5189 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -271,19 +271,15 @@ Using `runNow` you can instruct TaskManger to run an existing task on-demand, wi ```js const taskManager = server.plugins.task_manager; - -if(taskRunResult.error){ - try { - const taskRunResult = await taskManager.runNow('91760f10-1799-11ea-ba42-698eedde9799'); - // if no error is thrown, the task has completed successfully. - // we don't expose internal state for security reasons, but rest assured the task has completed - // if no error is thrown and a `RunNowResult` ({ id: "91760f10-1799-11ea-ba42-698eedde9799" }) - } catch(err: Error) { - // error happened, so err will have the folllowing messages - // when the task doesnt exist: `Error: failed to run task "${id}" as it does not exist` - // when the task is already running:`Error: failed to run task "${id}" as it is currently running` - } - +try { + const taskRunResult = await taskManager.runNow('91760f10-1799-11ea-ba42-698eedde9799'); + // if no error is thrown, the task has completed successfully. + // we don't expose internal state for security reasons, but rest assured the task has completed + // if no error is thrown and a `RunNowResult` ({ id: "91760f10-1799-11ea-ba42-698eedde9799" }) +} catch(err: Error) { + // error happened, so err will have the folllowing messages + // when the task doesnt exist: `Error: failed to run task "${id}" as it does not exist` + // when the task is already running:`Error: failed to run task "${id}" as it is currently running` } ``` From 55c2dfc3b37ed5ad5dc13d52aa5c096494609a2f Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 15:55:09 +0000 Subject: [PATCH 24/45] cleaned up TM constructor --- x-pack/legacy/plugins/task_manager/task_manager.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 637052cb40317..6eaf5e28ef2cf 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -118,6 +118,9 @@ export class TaskManager { this.logger.info(`TaskManager is identified by the Kibana UUID: ${taskManagerId}`); } + // all task related events (task claimed, task marked as running, etc.) are emitted through events$ + this.events$ = new Subject(); + this.store = new TaskStore({ serializer: opts.serializer, savedObjectsRepository: opts.savedObjectsRepository, @@ -127,17 +130,15 @@ export class TaskManager { definitions: this.definitions, taskManagerId: `kibana:${taskManagerId}`, }); + // pipe store events into the TaskManager's event stream + this.store.events.subscribe(event => this.events$.next(event)); this.pool = new TaskPool({ logger: this.logger, maxWorkers: this.maxWorkers, }); - this.events$ = new Subject(); this.claimRequests$ = new Subject(); - // if we end up with only one stream, remove merge - merge(this.store.events).subscribe(event => this.events$.next(event)); - this.pollingSubscription = Subscription.EMPTY; this.poller$ = createTaskPoller({ pollInterval: opts.config.get('xpack.task_manager.poll_interval'), From 951341901025d36f410cbb89af6975cf4add358d Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 16:03:44 +0000 Subject: [PATCH 25/45] improved documentation --- x-pack/legacy/plugins/task_manager/task_poller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index ba70b28d78153..9e28d85b45104 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -88,7 +88,7 @@ export function createTaskPoller({ return asOk(await work(...pullFromSet(set, getCapacity()))); }), tap(openSleepPerf), - // catch + // catch errors during polling for work catchError((err: Error) => of(asPollingError(err, PollingErrorType.WorkError))) ); From 4266303dedd63b1ac2a6c89a0e70ddc9a9d549cb Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 10 Dec 2019 16:27:31 +0000 Subject: [PATCH 26/45] fixed edgecase where execution could fail i nthe poller --- .../plugins/task_manager/task_manager.ts | 2 +- .../plugins/task_manager/task_poller.test.ts | 48 +++++++++++++++++++ .../plugins/task_manager/task_poller.ts | 10 +++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 6eaf5e28ef2cf..c128a8e7be233 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Subject, Observable, Subscription, merge } from 'rxjs'; +import { Subject, Observable, Subscription } from 'rxjs'; import { filter } from 'rxjs/operators'; import { performance } from 'perf_hooks'; diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts index 077a11d0e1ac1..c99e741fb7aa1 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts @@ -303,6 +303,54 @@ describe('TaskPoller', () => { }) ); + test( + 'continues polling after work fails', + fakeSchedulers(async advance => { + const pollInterval = 100; + const bufferCapacity = 2; + + const handler = jest.fn(); + const pollRequests$ = new Subject>(); + let callCount = 0; + const work = jest.fn(async () => { + callCount++; + if (callCount === 2) { + throw new Error('failed to work'); + } + return callCount; + }); + createTaskPoller({ + pollInterval, + bufferCapacity, + work, + getCapacity: () => 5, + pollRequests$, + }).subscribe(handler); + + advance(pollInterval); + await sleep(0); + + expect(handler).toHaveBeenCalledWith(asOk(1)); + + advance(pollInterval); + await sleep(0); + + const expectedError = new PollingError( + 'Failed to poll for work: Error: failed to work', + PollingErrorType.WorkError, + none + ); + expect(handler).toHaveBeenCalledWith(asErr(expectedError)); + expect(handler.mock.calls[1][0].error.type).toEqual(PollingErrorType.WorkError); + expect(handler).not.toHaveBeenCalledWith(asOk(2)); + + advance(pollInterval); + await sleep(0); + + expect(handler).toHaveBeenCalledWith(asOk(3)); + }) + ); + test( 'returns a request capcity error when new request is emitted but the poller is at buffer capacity', fakeSchedulers(async advance => { diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 9e28d85b45104..d3e493e70d98f 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -16,7 +16,7 @@ import { mapTo, filter, scan, concatMap, tap, catchError } from 'rxjs/operators' import { pipe } from 'fp-ts/lib/pipeable'; import { Option, none, map as mapOptional, getOrElse } from 'fp-ts/lib/Option'; import { pullFromSet } from './lib/pull_from_set'; -import { Result, Err, map as mapResult, asOk, asErr } from './lib/result_type'; +import { Result, Err, map as mapResult, asOk, asErr, promiseResult } from './lib/result_type'; type WorkFn = (...params: T[]) => Promise; @@ -85,7 +85,13 @@ export function createTaskPoller({ // those arguments. If the queue is empty this will still trigger work to be done concatMap(async (set: Set) => { closeSleepPerf(); - return asOk(await work(...pullFromSet(set, getCapacity()))); + return mapResult>>( + await promiseResult(work(...pullFromSet(set, getCapacity()))), + workResult => asOk(workResult), + (err: Error) => { + return asPollingError(err, PollingErrorType.WorkError); + } + ); }), tap(openSleepPerf), // catch errors during polling for work From d42e0403fbff9f0aeec544fda49233959c7b685a Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 11 Dec 2019 10:52:13 +0000 Subject: [PATCH 27/45] moved intialisations in TaskManager --- x-pack/legacy/plugins/task_manager/README.md | 39 +++++++++---------- .../plugins/task_manager/task_manager.ts | 22 +++++------ 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index f22b6a92c5189..9ca873e77fe68 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -272,14 +272,12 @@ Using `runNow` you can instruct TaskManger to run an existing task on-demand, wi const taskManager = server.plugins.task_manager; try { - const taskRunResult = await taskManager.runNow('91760f10-1799-11ea-ba42-698eedde9799'); - // if no error is thrown, the task has completed successfully. - // we don't expose internal state for security reasons, but rest assured the task has completed - // if no error is thrown and a `RunNowResult` ({ id: "91760f10-1799-11ea-ba42-698eedde9799" }) + const taskRunResult = await taskManager.runNow('91760f10-ba42-de9799'); + // If no error is thrown, the task has completed successfully. } catch(err: Error) { - // error happened, so err will have the folllowing messages - // when the task doesnt exist: `Error: failed to run task "${id}" as it does not exist` - // when the task is already running:`Error: failed to run task "${id}" as it is currently running` + // If running the task has failed, we throw an error with an appropriate message. + // For example, if the requested task doesnt exist: `Error: failed to run task "91760f10-ba42-de9799" as it does not exist` + // Or if, for example, the task is already running: `Error: failed to run task "91760f10-ba42-de9799" as it is currently running` } ``` @@ -327,31 +325,32 @@ server.plugins.task_manager.addMiddleware({ ## Task Poller: polling for work TaskManager used to work in a `pull` model, but it now needs to support both `push` and `pull`, so it has been remodeled internally to support a single `push` model. -Task Manager's lifecycle is pushed by the following operations: +Task Manager's _push_ mechanism is driven by the following operations: 1. A polling interval has been reached. 2. A new Task is scheduled. 3. A Task is run using `runNow`. -The polling interval straight forward: TaskPoller is configured to emit an event at a fixed interval. -We wish to ignore any polling interval that goes off when there are no workers available, so we'll throttle that on workerAvailability +The polling interval is straight forward: TaskPoller is configured to emit an event at a fixed interval. +That said, if there are no workers available, we want to ignore these events, so we'll throttle the interval on worker availability. -Every time a Task is scheduled we want to trigger an early polling in order to respond to the newly scheduled task asap, but this too we only wish to do if there are available workers, so we can throttle this too. +Whenever a user uses the `schedule` api to schedule a new Task, we want to trigger an early polling in order to respond to the newly scheduled task as soon as possible, but this too we only wish to do if there are available workers, so we can throttle this too. -When a runNow call is made we need to force a poll as the user will now be waiting on the result of the runNow call, but -there is a complexity here- we don't want to force polling as there might not be any worker capacity, but we also can't throttle, as we can't afford to "drop" these requests (as we are bypassing normal scheduling), so we'll have to buffer these. +When a `runNow` call is made we need to force a poll as the user will now be waiting on the result of the `runNow` call, but +there is a complexity here- we don't want to force polling (as there might not be any worker capacity and it's possible that a polling cycle is already running), but we also can't throttle, as we can't afford to "drop" these requests, so we'll have to buffer these. -We now want to respond to all three of these push events, but we still need to balance against our worker capacity, so if there are too many requests buffered, we only want to `take` as many requests as we have capacity top handle. -Luckily, `Polling Interval` and `Task Scheduled` simply denote a request to "poll for work as soon as possible", unlike `Run Task Now` which also means "poll for these specific tasks", so our capacity only needs to be applied to `Run Task Now`. +We now want to respond to all three of these push events, but we still need to balance against our worker capacity, so if there are too many requests buffered, we only want to `take` as many requests as we have capacity to handle. +Luckily, `Polling Interval` and `Task Scheduled` simply denote a request to "poll for work as soon as possible", unlike `Run Task Now` which also means "poll for these specific tasks", so our worker capacity only needs to be applied to `Run Task Now`. -We achieve this model by maintaining a queue using a Set (which removes duplicated). -TODO: We don't want an unbounded queue, sobest to add a configurable cap and return an error to the `runNow` call when this cap is reached. +We achieve this model by buffering requests into a queue using a Set (which removes duplicated). As we don't want an unbounded queue in our system, we have limited the size of this queue (configurable by the `xpack.task_manager.request_capacity` config, defaulting to 1,000 requests) which forces us to throw an error once this cap is reachedand to all subsequent calls to `runNow` until the queue drain bellow the cap. Our current model, then, is this: ``` - Polling Interval --> filter(workerAvailability > 0) -- [] --\ - Task Scheduled --> filter(workerAvailability > 0) -- [] ---|==> Set([] + [] + [`ID`]) ==> work([`ID`]) - Run Task `ID` Now --> buffer(workerAvailability > 0) -- [`ID`] --/ + Polling Interval --> filter(availableWorkers > 0) - mapTo([]) -------\\ + Task Scheduled --> filter(availableWorkers > 0) - mapTo([]) --------||==>Set([]+[]+[`1`,`2`]) ==> work([`1`,`2`]) + Run Task `1` Now --\ // + ----> buffer(availableWorkers > 0) -- [`1`,`2`] -// + Run Task `2` Now --/ ``` ## Limitations in v1.0 diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index c128a8e7be233..5e26910c7fdfe 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -81,15 +81,18 @@ export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim | TaskRun * The public interface into the task manager system. */ export class TaskManager { - private maxWorkers: number; - private definitions: TaskDictionary; + private definitions: TaskDictionary = {}; private store: TaskStore; private logger: Logger; private pool: TaskPool; - private events$: Subject; - private claimRequests$: Subject>; - private pollingSubscription: Subscription; + // all task related events (task claimed, task marked as running, etc.) are emitted through events$ + private events$ = new Subject(); + // all on-demand requests we wish to pipe into the poller + private claimRequests$ = new Subject>(); + // the task poller that polls for work on fixed intervals and on demand private poller$: Observable>>; + // our subscription to the poller + private pollingSubscription: Subscription = Subscription.EMPTY; private startQueue: Array<() => void> = []; private middleware = { @@ -104,8 +107,6 @@ export class TaskManager { * mechanism. */ constructor(opts: TaskManagerOpts) { - this.maxWorkers = opts.config.get('xpack.task_manager.max_workers'); - this.definitions = {}; this.logger = opts.logger; const taskManagerId = opts.config.get('server.uuid'); @@ -118,9 +119,6 @@ export class TaskManager { this.logger.info(`TaskManager is identified by the Kibana UUID: ${taskManagerId}`); } - // all task related events (task claimed, task marked as running, etc.) are emitted through events$ - this.events$ = new Subject(); - this.store = new TaskStore({ serializer: opts.serializer, savedObjectsRepository: opts.savedObjectsRepository, @@ -135,11 +133,9 @@ export class TaskManager { this.pool = new TaskPool({ logger: this.logger, - maxWorkers: this.maxWorkers, + maxWorkers: opts.config.get('xpack.task_manager.max_workers'), }); - this.claimRequests$ = new Subject(); - this.pollingSubscription = Subscription.EMPTY; this.poller$ = createTaskPoller({ pollInterval: opts.config.get('xpack.task_manager.poll_interval'), bufferCapacity: opts.config.get('xpack.task_manager.request_capacity'), From dc6c2993ed07361b032d89ad14eb927e2f952d4d Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 11 Dec 2019 10:56:00 +0000 Subject: [PATCH 28/45] removed unneccesery null guard --- x-pack/legacy/plugins/task_manager/task_manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 5e26910c7fdfe..bcd44a1575a63 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -166,7 +166,7 @@ export class TaskManager { }; public get isStarted() { - return this.pollingSubscription && !this.pollingSubscription.closed; + return !this.pollingSubscription.closed; } private pollForWork = async (...tasksToClaim: string[]): Promise => { From f5fc2e3a41adbf2f1fb4cb9073e3a972b888ee06 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 11 Dec 2019 11:35:12 +0000 Subject: [PATCH 29/45] renames interval to recurringSchedule --- x-pack/legacy/plugins/task_manager/README.md | 4 +-- .../legacy/plugins/task_manager/mappings.json | 2 +- .../legacy/plugins/task_manager/migrations.ts | 11 ++++++ .../mark_available_tasks_as_claimed.test.ts | 10 +++--- .../mark_available_tasks_as_claimed.ts | 4 ++- x-pack/legacy/plugins/task_manager/task.ts | 8 +++-- .../plugins/task_manager/task_manager.test.ts | 2 +- .../plugins/task_manager/task_runner.test.ts | 34 +++++++++--------- .../plugins/task_manager/task_runner.ts | 17 +++++---- .../plugins/task_manager/task_store.test.ts | 36 +++++++++---------- .../legacy/plugins/task_manager/task_store.ts | 6 ++-- .../plugins/task_manager/init_routes.js | 4 +-- .../task_manager/task_manager_integration.js | 14 ++++---- 13 files changed, 83 insertions(+), 69 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index f22b6a92c5189..6282bb9f5fdb0 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -164,7 +164,7 @@ The data stored for a task instance looks something like this: // Indicates that this is a recurring task. We currently only support // minute syntax `5m` or second syntax `10s`. - interval: '5m', + recurringSchedule: '5m', // How many times this task has been unsuccesfully attempted, // this will be reset to 0 if the task ever succesfully completes. @@ -233,7 +233,7 @@ const taskManager = server.plugins.task_manager; const task = await taskManager.schedule({ taskType, runAt, - interval, + recurringSchedule, params, scope: ['my-fanci-app'], }); diff --git a/x-pack/legacy/plugins/task_manager/mappings.json b/x-pack/legacy/plugins/task_manager/mappings.json index 96653a4de1b31..71684201b4574 100644 --- a/x-pack/legacy/plugins/task_manager/mappings.json +++ b/x-pack/legacy/plugins/task_manager/mappings.json @@ -16,7 +16,7 @@ "retryAt": { "type": "date" }, - "interval": { + "recurringSchedule": { "type": "text" }, "attempts": { diff --git a/x-pack/legacy/plugins/task_manager/migrations.ts b/x-pack/legacy/plugins/task_manager/migrations.ts index dd6651fddb90a..4a4c5db88bbce 100644 --- a/x-pack/legacy/plugins/task_manager/migrations.ts +++ b/x-pack/legacy/plugins/task_manager/migrations.ts @@ -12,5 +12,16 @@ export const migrations = { ...doc, updated_at: new Date().toISOString(), }), + '7.6.0': renameAttribute('interval', 'recurringSchedule'), }, }; + +function renameAttribute(oldName: string, newName: string) { + return ({ attributes: { [oldName]: value, ...attributes }, ...doc }: SavedObject) => ({ + ...doc, + attributes: { + ...attributes, + [newName]: value, + }, + }); +} diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts index b8ea7d2db832e..ef490685d895a 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts @@ -18,7 +18,7 @@ import { updateFields, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - RecuringTaskWithInterval, + TaskWithRecurringSchedule, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, } from './mark_available_tasks_as_claimed'; @@ -50,9 +50,9 @@ describe('mark_available_tasks_as_claimed', () => { // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an interval or the attempts < the maximum configured + // Either task has an recurringSchedule or the attempts < the maximum configured shouldBeOneOf( - RecuringTaskWithInterval, + TaskWithRecurringSchedule, ...Object.entries(definitions).map(([type, { maxAttempts }]) => taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) ) @@ -100,11 +100,11 @@ describe('mark_available_tasks_as_claimed', () => { ], }, }, - // Either task has an interval or the attempts < the maximum configured + // Either task has an recurring schedule or the attempts < the maximum configured { bool: { should: [ - { exists: { field: 'task.interval' } }, + { exists: { field: 'task.recurringSchedule' } }, { bool: { must: [ diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index 1da4550998885..4848bafda4439 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -14,7 +14,9 @@ import { RangeBoolClause, } from './query_clauses'; -export const RecuringTaskWithInterval: ExistsBoolClause = { exists: { field: 'task.interval' } }; +export const TaskWithRecurringSchedule: ExistsBoolClause = { + exists: { field: 'task.recurringSchedule' }, +}; export function taskWithLessThanMaxAttempts( type: string, maxAttempts: number diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index 2bde8a9a76544..c8f9b0480a8d0 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -133,7 +133,7 @@ export interface TaskDefinition { * function can return `true`, `false` or a Date. True will tell task manager * to retry using default delay logic. False will tell task manager to stop retrying * this task. Date will suggest when to the task manager the task should retry. - * This function isn't used for interval type tasks, those retry at the next interval. + * This function isn't used for recurring tasks, those retry as per their configured recurring schedule. */ getRetry?: (attempts: number, error: object) => boolean | Date; @@ -220,9 +220,11 @@ export interface TaskInstance { runAt?: Date; /** - * An interval in minutes (e.g. '5m'). If specified, this is a recurring task. + * A TaskSchedule string, which specifies this as a recurring task. + * + * Currently, this supports a single format: an interval in minutes or seconds (e.g. '5m', '30s'). */ - interval?: string; + recurringSchedule?: string; /** * A task-specific set of parameters, used by the task's run function to tailor diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts index 521d4a0f323ac..7029e218f0214 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts @@ -487,7 +487,7 @@ describe('TaskManager', () => { conflicts: 'proceed', }, body: - '{"query":{"bool":{"must":[{"term":{"type":"task"}},{"bool":{"must":[{"bool":{"should":[{"bool":{"must":[{"term":{"task.status":"idle"}},{"range":{"task.runAt":{"lte":"now"}}}]}},{"bool":{"must":[{"bool":{"should":[{"term":{"task.status":"running"}},{"term":{"task.status":"claiming"}}]}},{"range":{"task.retryAt":{"lte":"now"}}}]}}]}},{"bool":{"should":[{"exists":{"field":"task.interval"}},{"bool":{"must":[{"term":{"task.taskType":"vis_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"lens_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.server-log"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.slack"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.email"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.index"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.pagerduty"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.webhook"}},{"range":{"task.attempts":{"lt":1}}}]}}]}}]}}]}},"sort":{"_script":{"type":"number","order":"asc","script":{"lang":"expression","source":"doc[\'task.retryAt\'].value || doc[\'task.runAt\'].value"}}},"seq_no_primary_term":true,"script":{"source":"ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;","lang":"painless","params":{"ownerId":"kibana:5b2de169-2785-441b-ae8c-186a1936b17d","retryAt":"2019-10-31T13:35:43.579Z","status":"claiming"}}}', + '{"query":{"bool":{"must":[{"term":{"type":"task"}},{"bool":{"must":[{"bool":{"should":[{"bool":{"must":[{"term":{"task.status":"idle"}},{"range":{"task.runAt":{"lte":"now"}}}]}},{"bool":{"must":[{"bool":{"should":[{"term":{"task.status":"running"}},{"term":{"task.status":"claiming"}}]}},{"range":{"task.retryAt":{"lte":"now"}}}]}}]}},{"bool":{"should":[{"exists":{"field":"task.recurringSchedule"}},{"bool":{"must":[{"term":{"task.taskType":"vis_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"lens_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.server-log"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.slack"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.email"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.index"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.pagerduty"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.webhook"}},{"range":{"task.attempts":{"lt":1}}}]}}]}}]}}]}},"sort":{"_script":{"type":"number","order":"asc","script":{"lang":"expression","source":"doc[\'task.retryAt\'].value || doc[\'task.runAt\'].value"}}},"seq_no_primary_term":true,"script":{"source":"ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;","lang":"painless","params":{"ownerId":"kibana:5b2de169-2785-441b-ae8c-186a1936b17d","retryAt":"2019-10-31T13:35:43.579Z","status":"claiming"}}}', statusCode: 400, response: '{"error":{"root_cause":[{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts"}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":".kibana_task_manager_1","node":"24A4QbjHSK6prvtopAKLKw","reason":{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts"}}],"caused_by":{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts","caused_by":{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts"}}},"status":400}', diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts index c9fc010ca7912..dd918e339afa4 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts @@ -86,10 +86,10 @@ describe('TaskManagerRunner', () => { expect(instance.state).toEqual({ hey: 'there' }); }); - test('reschedules tasks that have an interval', async () => { + test('reschedules tasks that have an recurringSchedule', async () => { const { runner, store } = testOpts({ instance: { - interval: '10m', + recurringSchedule: '10m', status: TaskStatus.Running, startedAt: new Date(), }, @@ -133,11 +133,11 @@ describe('TaskManagerRunner', () => { sinon.assert.calledWithMatch(store.update, { runAt }); }); - test('tasks that return runAt override interval', async () => { + test('tasks that return runAt override the recurringSchedule', async () => { const runAt = minutesFromNow(_.random(5)); const { runner, store } = testOpts({ instance: { - interval: '20m', + recurringSchedule: '20m', }, definitions: { bar: { @@ -161,7 +161,7 @@ describe('TaskManagerRunner', () => { const { runner, store } = testOpts({ instance: { id, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -237,7 +237,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -366,7 +366,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: '1m', + recurringSchedule: '1m', startedAt: new Date(), }, definitions: { @@ -402,7 +402,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -436,7 +436,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -468,7 +468,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -503,7 +503,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -538,7 +538,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -570,7 +570,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: '1m', + recurringSchedule: '1m', startedAt: new Date(), }, definitions: { @@ -601,7 +601,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: undefined, + recurringSchedule: undefined, }, definitions: { bar: { @@ -633,7 +633,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - interval: `${intervalSeconds}s`, + recurringSchedule: `${intervalSeconds}s`, startedAt: new Date(), }, definitions: { @@ -749,7 +749,7 @@ describe('TaskManagerRunner', () => { onTaskEvent, instance: { id, - interval: '1m', + recurringSchedule: '1m', }, definitions: { bar: { @@ -800,7 +800,7 @@ describe('TaskManagerRunner', () => { onTaskEvent, instance: { id, - interval: '1m', + recurringSchedule: '1m', startedAt: new Date(), }, definitions: { diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index 4384f1cbbf5c0..dbcfb6d7da783 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -205,7 +205,7 @@ export class TaskManagerRunner implements TaskRunner { status: TaskStatus.Running, startedAt: now, attempts, - retryAt: this.instance.interval + retryAt: this.instance.recurringSchedule ? intervalFromNow(this.definition.timeout)! : this.getRetryDelay({ attempts, @@ -273,7 +273,7 @@ export class TaskManagerRunner implements TaskRunner { } private shouldTryToScheduleRetry(): boolean { - if (this.instance.interval) { + if (this.instance.recurringSchedule) { return true; } @@ -287,8 +287,8 @@ export class TaskManagerRunner implements TaskRunner { if (this.shouldTryToScheduleRetry()) { const { runAt, state, error } = failureResult; // if we're retrying, keep the number of attempts - const { interval, attempts } = this.instance; - if (runAt || interval) { + const { recurringSchedule, attempts } = this.instance; + if (runAt || recurringSchedule) { return asOk({ state, attempts, runAt }); } else { // when result.error is truthy, then we're retrying because it failed @@ -312,12 +312,11 @@ export class TaskManagerRunner implements TaskRunner { const fieldUpdates = flow( // if running the task has failed ,try to correct by scheduling a retry in the near future mapErr(this.rescheduleFailedRun), - // if retrying is possible (new runAt) or this is simply an interval - // based task - reschedule + // if retrying is possible (new runAt) or this is an recurring task - reschedule mapOk(({ runAt, state, attempts = 0 }: Partial) => { - const { startedAt, interval } = this.instance; + const { startedAt, recurringSchedule } = this.instance; return asOk({ - runAt: runAt || intervalFromDate(startedAt!, interval)!, + runAt: runAt || intervalFromDate(startedAt!, recurringSchedule)!, state, attempts, status: TaskStatus.Idle, @@ -359,7 +358,7 @@ export class TaskManagerRunner implements TaskRunner { await eitherAsync( result, async ({ runAt }: SuccessfulRunResult) => { - if (runAt || this.instance.interval) { + if (runAt || this.instance.recurringSchedule) { await this.processResultForRecurringTask(result); } else { await this.processResultWhenDone(); diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index cde78cf9660c6..ec2da665eca35 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -98,7 +98,7 @@ describe('TaskStore', () => { 'task', { attempts: 0, - interval: undefined, + recurringSchedule: undefined, params: '{"hello":"world"}', retryAt: null, runAt: '2019-02-12T21:01:22.479Z', @@ -119,7 +119,7 @@ describe('TaskStore', () => { expect(result).toEqual({ id: 'testid', attempts: 0, - interval: undefined, + recurringSchedule: undefined, params: { hello: 'world' }, retryAt: null, runAt: mockedDate, @@ -271,7 +271,7 @@ describe('TaskStore', () => { task: { runAt, taskType: 'foo', - interval: undefined, + recurringSchedule: undefined, attempts: 0, status: 'idle', params: '{ "hello": "world" }', @@ -289,7 +289,7 @@ describe('TaskStore', () => { task: { runAt, taskType: 'bar', - interval: '5m', + recurringSchedule: '5m', attempts: 2, status: 'running', params: '{ "shazm": 1 }', @@ -307,7 +307,7 @@ describe('TaskStore', () => { { attempts: 0, id: 'aaa', - interval: undefined, + recurringSchedule: undefined, params: { hello: 'world' }, runAt, scheduledAt: mockedDate, @@ -322,7 +322,7 @@ describe('TaskStore', () => { { attempts: 2, id: 'bbb', - interval: '5m', + recurringSchedule: '5m', params: { shazm: 1 }, runAt, scheduledAt: mockedDate, @@ -476,7 +476,7 @@ describe('TaskStore', () => { { bool: { should: [ - { exists: { field: 'task.interval' } }, + { exists: { field: 'task.recurringSchedule' } }, { bool: { must: [ @@ -594,7 +594,7 @@ describe('TaskStore', () => { { bool: { should: [ - { exists: { field: 'task.interval' } }, + { exists: { field: 'task.recurringSchedule' } }, { bool: { must: [ @@ -729,7 +729,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'foo', - interval: undefined, + recurringSchedule: undefined, attempts: 0, status: 'idle', params: '{ "hello": "world" }', @@ -750,7 +750,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'bar', - interval: '5m', + recurringSchedule: '5m', attempts: 2, status: 'running', params: '{ "shazm": 1 }', @@ -800,7 +800,7 @@ if (doc['task.runAt'].size()!=0) { { attempts: 0, id: 'aaa', - interval: undefined, + recurringSchedule: undefined, params: { hello: 'world' }, runAt, scope: ['reporting'], @@ -813,7 +813,7 @@ if (doc['task.runAt'].size()!=0) { { attempts: 2, id: 'bbb', - interval: '5m', + recurringSchedule: '5m', params: { shazm: 1 }, runAt, scope: ['reporting', 'ceo'], @@ -873,7 +873,7 @@ if (doc['task.runAt'].size()!=0) { task.id, { attempts: task.attempts, - interval: undefined, + recurringSchedule: undefined, params: JSON.stringify(task.params), retryAt: null, runAt: task.runAt.toISOString(), @@ -891,7 +891,7 @@ if (doc['task.runAt'].size()!=0) { expect(result).toEqual({ ...task, - interval: undefined, + recurringSchedule: undefined, retryAt: null, scope: undefined, startedAt: null, @@ -1069,7 +1069,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'foo', - interval: undefined, + recurringSchedule: undefined, attempts: 0, status: 'idle', params: '{ "hello": "world" }', @@ -1093,7 +1093,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'bar', - interval: '5m', + recurringSchedule: '5m', attempts: 2, status: 'running', params: '{ "shazm": 1 }', @@ -1146,7 +1146,7 @@ if (doc['task.runAt'].size()!=0) { id: 'aaa', runAt, taskType: 'foo', - interval: undefined, + recurringSchedule: undefined, attempts: 0, status: 'idle' as TaskStatus, params: { hello: 'world' }, @@ -1203,7 +1203,7 @@ if (doc['task.runAt'].size()!=0) { id: 'bbb', runAt, taskType: 'bar', - interval: '5m', + recurringSchedule: '5m', attempts: 2, status: 'running' as TaskStatus, params: { shazm: 1 }, diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 023afb03ace5d..a88af5d715b40 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -47,7 +47,7 @@ import { updateFields, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - RecuringTaskWithInterval, + TaskWithRecurringSchedule, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, taskWithIDsAndRunnableStatus, @@ -246,9 +246,9 @@ export class TaskStore { // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an interval or the attempts < the maximum configured + // Either task has a recurringSchedule or the attempts < the maximum configured shouldBeOneOf( - RecuringTaskWithInterval, + TaskWithRecurringSchedule, ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) ) diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index 42692575e555b..492771367c7b4 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -34,7 +34,7 @@ export function initRoutes(server, taskTestingEvents) { payload: Joi.object({ task: Joi.object({ taskType: Joi.string().required(), - interval: Joi.string().optional(), + recurringSchedule: Joi.string().optional(), params: Joi.object().required(), state: Joi.object().optional(), id: Joi.string().optional() @@ -89,7 +89,7 @@ export function initRoutes(server, taskTestingEvents) { payload: Joi.object({ task: Joi.object({ taskType: Joi.string().required(), - interval: Joi.string().optional(), + recurringSchedule: Joi.string().optional(), params: Joi.object().required(), state: Joi.object().optional(), id: Joi.string().optional() diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index f92a25db99b38..ae25840e29b59 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -110,7 +110,7 @@ export default function ({ getService }) { const scheduledTask = await scheduleTask({ taskType: 'sampleTask', - interval: '30m', + recurringSchedule: '30m', params: { historyItem }, }); log.debug(`Task created: ${scheduledTask.id}`); @@ -215,7 +215,7 @@ export default function ({ getService }) { const originalTask = await scheduleTask({ taskType: 'sampleTask', - interval: `${interval}m`, + recurringSchedule: `${interval}m`, params: { }, }); @@ -234,7 +234,7 @@ export default function ({ getService }) { const originalTask = await scheduleTask({ taskType: 'sampleTask', - interval: `30m`, + recurringSchedule: `30m`, params: { }, }); @@ -274,7 +274,7 @@ export default function ({ getService }) { it('should return a task run error result when running a task now fails', async () => { const originalTask = await scheduleTask({ taskType: 'sampleTask', - interval: `30m`, + recurringSchedule: `30m`, params: { failWith: 'error on run now', failOn: 3 }, }); @@ -333,7 +333,7 @@ export default function ({ getService }) { it('should return a task run error result when trying to run a task now which is already running', async () => { const longRunningTask = await scheduleTask({ taskType: 'sampleTask', - interval: '30m', + recurringSchedule: '30m', params: { waitForParams: true }, @@ -428,13 +428,13 @@ export default function ({ getService }) { */ const fastTask = await scheduleTask({ taskType: 'sampleTask', - interval: `1s`, + recurringSchedule: `1s`, params: { }, }); const longRunningTask = await scheduleTask({ taskType: 'sampleTask', - interval: `1s`, + recurringSchedule: `1s`, params: { waitForEvent: 'rescheduleHasHappened' }, From bd0cb9639d212925900f42776c114e5304cc0d30 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 11 Dec 2019 12:27:45 +0000 Subject: [PATCH 30/45] added support for interval as a deprecated field --- .../lib/correct_deprecated_fields.test.ts | 59 +++++++++++++++++++ .../lib/correct_deprecated_fields.ts | 18 ++++++ x-pack/legacy/plugins/task_manager/task.ts | 12 ++++ .../plugins/task_manager/task_manager.ts | 10 +++- .../plugins/task_manager/init_routes.js | 1 + .../task_manager/task_manager_integration.js | 21 +++++++ 6 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts create mode 100644 x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts diff --git a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts new file mode 100644 index 0000000000000..6029c75950103 --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ensureDeprecatedFieldsAreCorrected } from './correct_deprecated_fields'; + +describe('ensureDeprecatedFieldsAreCorrected', () => { + test('doesnt change tasks without any schedule fields', async () => { + expect( + ensureDeprecatedFieldsAreCorrected({ + id: 'my-foo-id', + taskType: 'foo', + params: {}, + state: {}, + }) + ).toEqual({ + id: 'my-foo-id', + taskType: 'foo', + params: {}, + state: {}, + }); + }); + test('doesnt change tasks with the recurringSchedule field', async () => { + expect( + ensureDeprecatedFieldsAreCorrected({ + id: 'my-foo-id', + taskType: 'foo', + recurringSchedule: '10m', + params: {}, + state: {}, + }) + ).toEqual({ + id: 'my-foo-id', + taskType: 'foo', + recurringSchedule: '10m', + params: {}, + state: {}, + }); + }); + test('corrects tasks with the deprecated inteval field', async () => { + expect( + ensureDeprecatedFieldsAreCorrected({ + id: 'my-foo-id', + taskType: 'foo', + interval: '10m', + params: {}, + state: {}, + }) + ).toEqual({ + id: 'my-foo-id', + taskType: 'foo', + recurringSchedule: '10m', + params: {}, + state: {}, + }); + }); +}); diff --git a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts new file mode 100644 index 0000000000000..a013e0cc26460 --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TaskInstance, TaskInstanceWithDeprecatedFields } from '../task'; + +export function ensureDeprecatedFieldsAreCorrected({ + interval, + recurringSchedule, + ...taskInstance +}: TaskInstanceWithDeprecatedFields): TaskInstance { + return { + ...taskInstance, + recurringSchedule: recurringSchedule || interval, + }; +} diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index c8f9b0480a8d0..1b9992a3111cf 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -256,6 +256,18 @@ export interface TaskInstance { ownerId?: string | null; } +/** + * Support for the depracated interval field, this should be removed in version 8.0.0 + * and marked as a breaking change, ideally nutil then all usage of `interval` will be + * replaced with use of `recurringSchedule` + */ +export interface TaskInstanceWithDeprecatedFields extends TaskInstance { + /** + * An interval in minutes (e.g. '5m'). If specified, this is a recurring task. + * */ + interval?: string; +} + /** * A task instance that has an id. */ diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index c128a8e7be233..39ce4c416c72a 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -34,7 +34,7 @@ import { ConcreteTaskInstance, RunContext, TaskInstanceWithId, - TaskInstance, + TaskInstanceWithDeprecatedFields, TaskLifecycle, TaskLifecycleResult, TaskStatus, @@ -50,6 +50,7 @@ import { ClaimOwnershipResult, } from './task_store'; import { identifyEsError } from './lib/identify_es_error'; +import { ensureDeprecatedFieldsAreCorrected } from './lib/correct_deprecated_fields'; const VERSION_CONFLICT_STATUS = 409; @@ -268,11 +269,14 @@ export class TaskManager { * @param task - The task being scheduled. * @returns {Promise} */ - public async schedule(taskInstance: TaskInstance, options?: any): Promise { + public async schedule( + taskInstance: TaskInstanceWithDeprecatedFields, + options?: any + ): Promise { await this.waitUntilStarted(); const { taskInstance: modifiedTask } = await this.middleware.beforeSave({ ...options, - taskInstance, + taskInstance: ensureDeprecatedFieldsAreCorrected(taskInstance), }); const result = await this.store.schedule(modifiedTask); this.attemptToRun(); diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index 492771367c7b4..a667fcb7b3175 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -35,6 +35,7 @@ export function initRoutes(server, taskTestingEvents) { task: Joi.object({ taskType: Joi.string().required(), recurringSchedule: Joi.string().optional(), + interval: Joi.string().optional(), params: Joi.object().required(), state: Joi.object().optional(), id: Joi.string().optional() diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index ae25840e29b59..fb560273b36b7 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -230,6 +230,27 @@ export default function ({ getService }) { }); }); + it('should support the deprecated interval field', async () => { + const interval = _.random(5, 200); + const intervalMilliseconds = interval * 60000; + + const originalTask = await scheduleTask({ + taskType: 'sampleTask', + interval: `${interval}m`, + params: { }, + }); + + await retry.try(async () => { + expect((await historyDocs()).length).to.eql(1); + + const [task] = (await currentTasks()).docs; + expect(task.attempts).to.eql(0); + expect(task.state.count).to.eql(1); + + expectReschedule(Date.parse(originalTask.runAt), task, intervalMilliseconds); + }); + }); + it('should return a task run result when asked to run a task now', async () => { const originalTask = await scheduleTask({ From 1a09f3c909242f6e49e12f1c93465a77e0ffa394 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Wed, 11 Dec 2019 12:58:55 +0000 Subject: [PATCH 31/45] added missing tests for pull from set --- .../task_manager/lib/pull_from_set.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 x-pack/legacy/plugins/task_manager/lib/pull_from_set.test.ts diff --git a/x-pack/legacy/plugins/task_manager/lib/pull_from_set.test.ts b/x-pack/legacy/plugins/task_manager/lib/pull_from_set.test.ts new file mode 100644 index 0000000000000..ffc752751550d --- /dev/null +++ b/x-pack/legacy/plugins/task_manager/lib/pull_from_set.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pullFromSet } from './pull_from_set'; + +describe(`pullFromSet`, () => { + test(`doesnt pull from an empty set`, () => { + expect(pullFromSet(new Set(), 10)).toEqual([]); + }); + + test(`doesnt pull when there is no capacity`, () => { + expect(pullFromSet(new Set([1, 2, 3]), 0)).toEqual([]); + }); + + test(`pulls as many values as there are in the set`, () => { + expect(pullFromSet(new Set([1, 2, 3]), 3)).toEqual([1, 2, 3]); + }); + + test(`pulls as many values as there are in the set up to capacity`, () => { + expect(pullFromSet(new Set([1, 2, 3]), 2)).toEqual([1, 2]); + }); + + test(`modifies the orginal set`, () => { + const set = new Set([1, 2, 3]); + expect(pullFromSet(set, 2)).toEqual([1, 2]); + expect(set).toEqual(new Set([3])); + }); +}); From f36dbc5089113a9345cfd883496ef751b7086211 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 09:46:35 +0000 Subject: [PATCH 32/45] moved interval under schedule key --- x-pack/legacy/plugins/task_manager/README.md | 6 ++-- .../lib/correct_deprecated_fields.test.ts | 8 ++--- .../lib/correct_deprecated_fields.ts | 4 +-- .../legacy/plugins/task_manager/mappings.json | 8 +++-- .../legacy/plugins/task_manager/migrations.ts | 19 +++++++--- .../mark_available_tasks_as_claimed.test.ts | 4 +-- .../mark_available_tasks_as_claimed.ts | 2 +- x-pack/legacy/plugins/task_manager/task.ts | 11 ++++-- .../plugins/task_manager/task_manager.test.ts | 2 +- .../plugins/task_manager/task_runner.test.ts | 34 +++++++++--------- .../plugins/task_manager/task_runner.ts | 14 ++++---- .../plugins/task_manager/task_store.test.ts | 36 +++++++++---------- .../legacy/plugins/task_manager/task_store.ts | 2 +- .../plugins/task_manager/init_routes.js | 6 ++-- .../task_manager/task_manager_integration.js | 14 ++++---- 15 files changed, 96 insertions(+), 74 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md index 6282bb9f5fdb0..cac5ce56135a2 100644 --- a/x-pack/legacy/plugins/task_manager/README.md +++ b/x-pack/legacy/plugins/task_manager/README.md @@ -163,8 +163,8 @@ The data stored for a task instance looks something like this: runAt: "2020-07-24T17:34:35.272Z", // Indicates that this is a recurring task. We currently only support - // minute syntax `5m` or second syntax `10s`. - recurringSchedule: '5m', + // interval syntax with either minutes such as `5m` or seconds `10s`. + schedule: { interval: '5m' }, // How many times this task has been unsuccesfully attempted, // this will be reset to 0 if the task ever succesfully completes. @@ -233,7 +233,7 @@ const taskManager = server.plugins.task_manager; const task = await taskManager.schedule({ taskType, runAt, - recurringSchedule, + schedule, params, scope: ['my-fanci-app'], }); diff --git a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts index 6029c75950103..c712b5c8ca2cc 100644 --- a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts +++ b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts @@ -22,19 +22,19 @@ describe('ensureDeprecatedFieldsAreCorrected', () => { state: {}, }); }); - test('doesnt change tasks with the recurringSchedule field', async () => { + test('doesnt change tasks with the schedule field', async () => { expect( ensureDeprecatedFieldsAreCorrected({ id: 'my-foo-id', taskType: 'foo', - recurringSchedule: '10m', + schedule: { interval: '10m' }, params: {}, state: {}, }) ).toEqual({ id: 'my-foo-id', taskType: 'foo', - recurringSchedule: '10m', + schedule: { interval: '10m' }, params: {}, state: {}, }); @@ -51,7 +51,7 @@ describe('ensureDeprecatedFieldsAreCorrected', () => { ).toEqual({ id: 'my-foo-id', taskType: 'foo', - recurringSchedule: '10m', + schedule: { interval: '10m' }, params: {}, state: {}, }); diff --git a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts index a013e0cc26460..a879e1ae1841a 100644 --- a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts +++ b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts @@ -8,11 +8,11 @@ import { TaskInstance, TaskInstanceWithDeprecatedFields } from '../task'; export function ensureDeprecatedFieldsAreCorrected({ interval, - recurringSchedule, + schedule, ...taskInstance }: TaskInstanceWithDeprecatedFields): TaskInstance { return { ...taskInstance, - recurringSchedule: recurringSchedule || interval, + schedule: schedule || (interval ? { interval } : undefined), }; } diff --git a/x-pack/legacy/plugins/task_manager/mappings.json b/x-pack/legacy/plugins/task_manager/mappings.json index 71684201b4574..fc8bd29028fd1 100644 --- a/x-pack/legacy/plugins/task_manager/mappings.json +++ b/x-pack/legacy/plugins/task_manager/mappings.json @@ -16,8 +16,12 @@ "retryAt": { "type": "date" }, - "recurringSchedule": { - "type": "text" + "schedule": { + "properties": { + "interval": { + "type": "text" + } + } }, "attempts": { "type": "integer" diff --git a/x-pack/legacy/plugins/task_manager/migrations.ts b/x-pack/legacy/plugins/task_manager/migrations.ts index 4a4c5db88bbce..03ff2fd45efca 100644 --- a/x-pack/legacy/plugins/task_manager/migrations.ts +++ b/x-pack/legacy/plugins/task_manager/migrations.ts @@ -12,16 +12,25 @@ export const migrations = { ...doc, updated_at: new Date().toISOString(), }), - '7.6.0': renameAttribute('interval', 'recurringSchedule'), + '7.6.0': moveIntervalIntoSchedule, }, }; -function renameAttribute(oldName: string, newName: string) { - return ({ attributes: { [oldName]: value, ...attributes }, ...doc }: SavedObject) => ({ +function moveIntervalIntoSchedule({ + attributes: { interval, ...attributes }, + ...doc +}: SavedObject) { + return { ...doc, attributes: { ...attributes, - [newName]: value, + ...(interval + ? { + schedule: { + interval, + }, + } + : {}), }, - }); + }; } diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts index ef490685d895a..bd1936a078a26 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts @@ -50,7 +50,7 @@ describe('mark_available_tasks_as_claimed', () => { // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an recurringSchedule or the attempts < the maximum configured + // Either task has an schedule or the attempts < the maximum configured shouldBeOneOf( TaskWithRecurringSchedule, ...Object.entries(definitions).map(([type, { maxAttempts }]) => @@ -104,7 +104,7 @@ describe('mark_available_tasks_as_claimed', () => { { bool: { should: [ - { exists: { field: 'task.recurringSchedule' } }, + { exists: { field: 'task.schedule' } }, { bool: { must: [ diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index 4848bafda4439..d70e17978bc7d 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -15,7 +15,7 @@ import { } from './query_clauses'; export const TaskWithRecurringSchedule: ExistsBoolClause = { - exists: { field: 'task.recurringSchedule' }, + exists: { field: 'task.schedule' }, }; export function taskWithLessThanMaxAttempts( type: string, diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts index 1b9992a3111cf..48e87582ce3fe 100644 --- a/x-pack/legacy/plugins/task_manager/task.ts +++ b/x-pack/legacy/plugins/task_manager/task.ts @@ -176,6 +176,13 @@ export enum TaskLifecycleResult { export type TaskLifecycle = TaskStatus | TaskLifecycleResult; +export interface IntervalSchedule { + /** + * An interval in minutes (e.g. '5m'). If specified, this is a recurring task. + * */ + interval: string; +} + /* * A task instance represents all of the data required to store, fetch, * and execute a task. @@ -224,7 +231,7 @@ export interface TaskInstance { * * Currently, this supports a single format: an interval in minutes or seconds (e.g. '5m', '30s'). */ - recurringSchedule?: string; + schedule?: IntervalSchedule; /** * A task-specific set of parameters, used by the task's run function to tailor @@ -259,7 +266,7 @@ export interface TaskInstance { /** * Support for the depracated interval field, this should be removed in version 8.0.0 * and marked as a breaking change, ideally nutil then all usage of `interval` will be - * replaced with use of `recurringSchedule` + * replaced with use of `schedule` */ export interface TaskInstanceWithDeprecatedFields extends TaskInstance { /** diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts index 7029e218f0214..34b4d04249039 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts @@ -487,7 +487,7 @@ describe('TaskManager', () => { conflicts: 'proceed', }, body: - '{"query":{"bool":{"must":[{"term":{"type":"task"}},{"bool":{"must":[{"bool":{"should":[{"bool":{"must":[{"term":{"task.status":"idle"}},{"range":{"task.runAt":{"lte":"now"}}}]}},{"bool":{"must":[{"bool":{"should":[{"term":{"task.status":"running"}},{"term":{"task.status":"claiming"}}]}},{"range":{"task.retryAt":{"lte":"now"}}}]}}]}},{"bool":{"should":[{"exists":{"field":"task.recurringSchedule"}},{"bool":{"must":[{"term":{"task.taskType":"vis_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"lens_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.server-log"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.slack"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.email"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.index"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.pagerduty"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.webhook"}},{"range":{"task.attempts":{"lt":1}}}]}}]}}]}}]}},"sort":{"_script":{"type":"number","order":"asc","script":{"lang":"expression","source":"doc[\'task.retryAt\'].value || doc[\'task.runAt\'].value"}}},"seq_no_primary_term":true,"script":{"source":"ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;","lang":"painless","params":{"ownerId":"kibana:5b2de169-2785-441b-ae8c-186a1936b17d","retryAt":"2019-10-31T13:35:43.579Z","status":"claiming"}}}', + '{"query":{"bool":{"must":[{"term":{"type":"task"}},{"bool":{"must":[{"bool":{"should":[{"bool":{"must":[{"term":{"task.status":"idle"}},{"range":{"task.runAt":{"lte":"now"}}}]}},{"bool":{"must":[{"bool":{"should":[{"term":{"task.status":"running"}},{"term":{"task.status":"claiming"}}]}},{"range":{"task.retryAt":{"lte":"now"}}}]}}]}},{"bool":{"should":[{"exists":{"field":"task.schedule"}},{"bool":{"must":[{"term":{"task.taskType":"vis_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"lens_telemetry"}},{"range":{"task.attempts":{"lt":3}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.server-log"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.slack"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.email"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.index"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.pagerduty"}},{"range":{"task.attempts":{"lt":1}}}]}},{"bool":{"must":[{"term":{"task.taskType":"actions:.webhook"}},{"range":{"task.attempts":{"lt":1}}}]}}]}}]}}]}},"sort":{"_script":{"type":"number","order":"asc","script":{"lang":"expression","source":"doc[\'task.retryAt\'].value || doc[\'task.runAt\'].value"}}},"seq_no_primary_term":true,"script":{"source":"ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;","lang":"painless","params":{"ownerId":"kibana:5b2de169-2785-441b-ae8c-186a1936b17d","retryAt":"2019-10-31T13:35:43.579Z","status":"claiming"}}}', statusCode: 400, response: '{"error":{"root_cause":[{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts"}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":".kibana_task_manager_1","node":"24A4QbjHSK6prvtopAKLKw","reason":{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts"}}],"caused_by":{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts","caused_by":{"type":"illegal_argument_exception","reason":"cannot execute [inline] scripts"}}},"status":400}', diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts index dd918e339afa4..dd33f49a44603 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts @@ -86,10 +86,10 @@ describe('TaskManagerRunner', () => { expect(instance.state).toEqual({ hey: 'there' }); }); - test('reschedules tasks that have an recurringSchedule', async () => { + test('reschedules tasks that have an schedule', async () => { const { runner, store } = testOpts({ instance: { - recurringSchedule: '10m', + schedule: { interval: '10m' }, status: TaskStatus.Running, startedAt: new Date(), }, @@ -133,11 +133,11 @@ describe('TaskManagerRunner', () => { sinon.assert.calledWithMatch(store.update, { runAt }); }); - test('tasks that return runAt override the recurringSchedule', async () => { + test('tasks that return runAt override the schedule', async () => { const runAt = minutesFromNow(_.random(5)); const { runner, store } = testOpts({ instance: { - recurringSchedule: '20m', + schedule: { interval: '20m' }, }, definitions: { bar: { @@ -161,7 +161,7 @@ describe('TaskManagerRunner', () => { const { runner, store } = testOpts({ instance: { id, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -237,7 +237,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -366,7 +366,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: '1m', + schedule: { interval: '1m' }, startedAt: new Date(), }, definitions: { @@ -402,7 +402,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -436,7 +436,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -468,7 +468,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -503,7 +503,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -538,7 +538,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -570,7 +570,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: '1m', + schedule: { interval: '1m' }, startedAt: new Date(), }, definitions: { @@ -601,7 +601,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: undefined, + schedule: undefined, }, definitions: { bar: { @@ -633,7 +633,7 @@ describe('TaskManagerRunner', () => { instance: { id, attempts: initialAttempts, - recurringSchedule: `${intervalSeconds}s`, + schedule: { interval: `${intervalSeconds}s` }, startedAt: new Date(), }, definitions: { @@ -749,7 +749,7 @@ describe('TaskManagerRunner', () => { onTaskEvent, instance: { id, - recurringSchedule: '1m', + schedule: { interval: '1m' }, }, definitions: { bar: { @@ -800,7 +800,7 @@ describe('TaskManagerRunner', () => { onTaskEvent, instance: { id, - recurringSchedule: '1m', + schedule: { interval: '1m' }, startedAt: new Date(), }, definitions: { diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts index dbcfb6d7da783..56ab49bdc629e 100644 --- a/x-pack/legacy/plugins/task_manager/task_runner.ts +++ b/x-pack/legacy/plugins/task_manager/task_runner.ts @@ -205,7 +205,7 @@ export class TaskManagerRunner implements TaskRunner { status: TaskStatus.Running, startedAt: now, attempts, - retryAt: this.instance.recurringSchedule + retryAt: this.instance.schedule ? intervalFromNow(this.definition.timeout)! : this.getRetryDelay({ attempts, @@ -273,7 +273,7 @@ export class TaskManagerRunner implements TaskRunner { } private shouldTryToScheduleRetry(): boolean { - if (this.instance.recurringSchedule) { + if (this.instance.schedule) { return true; } @@ -287,8 +287,8 @@ export class TaskManagerRunner implements TaskRunner { if (this.shouldTryToScheduleRetry()) { const { runAt, state, error } = failureResult; // if we're retrying, keep the number of attempts - const { recurringSchedule, attempts } = this.instance; - if (runAt || recurringSchedule) { + const { schedule, attempts } = this.instance; + if (runAt || schedule) { return asOk({ state, attempts, runAt }); } else { // when result.error is truthy, then we're retrying because it failed @@ -314,9 +314,9 @@ export class TaskManagerRunner implements TaskRunner { mapErr(this.rescheduleFailedRun), // if retrying is possible (new runAt) or this is an recurring task - reschedule mapOk(({ runAt, state, attempts = 0 }: Partial) => { - const { startedAt, recurringSchedule } = this.instance; + const { startedAt, schedule: { interval = undefined } = {} } = this.instance; return asOk({ - runAt: runAt || intervalFromDate(startedAt!, recurringSchedule)!, + runAt: runAt || intervalFromDate(startedAt!, interval)!, state, attempts, status: TaskStatus.Idle, @@ -358,7 +358,7 @@ export class TaskManagerRunner implements TaskRunner { await eitherAsync( result, async ({ runAt }: SuccessfulRunResult) => { - if (runAt || this.instance.recurringSchedule) { + if (runAt || this.instance.schedule) { await this.processResultForRecurringTask(result); } else { await this.processResultWhenDone(); diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts index ec2da665eca35..895a80a4ac973 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.test.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts @@ -98,7 +98,7 @@ describe('TaskStore', () => { 'task', { attempts: 0, - recurringSchedule: undefined, + schedule: undefined, params: '{"hello":"world"}', retryAt: null, runAt: '2019-02-12T21:01:22.479Z', @@ -119,7 +119,7 @@ describe('TaskStore', () => { expect(result).toEqual({ id: 'testid', attempts: 0, - recurringSchedule: undefined, + schedule: undefined, params: { hello: 'world' }, retryAt: null, runAt: mockedDate, @@ -271,7 +271,7 @@ describe('TaskStore', () => { task: { runAt, taskType: 'foo', - recurringSchedule: undefined, + schedule: undefined, attempts: 0, status: 'idle', params: '{ "hello": "world" }', @@ -289,7 +289,7 @@ describe('TaskStore', () => { task: { runAt, taskType: 'bar', - recurringSchedule: '5m', + schedule: { interval: '5m' }, attempts: 2, status: 'running', params: '{ "shazm": 1 }', @@ -307,7 +307,7 @@ describe('TaskStore', () => { { attempts: 0, id: 'aaa', - recurringSchedule: undefined, + schedule: undefined, params: { hello: 'world' }, runAt, scheduledAt: mockedDate, @@ -322,7 +322,7 @@ describe('TaskStore', () => { { attempts: 2, id: 'bbb', - recurringSchedule: '5m', + schedule: { interval: '5m' }, params: { shazm: 1 }, runAt, scheduledAt: mockedDate, @@ -476,7 +476,7 @@ describe('TaskStore', () => { { bool: { should: [ - { exists: { field: 'task.recurringSchedule' } }, + { exists: { field: 'task.schedule' } }, { bool: { must: [ @@ -594,7 +594,7 @@ describe('TaskStore', () => { { bool: { should: [ - { exists: { field: 'task.recurringSchedule' } }, + { exists: { field: 'task.schedule' } }, { bool: { must: [ @@ -729,7 +729,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'foo', - recurringSchedule: undefined, + schedule: undefined, attempts: 0, status: 'idle', params: '{ "hello": "world" }', @@ -750,7 +750,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'bar', - recurringSchedule: '5m', + schedule: { interval: '5m' }, attempts: 2, status: 'running', params: '{ "shazm": 1 }', @@ -800,7 +800,7 @@ if (doc['task.runAt'].size()!=0) { { attempts: 0, id: 'aaa', - recurringSchedule: undefined, + schedule: undefined, params: { hello: 'world' }, runAt, scope: ['reporting'], @@ -813,7 +813,7 @@ if (doc['task.runAt'].size()!=0) { { attempts: 2, id: 'bbb', - recurringSchedule: '5m', + schedule: { interval: '5m' }, params: { shazm: 1 }, runAt, scope: ['reporting', 'ceo'], @@ -873,7 +873,7 @@ if (doc['task.runAt'].size()!=0) { task.id, { attempts: task.attempts, - recurringSchedule: undefined, + schedule: undefined, params: JSON.stringify(task.params), retryAt: null, runAt: task.runAt.toISOString(), @@ -891,7 +891,7 @@ if (doc['task.runAt'].size()!=0) { expect(result).toEqual({ ...task, - recurringSchedule: undefined, + schedule: undefined, retryAt: null, scope: undefined, startedAt: null, @@ -1069,7 +1069,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'foo', - recurringSchedule: undefined, + schedule: undefined, attempts: 0, status: 'idle', params: '{ "hello": "world" }', @@ -1093,7 +1093,7 @@ if (doc['task.runAt'].size()!=0) { task: { runAt, taskType: 'bar', - recurringSchedule: '5m', + schedule: { interval: '5m' }, attempts: 2, status: 'running', params: '{ "shazm": 1 }', @@ -1146,7 +1146,7 @@ if (doc['task.runAt'].size()!=0) { id: 'aaa', runAt, taskType: 'foo', - recurringSchedule: undefined, + schedule: undefined, attempts: 0, status: 'idle' as TaskStatus, params: { hello: 'world' }, @@ -1203,7 +1203,7 @@ if (doc['task.runAt'].size()!=0) { id: 'bbb', runAt, taskType: 'bar', - recurringSchedule: '5m', + schedule: { interval: '5m' }, attempts: 2, status: 'running' as TaskStatus, params: { shazm: 1 }, diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index a88af5d715b40..b7aeaf27c3643 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -246,7 +246,7 @@ export class TaskStore { // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has a recurringSchedule or the attempts < the maximum configured + // Either task has a schedule or the attempts < the maximum configured shouldBeOneOf( TaskWithRecurringSchedule, ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index a667fcb7b3175..a4fcd27c34b73 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -34,7 +34,9 @@ export function initRoutes(server, taskTestingEvents) { payload: Joi.object({ task: Joi.object({ taskType: Joi.string().required(), - recurringSchedule: Joi.string().optional(), + schedule: Joi.object({ + interval: Joi.string() + }).optional(), interval: Joi.string().optional(), params: Joi.object().required(), state: Joi.object().optional(), @@ -90,7 +92,7 @@ export function initRoutes(server, taskTestingEvents) { payload: Joi.object({ task: Joi.object({ taskType: Joi.string().required(), - recurringSchedule: Joi.string().optional(), + schedule: Joi.string().optional(), params: Joi.object().required(), state: Joi.object().optional(), id: Joi.string().optional() diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js index fb560273b36b7..af9f77d650c28 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js @@ -110,7 +110,7 @@ export default function ({ getService }) { const scheduledTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: '30m', + schedule: { interval: '30m' }, params: { historyItem }, }); log.debug(`Task created: ${scheduledTask.id}`); @@ -215,7 +215,7 @@ export default function ({ getService }) { const originalTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: `${interval}m`, + schedule: { interval: `${interval}m` }, params: { }, }); @@ -255,7 +255,7 @@ export default function ({ getService }) { const originalTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: `30m`, + schedule: { interval: `30m` }, params: { }, }); @@ -295,7 +295,7 @@ export default function ({ getService }) { it('should return a task run error result when running a task now fails', async () => { const originalTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: `30m`, + schedule: { interval: `30m` }, params: { failWith: 'error on run now', failOn: 3 }, }); @@ -354,7 +354,7 @@ export default function ({ getService }) { it('should return a task run error result when trying to run a task now which is already running', async () => { const longRunningTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: '30m', + schedule: { interval: '30m' }, params: { waitForParams: true }, @@ -449,13 +449,13 @@ export default function ({ getService }) { */ const fastTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: `1s`, + schedule: { interval: `1s` }, params: { }, }); const longRunningTask = await scheduleTask({ taskType: 'sampleTask', - recurringSchedule: `1s`, + schedule: { interval: `1s` }, params: { waitForEvent: 'rescheduleHasHappened' }, From 31f751e212fb958792a85cac57f98d8139b8b01d Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 10:05:55 +0000 Subject: [PATCH 33/45] make interval field a keyword so we can search on it i nthe future --- x-pack/legacy/plugins/task_manager/mappings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/task_manager/mappings.json b/x-pack/legacy/plugins/task_manager/mappings.json index fc8bd29028fd1..1728c8f1c552b 100644 --- a/x-pack/legacy/plugins/task_manager/mappings.json +++ b/x-pack/legacy/plugins/task_manager/mappings.json @@ -19,7 +19,7 @@ "schedule": { "properties": { "interval": { - "type": "text" + "type": "keyword" } } }, From 0213ea3a29a4736aa93360fd90350735fd92ad63 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 13:04:46 +0000 Subject: [PATCH 34/45] moved interval field into generic schedul object on alerts --- x-pack/legacy/plugins/alerting/README.md | 12 +- x-pack/legacy/plugins/alerting/mappings.json | 8 +- .../alerting/server/alerts_client.test.ts | 114 +++++++++++------- .../plugins/alerting/server/alerts_client.ts | 25 ++-- .../server/lib/get_next_run_at.test.ts | 4 +- .../alerting/server/lib/get_next_run_at.ts | 5 +- .../server/lib/task_runner_factory.test.ts | 2 +- .../server/lib/task_runner_factory.ts | 10 +- .../alerting/server/routes/create.test.ts | 10 +- .../plugins/alerting/server/routes/create.ts | 9 +- .../alerting/server/routes/get.test.ts | 2 +- .../alerting/server/routes/update.test.ts | 8 +- .../plugins/alerting/server/routes/update.ts | 9 +- .../legacy/plugins/alerting/server/types.ts | 8 +- .../common/lib/alert_utils.ts | 2 +- .../common/lib/get_test_alert_data.ts | 2 +- .../tests/alerting/alerts.ts | 12 +- .../tests/alerting/create.ts | 32 +++-- .../tests/alerting/find.ts | 4 +- .../security_and_spaces/tests/alerting/get.ts | 2 +- .../tests/alerting/update.ts | 26 ++-- .../spaces_only/tests/alerting/alerts.ts | 2 +- .../spaces_only/tests/alerting/create.ts | 2 +- .../spaces_only/tests/alerting/find.ts | 2 +- .../spaces_only/tests/alerting/get.ts | 2 +- .../spaces_only/tests/alerting/update.ts | 4 +- 26 files changed, 197 insertions(+), 121 deletions(-) diff --git a/x-pack/legacy/plugins/alerting/README.md b/x-pack/legacy/plugins/alerting/README.md index 85dbd75e14174..33679cf6fa422 100644 --- a/x-pack/legacy/plugins/alerting/README.md +++ b/x-pack/legacy/plugins/alerting/README.md @@ -200,7 +200,7 @@ Payload: |name|A name to reference and search in the future.|string| |tags|A list of keywords to reference and search in the future.|string[]| |alertTypeId|The id value of the alert type you want to call when the alert is scheduled to execute.|string| -|interval|The interval in seconds, minutes, hours or days the alert should execute. Example: `10s`, `5m`, `1h`, `1d`.|string| +|schedule|The schedule specifying when this alert should be run, using one of the available schedule formats specified under _Schedule Formats_ below|object| |params|The parameters to pass in to the alert type executor `params` value. This will also validate against the alert type params validator if defined.|object| |actions|Array of the following:
- `group` (string): We support grouping actions in the scenario of escalations or different types of alert instances. If you don't need this, feel free to use `default` as a value.
- `id` (string): The id of the action saved object to execute.
- `params` (object): The map to the `params` the action type will receive. In order to help apply context to strings, we handle them as mustache templates and pass in a default set of context. (see templating actions).|array| @@ -242,7 +242,7 @@ Payload: |Property|Description|Type| |---|---|---| -|interval|The interval in seconds, minutes, hours or days the alert should execute. Example: `10s`, `5m`, `1h`, `1d`.|string| +|schedule|The schedule specifying when this alert should be run, using one of the available schedule formats specified under _Schedule Formats_ below|object| |name|A name to reference and search in the future.|string| |tags|A list of keywords to reference and search in the future.|string[]| |params|The parameters to pass in to the alert type executor `params` value. This will also validate against the alert type params validator if defined.|object| @@ -304,6 +304,14 @@ Params: |---|---|---| |id|The id of the alert you're trying to update the API key for. System will use user in request context to generate an API key for.|string| +##### Schedule Formats +A schedule is structured such that the key specifies the format you wish to use and its value specifies the schedule. + +We currently support the _Interval format_ which specifies the interval in seconds, minutes, hours or days at which the alert should execute. +Example: `{ interval: "10s" }`, `{ interval: "5m" }`, `{ interval: "1h" }`, `{ interval: "1d" }`. + +There are plans to support multiple other schedule formats in the near fuiture. + ## Alert instance factory **alertInstanceFactory(id)** diff --git a/x-pack/legacy/plugins/alerting/mappings.json b/x-pack/legacy/plugins/alerting/mappings.json index 7a7446602351d..9536187116031 100644 --- a/x-pack/legacy/plugins/alerting/mappings.json +++ b/x-pack/legacy/plugins/alerting/mappings.json @@ -13,8 +13,12 @@ "alertTypeId": { "type": "keyword" }, - "interval": { - "type": "keyword" + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } }, "actions": { "type": "nested", diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts index 37eb6a9b21d44..b07dad68da72d 100644 --- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts +++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts @@ -47,7 +47,7 @@ function getMockData(overwrites: Record = {}) { name: 'abc', tags: ['foo'], alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, throttle: null, params: { bar: true, @@ -92,7 +92,7 @@ describe('create()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -157,10 +157,12 @@ describe('create()', () => { ], "alertTypeId": "123", "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", } `); @@ -184,13 +186,15 @@ describe('create()', () => { "apiKeyOwner": undefined, "createdBy": "elastic", "enabled": true, - "interval": "10s", "muteAll": false, "mutedInstanceIds": Array [], "name": "abc", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "tags": Array [ "foo", ], @@ -298,7 +302,7 @@ describe('create()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -399,10 +403,12 @@ describe('create()', () => { ], "alertTypeId": "123", "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", } `); @@ -445,7 +451,7 @@ describe('create()', () => { attributes: { enabled: false, alertTypeId: '123', - interval: 10000, + schedule: { interval: 10000 }, params: { bar: true, }, @@ -484,10 +490,12 @@ describe('create()', () => { "alertTypeId": "123", "enabled": false, "id": "1", - "interval": 10000, "params": Object { "bar": true, }, + "schedule": Object { + "interval": 10000, + }, } `); expect(savedObjectsClient.create).toHaveBeenCalledTimes(1); @@ -585,7 +593,7 @@ describe('create()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -648,7 +656,7 @@ describe('create()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -722,7 +730,7 @@ describe('create()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -794,7 +802,7 @@ describe('create()', () => { createdBy: 'elastic', updatedBy: 'elastic', enabled: true, - interval: '10s', + schedule: { interval: '10s' }, throttle: null, muteAll: false, mutedInstanceIds: [], @@ -820,7 +828,7 @@ describe('enable()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: false, }, @@ -846,7 +854,7 @@ describe('enable()', () => { 'alert', '1', { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -879,7 +887,7 @@ describe('enable()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, }, @@ -897,7 +905,7 @@ describe('enable()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: false, }, @@ -927,7 +935,7 @@ describe('enable()', () => { 'alert', '1', { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -962,7 +970,7 @@ describe('disable()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -976,7 +984,7 @@ describe('disable()', () => { 'alert', '1', { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', apiKey: null, apiKeyOwner: null, @@ -997,7 +1005,7 @@ describe('disable()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: false, scheduledTaskId: 'task-123', @@ -1060,7 +1068,7 @@ describe('muteInstance()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -1088,7 +1096,7 @@ describe('muteInstance()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -1107,7 +1115,7 @@ describe('muteInstance()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -1129,7 +1137,7 @@ describe('unmuteInstance()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -1157,7 +1165,7 @@ describe('unmuteInstance()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -1176,7 +1184,7 @@ describe('unmuteInstance()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, scheduledTaskId: 'task-123', @@ -1199,7 +1207,7 @@ describe('get()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1235,10 +1243,12 @@ describe('get()', () => { ], "alertTypeId": "123", "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, } `); expect(savedObjectsClient.get).toHaveBeenCalledTimes(1); @@ -1257,7 +1267,7 @@ describe('get()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1292,7 +1302,7 @@ describe('find()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1332,10 +1342,12 @@ describe('find()', () => { ], "alertTypeId": "123", "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, }, ], "page": 1, @@ -1362,7 +1374,7 @@ describe('delete()', () => { type: 'alert', attributes: { alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1443,7 +1455,7 @@ describe('update()', () => { type: 'alert', attributes: { enabled: true, - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1470,7 +1482,7 @@ describe('update()', () => { const result = await alertsClient.update({ id: '1', data: { - interval: '10s', + schedule: { interval: '10s' }, name: 'abc', tags: ['foo'], params: { @@ -1501,10 +1513,12 @@ describe('update()', () => { ], "enabled": true, "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", } `); @@ -1528,11 +1542,13 @@ describe('update()', () => { "apiKey": null, "apiKeyOwner": null, "enabled": true, - "interval": "10s", "name": "abc", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", "tags": Array [ "foo", @@ -1598,7 +1614,7 @@ describe('update()', () => { type: 'alert', attributes: { enabled: true, - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1651,7 +1667,7 @@ describe('update()', () => { const result = await alertsClient.update({ id: '1', data: { - interval: '10s', + schedule: { interval: '10s' }, name: 'abc', tags: ['foo'], params: { @@ -1712,10 +1728,12 @@ describe('update()', () => { ], "enabled": true, "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", } `); @@ -1771,7 +1789,7 @@ describe('update()', () => { type: 'alert', attributes: { enabled: true, - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, @@ -1799,7 +1817,7 @@ describe('update()', () => { const result = await alertsClient.update({ id: '1', data: { - interval: '10s', + schedule: { interval: '10s' }, name: 'abc', tags: ['foo'], params: { @@ -1831,10 +1849,12 @@ describe('update()', () => { "apiKey": "MTIzOmFiYw==", "enabled": true, "id": "1", - "interval": "10s", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", } `); @@ -1858,11 +1878,13 @@ describe('update()', () => { "apiKey": "MTIzOmFiYw==", "apiKeyOwner": "elastic", "enabled": true, - "interval": "10s", "name": "abc", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "scheduledTaskId": "task-123", "tags": Array [ "foo", @@ -1909,7 +1931,7 @@ describe('update()', () => { alertsClient.update({ id: '1', data: { - interval: '10s', + schedule: { interval: '10s' }, name: 'abc', tags: ['foo'], params: { @@ -1939,7 +1961,7 @@ describe('updateApiKey()', () => { id: '1', type: 'alert', attributes: { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, }, @@ -1956,7 +1978,7 @@ describe('updateApiKey()', () => { 'alert', '1', { - interval: '10s', + schedule: { interval: '10s' }, alertTypeId: '2', enabled: true, apiKey: Buffer.from('123:abc').toString('base64'), diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.ts index 27fda9871e685..578daa445b6ff 100644 --- a/x-pack/legacy/plugins/alerting/server/alerts_client.ts +++ b/x-pack/legacy/plugins/alerting/server/alerts_client.ts @@ -8,7 +8,14 @@ import Boom from 'boom'; import { omit } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Logger, SavedObjectsClientContract, SavedObjectReference } from 'src/core/server'; -import { Alert, RawAlert, AlertTypeRegistry, AlertAction, AlertType } from './types'; +import { + Alert, + RawAlert, + AlertTypeRegistry, + AlertAction, + AlertType, + IntervalSchedule, +} from './types'; import { TaskManagerStartContract } from './shim'; import { validateAlertTypeParams } from './lib'; import { CreateAPIKeyResult as SecurityPluginCreateAPIKeyResult } from '../../../../plugins/security/server'; @@ -82,7 +89,7 @@ interface UpdateOptions { data: { name: string; tags: string[]; - interval: string; + schedule: IntervalSchedule; actions: NormalizedAlertAction[]; params: Record; }; @@ -145,11 +152,7 @@ export class AlertsClient { if (data.enabled) { let scheduledTask; try { - scheduledTask = await this.scheduleAlert( - createdAlert.id, - rawAlert.alertTypeId, - rawAlert.interval - ); + scheduledTask = await this.scheduleAlert(createdAlert.id, rawAlert.alertTypeId); } catch (e) { // Cleanup data, something went wrong scheduling the task try { @@ -259,11 +262,7 @@ export class AlertsClient { const { attributes, version } = await this.savedObjectsClient.get('alert', id); if (attributes.enabled === false) { const apiKey = await this.createAPIKey(); - const scheduledTask = await this.scheduleAlert( - id, - attributes.alertTypeId, - attributes.interval - ); + const scheduledTask = await this.scheduleAlert(id, attributes.alertTypeId); const username = await this.getUserName(); await this.savedObjectsClient.update( 'alert', @@ -364,7 +363,7 @@ export class AlertsClient { } } - private async scheduleAlert(id: string, alertTypeId: string, interval: string) { + private async scheduleAlert(id: string, alertTypeId: string) { return await this.taskManager.schedule({ taskType: `alerting:${alertTypeId}`, params: { diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.test.ts b/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.test.ts index 852e412689b35..1c4d8a42d2830 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.test.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.test.ts @@ -15,12 +15,12 @@ const mockedNow = new Date('2019-06-03T18:55:25.982Z'); test('Adds interface to given date when result is > Date.now()', () => { const currentRunAt = new Date('2019-06-03T18:55:23.982Z'); - const result = getNextRunAt(currentRunAt, '10s'); + const result = getNextRunAt(currentRunAt, { interval: '10s' }); expect(result).toEqual(new Date('2019-06-03T18:55:33.982Z')); }); test('Uses Date.now() when the result would of been a date in the past', () => { const currentRunAt = new Date('2019-06-03T18:55:13.982Z'); - const result = getNextRunAt(currentRunAt, '10s'); + const result = getNextRunAt(currentRunAt, { interval: '10s' }); expect(result).toEqual(mockedNow); }); diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.ts b/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.ts index 901b614b4d68c..f9867b5372908 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/get_next_run_at.ts @@ -5,9 +5,10 @@ */ import { parseDuration } from './parse_duration'; +import { IntervalSchedule } from '../types'; -export function getNextRunAt(currentRunAt: Date, interval: string) { - let nextRunAt = currentRunAt.getTime() + parseDuration(interval); +export function getNextRunAt(currentRunAt: Date, schedule: IntervalSchedule) { + let nextRunAt = currentRunAt.getTime() + parseDuration(schedule.interval); if (nextRunAt < Date.now()) { // To prevent returning dates in the past, we'll return now instead nextRunAt = Date.now(); diff --git a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts index c21c419977bbe..7966f98c749c8 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.test.ts @@ -74,7 +74,7 @@ const mockedAlertTypeSavedObject = { attributes: { enabled: true, alertTypeId: '123', - interval: '10s', + schedule: { interval: '10s' }, mutedInstanceIds: [], params: { bar: true, diff --git a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts index 051b15fc8dd8f..7bf50d7321cc4 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts @@ -20,6 +20,7 @@ import { GetServicesFunction, RawAlert, SpaceIdToNamespaceFunction, + IntervalSchedule, } from '../types'; export interface TaskRunnerContext { @@ -94,7 +95,7 @@ export class TaskRunnerFactory { const services = getServices(fakeRequest); // Ensure API key is still valid and user has access const { - attributes: { params, actions, interval, throttle, muteAll, mutedInstanceIds }, + attributes: { params, actions, schedule, throttle, muteAll, mutedInstanceIds }, references, } = await services.savedObjectsClient.get('alert', alertId); @@ -167,7 +168,12 @@ export class TaskRunnerFactory { }) ); - const nextRunAt = getNextRunAt(new Date(taskInstance.startedAt!), interval); + const nextRunAt = getNextRunAt( + new Date(taskInstance.startedAt!), + // as we currenrtly only support `interval` we can cast + // this safely + schedule as IntervalSchedule + ); return { state: { diff --git a/x-pack/legacy/plugins/alerting/server/routes/create.test.ts b/x-pack/legacy/plugins/alerting/server/routes/create.test.ts index 634a797880812..a804aff55ad42 100644 --- a/x-pack/legacy/plugins/alerting/server/routes/create.test.ts +++ b/x-pack/legacy/plugins/alerting/server/routes/create.test.ts @@ -13,7 +13,7 @@ server.route(createAlertRoute); const mockedAlert = { alertTypeId: '1', name: 'abc', - interval: '10s', + schedule: { interval: '10s' }, tags: ['foo'], params: { bar: true, @@ -65,11 +65,13 @@ test('creates an alert with proper parameters', async () => { ], "alertTypeId": "1", "id": "123", - "interval": "10s", "name": "abc", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "tags": Array [ "foo", ], @@ -91,11 +93,13 @@ test('creates an alert with proper parameters', async () => { ], "alertTypeId": "1", "enabled": true, - "interval": "10s", "name": "abc", "params": Object { "bar": true, }, + "schedule": Object { + "interval": "10s", + }, "tags": Array [ "foo", ], diff --git a/x-pack/legacy/plugins/alerting/server/routes/create.ts b/x-pack/legacy/plugins/alerting/server/routes/create.ts index cb5277ae19100..417072f978a92 100644 --- a/x-pack/legacy/plugins/alerting/server/routes/create.ts +++ b/x-pack/legacy/plugins/alerting/server/routes/create.ts @@ -7,6 +7,7 @@ import Hapi from 'hapi'; import Joi from 'joi'; import { getDurationSchema } from '../lib'; +import { IntervalSchedule } from '../types'; interface ScheduleRequest extends Hapi.Request { payload: { @@ -14,7 +15,7 @@ interface ScheduleRequest extends Hapi.Request { name: string; tags: string[]; alertTypeId: string; - interval: string; + schedule: IntervalSchedule; actions: Array<{ group: string; id: string; @@ -43,7 +44,11 @@ export const createAlertRoute = { .default([]), alertTypeId: Joi.string().required(), throttle: getDurationSchema().default(null), - interval: getDurationSchema().required(), + schedule: Joi.object() + .keys({ + interval: getDurationSchema().required(), + }) + .required(), params: Joi.object().required(), actions: Joi.array() .items( diff --git a/x-pack/legacy/plugins/alerting/server/routes/get.test.ts b/x-pack/legacy/plugins/alerting/server/routes/get.test.ts index 4d44ee9dfe6bd..b97762d10c960 100644 --- a/x-pack/legacy/plugins/alerting/server/routes/get.test.ts +++ b/x-pack/legacy/plugins/alerting/server/routes/get.test.ts @@ -13,7 +13,7 @@ server.route(getAlertRoute); const mockedAlert = { id: '1', alertTypeId: '1', - interval: '10s', + schedule: { interval: '10s' }, params: { bar: true, }, diff --git a/x-pack/legacy/plugins/alerting/server/routes/update.test.ts b/x-pack/legacy/plugins/alerting/server/routes/update.test.ts index 334fb2120319d..8ce9d94140e6d 100644 --- a/x-pack/legacy/plugins/alerting/server/routes/update.test.ts +++ b/x-pack/legacy/plugins/alerting/server/routes/update.test.ts @@ -16,7 +16,7 @@ const mockedResponse = { id: '1', alertTypeId: '1', tags: ['foo'], - interval: '12s', + schedule: { interval: '12s' }, params: { otherField: false, }, @@ -40,7 +40,7 @@ test('calls the update function with proper parameters', async () => { throttle: null, name: 'abc', tags: ['bar'], - interval: '12s', + schedule: { interval: '12s' }, params: { otherField: false, }, @@ -75,11 +75,13 @@ test('calls the update function with proper parameters', async () => { }, }, ], - "interval": "12s", "name": "abc", "params": Object { "otherField": false, }, + "schedule": Object { + "interval": "12s", + }, "tags": Array [ "bar", ], diff --git a/x-pack/legacy/plugins/alerting/server/routes/update.ts b/x-pack/legacy/plugins/alerting/server/routes/update.ts index 6e8f8557fb24a..bc55d48465602 100644 --- a/x-pack/legacy/plugins/alerting/server/routes/update.ts +++ b/x-pack/legacy/plugins/alerting/server/routes/update.ts @@ -7,6 +7,7 @@ import Joi from 'joi'; import Hapi from 'hapi'; import { getDurationSchema } from '../lib'; +import { IntervalSchedule } from '../types'; interface UpdateRequest extends Hapi.Request { params: { @@ -16,7 +17,7 @@ interface UpdateRequest extends Hapi.Request { alertTypeId: string; name: string; tags: string[]; - interval: string; + schedule: IntervalSchedule; actions: Array<{ group: string; id: string; @@ -45,7 +46,11 @@ export const updateAlertRoute = { tags: Joi.array() .items(Joi.string()) .required(), - interval: getDurationSchema().required(), + schedule: Joi.object() + .keys({ + interval: getDurationSchema().required(), + }) + .required(), params: Joi.object().required(), actions: Joi.array() .items( diff --git a/x-pack/legacy/plugins/alerting/server/types.ts b/x-pack/legacy/plugins/alerting/server/types.ts index 1bec2632d8082..e06e0c45e20b4 100644 --- a/x-pack/legacy/plugins/alerting/server/types.ts +++ b/x-pack/legacy/plugins/alerting/server/types.ts @@ -60,12 +60,16 @@ export interface RawAlertAction extends SavedObjectAttributes { params: AlertActionParams; } +export interface IntervalSchedule extends SavedObjectAttributes { + interval: string; +} + export interface Alert { enabled: boolean; name: string; tags: string[]; alertTypeId: string; - interval: string; + schedule: IntervalSchedule; actions: AlertAction[]; params: Record; scheduledTaskId?: string; @@ -83,7 +87,7 @@ export interface RawAlert extends SavedObjectAttributes { name: string; tags: string[]; alertTypeId: string; - interval: string; + schedule: SavedObjectAttributes; actions: RawAlertAction[]; params: SavedObjectAttributes; scheduledTaskId?: string; diff --git a/x-pack/test/alerting_api_integration/common/lib/alert_utils.ts b/x-pack/test/alerting_api_integration/common/lib/alert_utils.ts index 57b4b3b6c26c6..12b38e939712a 100644 --- a/x-pack/test/alerting_api_integration/common/lib/alert_utils.ts +++ b/x-pack/test/alerting_api_integration/common/lib/alert_utils.ts @@ -179,7 +179,7 @@ export class AlertUtils { const response = await request.send({ enabled: true, name: 'abc', - interval: '1m', + schedule: { interval: '1m' }, throttle: '1m', tags: [], alertTypeId: 'test.always-firing', diff --git a/x-pack/test/alerting_api_integration/common/lib/get_test_alert_data.ts b/x-pack/test/alerting_api_integration/common/lib/get_test_alert_data.ts index ae382652b6234..8655764e3fb8f 100644 --- a/x-pack/test/alerting_api_integration/common/lib/get_test_alert_data.ts +++ b/x-pack/test/alerting_api_integration/common/lib/get_test_alert_data.ts @@ -10,7 +10,7 @@ export function getTestAlertData(overwrites = {}) { name: 'abc', tags: ['foo'], alertTypeId: 'test.noop', - interval: '1m', + schedule: { interval: '1m' }, throttle: '1m', actions: [], params: {}, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/alerts.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/alerts.ts index 8f3996f958bb2..4a089e15eb458 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/alerts.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/alerts.ts @@ -450,7 +450,7 @@ export default function alertTests({ getService }: FtrProviderContext) { const response = await alertUtils.createAlwaysFiringAction({ reference, overwrites: { - interval: '1s', + schedule: { interval: '1s' }, }, }); @@ -490,7 +490,7 @@ export default function alertTests({ getService }: FtrProviderContext) { const response = await alertUtils.createAlwaysFiringAction({ reference, overwrites: { - interval: '1s', + schedule: { interval: '1s' }, params: { index: ES_TEST_INDEX_NAME, reference, @@ -560,7 +560,7 @@ export default function alertTests({ getService }: FtrProviderContext) { const response = await alertUtils.createAlwaysFiringAction({ reference, overwrites: { - interval: '1s', + schedule: { interval: '1s' }, params: { index: ES_TEST_INDEX_NAME, reference, @@ -606,7 +606,7 @@ export default function alertTests({ getService }: FtrProviderContext) { reference, overwrites: { enabled: false, - interval: '1s', + schedule: { interval: '1s' }, }, }); @@ -651,7 +651,7 @@ export default function alertTests({ getService }: FtrProviderContext) { reference, overwrites: { enabled: false, - interval: '1s', + schedule: { interval: '1s' }, }, }); @@ -696,7 +696,7 @@ export default function alertTests({ getService }: FtrProviderContext) { reference, overwrites: { enabled: false, - interval: '1s', + schedule: { interval: '1s' }, }, }); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/create.ts index c0a99ae068e71..3e736a2dcf8c4 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/create.ts @@ -89,7 +89,7 @@ export default function createAlertTests({ getService }: FtrProviderContext) { alertTypeId: 'test.noop', params: {}, createdBy: user.username, - interval: '1m', + schedule: { interval: '1m' }, scheduledTaskId: response.body.scheduledTaskId, throttle: '1m', updatedBy: user.username, @@ -201,10 +201,10 @@ export default function createAlertTests({ getService }: FtrProviderContext) { statusCode: 400, error: 'Bad Request', message: - 'child "name" fails because ["name" is required]. child "alertTypeId" fails because ["alertTypeId" is required]. child "interval" fails because ["interval" is required]. child "params" fails because ["params" is required]. child "actions" fails because ["actions" is required]', + 'child "name" fails because ["name" is required]. child "alertTypeId" fails because ["alertTypeId" is required]. child "schedule" fails because ["schedule" is required]. child "params" fails because ["params" is required]. child "actions" fails because ["actions" is required]', validation: { source: 'payload', - keys: ['name', 'alertTypeId', 'interval', 'params', 'actions'], + keys: ['name', 'alertTypeId', 'schedule', 'params', 'actions'], }, }); break; @@ -250,12 +250,12 @@ export default function createAlertTests({ getService }: FtrProviderContext) { } }); - it('should handle create alert request appropriately when interval is wrong syntax', async () => { + it('should handle create alert request appropriately when interval schedule is wrong syntax', async () => { const response = await supertestWithoutAuth .post(`${getUrlPrefix(space.id)}/api/alert`) .set('kbn-xsrf', 'foo') .auth(user.username, user.password) - .send(getTestAlertData(getTestAlertData({ interval: '10x' }))); + .send(getTestAlertData(getTestAlertData({ schedule: { interval: '10x' } }))); switch (scenario.id) { case 'no_kibana_privileges at space1': @@ -275,10 +275,15 @@ export default function createAlertTests({ getService }: FtrProviderContext) { statusCode: 400, error: 'Bad Request', message: - 'child "interval" fails because ["interval" with value "10x" fails to match the seconds pattern, "interval" with value "10x" fails to match the minutes pattern, "interval" with value "10x" fails to match the hours pattern, "interval" with value "10x" fails to match the days pattern]', + 'child "schedule" fails because [child "interval" fails because ["interval" with value "10x" fails to match the seconds pattern, "interval" with value "10x" fails to match the minutes pattern, "interval" with value "10x" fails to match the hours pattern, "interval" with value "10x" fails to match the days pattern]]', validation: { source: 'payload', - keys: ['interval', 'interval', 'interval', 'interval'], + keys: [ + 'schedule.interval', + 'schedule.interval', + 'schedule.interval', + 'schedule.interval', + ], }, }); break; @@ -287,12 +292,12 @@ export default function createAlertTests({ getService }: FtrProviderContext) { } }); - it('should handle create alert request appropriately when interval is 0', async () => { + it('should handle create alert request appropriately when interval schedule is 0', async () => { const response = await supertestWithoutAuth .post(`${getUrlPrefix(space.id)}/api/alert`) .set('kbn-xsrf', 'foo') .auth(user.username, user.password) - .send(getTestAlertData(getTestAlertData({ interval: '0s' }))); + .send(getTestAlertData(getTestAlertData({ schedule: { interval: '0s' } }))); switch (scenario.id) { case 'no_kibana_privileges at space1': @@ -312,10 +317,15 @@ export default function createAlertTests({ getService }: FtrProviderContext) { statusCode: 400, error: 'Bad Request', message: - 'child "interval" fails because ["interval" with value "0s" fails to match the seconds pattern, "interval" with value "0s" fails to match the minutes pattern, "interval" with value "0s" fails to match the hours pattern, "interval" with value "0s" fails to match the days pattern]', + 'child "schedule" fails because [child "interval" fails because ["interval" with value "0s" fails to match the seconds pattern, "interval" with value "0s" fails to match the minutes pattern, "interval" with value "0s" fails to match the hours pattern, "interval" with value "0s" fails to match the days pattern]]', validation: { source: 'payload', - keys: ['interval', 'interval', 'interval', 'interval'], + keys: [ + 'schedule.interval', + 'schedule.interval', + 'schedule.interval', + 'schedule.interval', + ], }, }); break; diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/find.ts index 359058f2ac23a..4da6c059c5a5e 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/find.ts @@ -59,7 +59,7 @@ export default function createFindTests({ getService }: FtrProviderContext) { name: 'abc', tags: ['foo'], alertTypeId: 'test.noop', - interval: '1m', + schedule: { interval: '1m' }, enabled: true, actions: [], params: {}, @@ -138,7 +138,7 @@ export default function createFindTests({ getService }: FtrProviderContext) { name: 'abc', tags: ['foo'], alertTypeId: 'test.noop', - interval: '1m', + schedule: { interval: '1m' }, enabled: false, actions: [ { diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/get.ts index 1a8109f6b6b3c..9c1f7fea93292 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/get.ts @@ -53,7 +53,7 @@ export default function createGetTests({ getService }: FtrProviderContext) { name: 'abc', tags: ['foo'], alertTypeId: 'test.noop', - interval: '1m', + schedule: { interval: '1m' }, enabled: true, actions: [], params: {}, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/update.ts index 1b1bcef9ad23f..0e2ec0f7bc534 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/update.ts @@ -36,7 +36,7 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { params: { foo: true, }, - interval: '12s', + schedule: { interval: '12s' }, actions: [], throttle: '2m', }; @@ -96,7 +96,7 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { params: { foo: true, }, - interval: '12s', + schedule: { interval: '12s' }, throttle: '1m', actions: [], }); @@ -145,7 +145,7 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { params: { foo: true, }, - interval: '12s', + schedule: { interval: '12s' }, actions: [], }); @@ -203,10 +203,10 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { statusCode: 400, error: 'Bad Request', message: - 'child "throttle" fails because ["throttle" is required]. child "name" fails because ["name" is required]. child "tags" fails because ["tags" is required]. child "interval" fails because ["interval" is required]. child "params" fails because ["params" is required]. child "actions" fails because ["actions" is required]', + 'child "throttle" fails because ["throttle" is required]. child "name" fails because ["name" is required]. child "tags" fails because ["tags" is required]. child "schedule" fails because ["schedule" is required]. child "params" fails because ["params" is required]. child "actions" fails because ["actions" is required]', validation: { source: 'payload', - keys: ['throttle', 'name', 'tags', 'interval', 'params', 'actions'], + keys: ['throttle', 'name', 'tags', 'schedule', 'params', 'actions'], }, }); break; @@ -237,7 +237,7 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { .send({ name: 'bcd', tags: ['bar'], - interval: '1m', + schedule: { interval: '1m' }, throttle: '1m', params: {}, actions: [], @@ -269,12 +269,12 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { } }); - it('should handle update alert request appropriately when interval is wrong syntax', async () => { + it('should handle update alert request appropriately when interval schedule is wrong syntax', async () => { const response = await supertestWithoutAuth .put(`${getUrlPrefix(space.id)}/api/alert/1`) .set('kbn-xsrf', 'foo') .auth(user.username, user.password) - .send(getTestAlertData({ interval: '10x', enabled: undefined })); + .send(getTestAlertData({ schedule: { interval: '10x' }, enabled: undefined })); switch (scenario.id) { case 'no_kibana_privileges at space1': @@ -294,10 +294,16 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { statusCode: 400, error: 'Bad Request', message: - 'child "interval" fails because ["interval" with value "10x" fails to match the seconds pattern, "interval" with value "10x" fails to match the minutes pattern, "interval" with value "10x" fails to match the hours pattern, "interval" with value "10x" fails to match the days pattern]. "alertTypeId" is not allowed', + 'child "schedule" fails because [child "interval" fails because ["interval" with value "10x" fails to match the seconds pattern, "interval" with value "10x" fails to match the minutes pattern, "interval" with value "10x" fails to match the hours pattern, "interval" with value "10x" fails to match the days pattern]]. "alertTypeId" is not allowed', validation: { source: 'payload', - keys: ['interval', 'interval', 'interval', 'interval', 'alertTypeId'], + keys: [ + 'schedule.interval', + 'schedule.interval', + 'schedule.interval', + 'schedule.interval', + 'alertTypeId', + ], }, }); break; diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts.ts index 5fafd8b0bfb61..03e973194b4e2 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts.ts @@ -123,7 +123,7 @@ export default function alertTests({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send( getTestAlertData({ - interval: '1m', + schedule: { interval: '1m' }, alertTypeId: 'test.always-firing', params: { index: ES_TEST_INDEX_NAME, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts index 929905a958abb..0e9011729eb3e 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/create.ts @@ -71,7 +71,7 @@ export default function createAlertTests({ getService }: FtrProviderContext) { alertTypeId: 'test.noop', params: {}, createdBy: null, - interval: '1m', + schedule: { interval: '1m' }, scheduledTaskId: response.body.scheduledTaskId, updatedBy: null, throttle: '1m', diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts index 0d12af6db79b2..3fdd9168eb5cb 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/find.ts @@ -42,7 +42,7 @@ export default function createFindTests({ getService }: FtrProviderContext) { name: 'abc', tags: ['foo'], alertTypeId: 'test.noop', - interval: '1m', + schedule: { interval: '1m' }, enabled: true, actions: [], params: {}, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts index 9e4797bcbf7ad..a49d3478d336d 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get.ts @@ -36,7 +36,7 @@ export default function createGetTests({ getService }: FtrProviderContext) { name: 'abc', tags: ['foo'], alertTypeId: 'test.noop', - interval: '1m', + schedule: { interval: '1m' }, enabled: true, actions: [], params: {}, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts index a6eccf88d9e26..46822781c0cd3 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/update.ts @@ -31,7 +31,7 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { params: { foo: true, }, - interval: '12s', + schedule: { interval: '12s' }, actions: [], throttle: '1m', }; @@ -71,7 +71,7 @@ export default function createUpdateTests({ getService }: FtrProviderContext) { params: { foo: true, }, - interval: '12s', + schedule: { interval: '12s' }, actions: [], throttle: '1m', }) From a7c651e94fa604aade195d194da9573fb1128ec9 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 13:29:39 +0000 Subject: [PATCH 35/45] update siem to use the new shcedule object --- .../routes/__mocks__/request_responses.ts | 2 +- .../server/lib/detection_engine/routes/rules/utils.ts | 2 +- .../server/lib/detection_engine/rules/create_rules.ts | 2 +- .../server/lib/detection_engine/rules/update_rules.ts | 11 +++++++++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 978434859ef95..b77ce920103a3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -186,7 +186,7 @@ export const getResult = (): RuleAlertType => ({ ], references: ['http://www.example.com', 'https://ww.example.com'], }, - interval: '5m', + schedule: { interval: '5m' }, enabled: true, actions: [], throttle: null, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts index c9ae3abdfdc6b..b32efbed66fad 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts @@ -38,7 +38,7 @@ export const transformAlertToRule = (alert: RuleAlertType): Partial Date: Thu, 12 Dec 2019 14:36:09 +0000 Subject: [PATCH 36/45] change comment in SIEM code to make clear why were using a cast --- .../siem/server/lib/detection_engine/rules/update_rules.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts index 4a93a368ddef6..a9761d45afab2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts @@ -119,8 +119,10 @@ export const updateRules = async ({ schedule: { interval: calculateInterval( interval, - // assume its an interval schedule, but obviously this is bad - - // we'll address this in Alerting before merging this change + // TODO: we assume the schedule is an interval schedule due to an problem + // in the Alerting api, which should be addressed by the following + // issue: https://github.com/elastic/kibana/issues/49703 + // Once this issue is closed, the type should be correctly returned by alerting (rule.schedule as IntervalSchedule).interval ), }, From 6ae62e216d7fda53b3d826a905cf8f7d943851b4 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 15:24:49 +0000 Subject: [PATCH 37/45] log a warning whenever the deprecated interval is used --- .../lib/correct_deprecated_fields.test.ts | 81 ++++++++++++++----- .../lib/correct_deprecated_fields.ts | 19 +++-- 2 files changed, 75 insertions(+), 25 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts index c712b5c8ca2cc..408e8d36d3491 100644 --- a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts +++ b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.test.ts @@ -5,16 +5,20 @@ */ import { ensureDeprecatedFieldsAreCorrected } from './correct_deprecated_fields'; +import { mockLogger } from '../test_utils'; describe('ensureDeprecatedFieldsAreCorrected', () => { test('doesnt change tasks without any schedule fields', async () => { expect( - ensureDeprecatedFieldsAreCorrected({ - id: 'my-foo-id', - taskType: 'foo', - params: {}, - state: {}, - }) + ensureDeprecatedFieldsAreCorrected( + { + id: 'my-foo-id', + taskType: 'foo', + params: {}, + state: {}, + }, + mockLogger() + ) ).toEqual({ id: 'my-foo-id', taskType: 'foo', @@ -24,13 +28,16 @@ describe('ensureDeprecatedFieldsAreCorrected', () => { }); test('doesnt change tasks with the schedule field', async () => { expect( - ensureDeprecatedFieldsAreCorrected({ - id: 'my-foo-id', - taskType: 'foo', - schedule: { interval: '10m' }, - params: {}, - state: {}, - }) + ensureDeprecatedFieldsAreCorrected( + { + id: 'my-foo-id', + taskType: 'foo', + schedule: { interval: '10m' }, + params: {}, + state: {}, + }, + mockLogger() + ) ).toEqual({ id: 'my-foo-id', taskType: 'foo', @@ -41,13 +48,16 @@ describe('ensureDeprecatedFieldsAreCorrected', () => { }); test('corrects tasks with the deprecated inteval field', async () => { expect( - ensureDeprecatedFieldsAreCorrected({ - id: 'my-foo-id', - taskType: 'foo', - interval: '10m', - params: {}, - state: {}, - }) + ensureDeprecatedFieldsAreCorrected( + { + id: 'my-foo-id', + taskType: 'foo', + interval: '10m', + params: {}, + state: {}, + }, + mockLogger() + ) ).toEqual({ id: 'my-foo-id', taskType: 'foo', @@ -56,4 +66,35 @@ describe('ensureDeprecatedFieldsAreCorrected', () => { state: {}, }); }); + test('logs a warning when a deprecated inteval is corrected on a task', async () => { + const logger = mockLogger(); + ensureDeprecatedFieldsAreCorrected( + { + taskType: 'foo', + interval: '10m', + params: {}, + state: {}, + }, + logger + ); + expect(logger.warn).toHaveBeenCalledWith( + `Task of type "foo" has been scheduled with the deprecated 'interval' field which is due to be removed in a future release` + ); + }); + test('logs a warning when a deprecated inteval is corrected on a task with an id', async () => { + const logger = mockLogger(); + ensureDeprecatedFieldsAreCorrected( + { + id: 'my-foo-id', + taskType: 'foo', + interval: '10m', + params: {}, + state: {}, + }, + logger + ); + expect(logger.warn).toHaveBeenCalledWith( + `Task "my-foo-id" of type "foo" has been scheduled with the deprecated 'interval' field which is due to be removed in a future release` + ); + }); }); diff --git a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts index a879e1ae1841a..2de95cbb8c2fa 100644 --- a/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts +++ b/x-pack/legacy/plugins/task_manager/lib/correct_deprecated_fields.ts @@ -5,13 +5,22 @@ */ import { TaskInstance, TaskInstanceWithDeprecatedFields } from '../task'; +import { Logger } from '../types'; -export function ensureDeprecatedFieldsAreCorrected({ - interval, - schedule, - ...taskInstance -}: TaskInstanceWithDeprecatedFields): TaskInstance { +export function ensureDeprecatedFieldsAreCorrected( + { id, taskType, interval, schedule, ...taskInstance }: TaskInstanceWithDeprecatedFields, + logger: Logger +): TaskInstance { + if (interval) { + logger.warn( + `Task${ + id ? ` "${id}"` : '' + } of type "${taskType}" has been scheduled with the deprecated 'interval' field which is due to be removed in a future release` + ); + } return { + id, + taskType, ...taskInstance, schedule: schedule || (interval ? { interval } : undefined), }; From 63d2ac26f5b9667b246cbdbf5340771ffa7d7da6 Mon Sep 17 00:00:00 2001 From: Gidi Meir Morris Date: Thu, 12 Dec 2019 15:46:16 +0000 Subject: [PATCH 38/45] typo --- .../siem/server/lib/detection_engine/rules/update_rules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts index a9761d45afab2..2903a410bb7e6 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts @@ -119,7 +119,7 @@ export const updateRules = async ({ schedule: { interval: calculateInterval( interval, - // TODO: we assume the schedule is an interval schedule due to an problem + // TODO: we assume the schedule is an interval schedule due to a problem // in the Alerting api, which should be addressed by the following // issue: https://github.com/elastic/kibana/issues/49703 // Once this issue is closed, the type should be correctly returned by alerting From 79c6d68f2c5a8bc55d59e7ade0886b0cb1e6bd7c Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 16:12:21 +0000 Subject: [PATCH 39/45] use logger in deprecation check --- x-pack/legacy/plugins/task_manager/task_manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts index 39ce4c416c72a..c1149c7bd835d 100644 --- a/x-pack/legacy/plugins/task_manager/task_manager.ts +++ b/x-pack/legacy/plugins/task_manager/task_manager.ts @@ -276,7 +276,7 @@ export class TaskManager { await this.waitUntilStarted(); const { taskInstance: modifiedTask } = await this.middleware.beforeSave({ ...options, - taskInstance: ensureDeprecatedFieldsAreCorrected(taskInstance), + taskInstance: ensureDeprecatedFieldsAreCorrected(taskInstance, this.logger), }); const result = await this.store.schedule(modifiedTask); this.attemptToRun(); From 6cc4c9b2539f12e679ba701451e1975c82f6aca3 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Thu, 12 Dec 2019 16:20:05 +0000 Subject: [PATCH 40/45] cleaned up comment on poller --- x-pack/legacy/plugins/task_manager/task_poller.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index d3e493e70d98f..e20237e7fe834 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -35,15 +35,17 @@ interface Opts { * @prop {number} pollInterval - How often, in milliseconds, we will an event be emnitted, assuming there's capacity to do so * @prop {() => number} getCapacity - A function specifying whether there is capacity to emit new events * @prop {Observable>} pollRequests$ - A stream of requests for polling which can provide an optional argument for the polling phase + * @prop {number} bufferCapacity - How many requests are do we allow our buffer to accumulate before rejecting requests? + * @prop {(...params: T[]) => Promise} work - The work we wish to execute in order to `poll`, this is the operation we're actually executing on request * * @returns {Observable>} - An observable which emits an event whenever a polling event is due to take place, providing access to a singleton Set representing a queue * of unique request argumets of type T. The queue holds all the buffered request arguments streamed in via pollRequests$ */ export function createTaskPoller({ - pollRequests$, - bufferCapacity, pollInterval, getCapacity, + pollRequests$, + bufferCapacity, work, }: Opts): Observable>> { const hasCapacity = () => getCapacity() > 0; From 45718ddf522ba911ffe25816c082df489feb3128 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Fri, 13 Dec 2019 10:21:17 +0000 Subject: [PATCH 41/45] limit subject api by typing as observable --- x-pack/legacy/plugins/task_manager/task_poller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index e20237e7fe834..5afb77659f5d7 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -24,7 +24,7 @@ interface Opts { pollInterval: number; bufferCapacity: number; getCapacity: () => number; - pollRequests$: Subject>; + pollRequests$: Observable>; work: WorkFn; } From a54f0b112ccf737a02ed9acbe026db522184b6ec Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Fri, 13 Dec 2019 15:44:43 +0000 Subject: [PATCH 42/45] simplified scan in poller --- .../plugins/task_manager/task_poller.ts | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts index 5afb77659f5d7..2f701050d760f 100644 --- a/x-pack/legacy/plugins/task_manager/task_poller.ts +++ b/x-pack/legacy/plugins/task_manager/task_poller.ts @@ -16,7 +16,15 @@ import { mapTo, filter, scan, concatMap, tap, catchError } from 'rxjs/operators' import { pipe } from 'fp-ts/lib/pipeable'; import { Option, none, map as mapOptional, getOrElse } from 'fp-ts/lib/Option'; import { pullFromSet } from './lib/pull_from_set'; -import { Result, Err, map as mapResult, asOk, asErr, promiseResult } from './lib/result_type'; +import { + Result, + Err, + isErr, + map as mapResult, + asOk, + asErr, + promiseResult, +} from './lib/result_type'; type WorkFn = (...params: T[]) => Promise; @@ -61,26 +69,19 @@ export function createTaskPoller({ // buffer all requests in a single set (to remove duplicates) as we don't want // work to take place in parallel (it could cause Task Manager to pull in the same // task twice) - scan, Set>( - (queue, request) => - mapResult( - pushOptionalIntoSet(queue, bufferCapacity, request), - // value has been successfully pushed into buffer - () => queue, - // value wasnt pushed into buffer, we must be at capacity - () => { - errors$.next( - asPollingError( - `request capacity reached`, - PollingErrorType.RequestCapacityReached, - request - ) - ); - return queue; - } - ), - new Set() - ), + scan, Set>((queue, request) => { + if (isErr(pushOptionalIntoSet(queue, bufferCapacity, request))) { + // value wasnt pushed into buffer, we must be at capacity + errors$.next( + asPollingError( + `request capacity reached`, + PollingErrorType.RequestCapacityReached, + request + ) + ); + } + return queue; + }, new Set()), // only emit polling events when there's capacity to handle them filter(hasCapacity), // take as many argumented calls as we have capacity for and call `work` with From 9aa684c45314232663546ef88626e6ca8f298ca3 Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Mon, 16 Dec 2019 14:44:45 +0000 Subject: [PATCH 43/45] improve meessaging on cast --- .../plugins/alerting/server/lib/task_runner_factory.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts index 7bf50d7321cc4..fe0979538d04e 100644 --- a/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts +++ b/x-pack/legacy/plugins/alerting/server/lib/task_runner_factory.ts @@ -170,8 +170,9 @@ export class TaskRunnerFactory { const nextRunAt = getNextRunAt( new Date(taskInstance.startedAt!), - // as we currenrtly only support `interval` we can cast - // this safely + // we do not currently have a good way of returning the type + // from SavedObjectsClient, and as we currenrtly require a schedule + // and we only support `interval`, we can cast this safely schedule as IntervalSchedule ); From 30e1ad8633df4f9282c0f419ea8c78379806e95f Mon Sep 17 00:00:00 2001 From: Gidi Morris Date: Tue, 17 Dec 2019 10:06:38 +0000 Subject: [PATCH 44/45] fixed naming --- .../queries/mark_available_tasks_as_claimed.test.ts | 4 ++-- .../task_manager/queries/mark_available_tasks_as_claimed.ts | 2 +- x-pack/legacy/plugins/task_manager/task_store.ts | 4 ++-- .../plugins/task_manager/init_routes.js | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts index bd1936a078a26..93a8187b673be 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.test.ts @@ -18,7 +18,7 @@ import { updateFields, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithRecurringSchedule, + TaskWithSchedule, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, } from './mark_available_tasks_as_claimed'; @@ -52,7 +52,7 @@ describe('mark_available_tasks_as_claimed', () => { shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), // Either task has an schedule or the attempts < the maximum configured shouldBeOneOf( - TaskWithRecurringSchedule, + TaskWithSchedule, ...Object.entries(definitions).map(([type, { maxAttempts }]) => taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) ) diff --git a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts index d70e17978bc7d..6691b31a546bc 100644 --- a/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/legacy/plugins/task_manager/queries/mark_available_tasks_as_claimed.ts @@ -14,7 +14,7 @@ import { RangeBoolClause, } from './query_clauses'; -export const TaskWithRecurringSchedule: ExistsBoolClause = { +export const TaskWithSchedule: ExistsBoolClause = { exists: { field: 'task.schedule' }, }; export function taskWithLessThanMaxAttempts( diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts index 51fa2d728e469..096a8774c8488 100644 --- a/x-pack/legacy/plugins/task_manager/task_store.ts +++ b/x-pack/legacy/plugins/task_manager/task_store.ts @@ -50,7 +50,7 @@ import { updateFields, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithRecurringSchedule, + TaskWithSchedule, taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, taskWithIDsAndRunnableStatus, @@ -251,7 +251,7 @@ export class TaskStore { shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), // Either task has a schedule or the attempts < the maximum configured shouldBeOneOf( - TaskWithRecurringSchedule, + TaskWithSchedule, ...Object.entries(this.definitions).map(([type, { maxAttempts }]) => taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) ) diff --git a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js index 0907396b170df..3330d08dfd0d2 100644 --- a/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js +++ b/x-pack/test/plugin_api_integration/plugins/task_manager/init_routes.js @@ -94,7 +94,6 @@ export function initRoutes(server, taskTestingEvents) { payload: Joi.object({ task: Joi.object({ taskType: Joi.string().required(), - schedule: Joi.string().optional(), params: Joi.object().required(), state: Joi.object().optional(), id: Joi.string().optional(), From 9956ce0e99c871dc2c6b804a6479fa2d67c88336 Mon Sep 17 00:00:00 2001 From: Gidi Meir Morris Date: Wed, 18 Dec 2019 10:01:56 +0000 Subject: [PATCH 45/45] Update x-pack/legacy/plugins/alerting/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Mike Côté --- x-pack/legacy/plugins/alerting/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/alerting/README.md b/x-pack/legacy/plugins/alerting/README.md index 33679cf6fa422..0b4024be39548 100644 --- a/x-pack/legacy/plugins/alerting/README.md +++ b/x-pack/legacy/plugins/alerting/README.md @@ -200,7 +200,7 @@ Payload: |name|A name to reference and search in the future.|string| |tags|A list of keywords to reference and search in the future.|string[]| |alertTypeId|The id value of the alert type you want to call when the alert is scheduled to execute.|string| -|schedule|The schedule specifying when this alert should be run, using one of the available schedule formats specified under _Schedule Formats_ below|object| +|schedule|The schedule specifying when this alert should run, using one of the available schedule formats specified under _Schedule Formats_ below|object| |params|The parameters to pass in to the alert type executor `params` value. This will also validate against the alert type params validator if defined.|object| |actions|Array of the following:
- `group` (string): We support grouping actions in the scenario of escalations or different types of alert instances. If you don't need this, feel free to use `default` as a value.
- `id` (string): The id of the action saved object to execute.
- `params` (object): The map to the `params` the action type will receive. In order to help apply context to strings, we handle them as mustache templates and pass in a default set of context. (see templating actions).|array|