[8.6] increase bulk action retry to 5 (#148169) (#148194)

# Backport

This will backport the following commits from `main` to `8.6`:
- [increase bulk action retry to 5
(#148169)](#148169)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Julia
Bardi","email":"90178898+juliaElastic@users.noreply.github.com"},"sourceCommit":{"committedDate":"2022-12-29T14:28:36Z","message":"increase
bulk action retry to 5 (#148169)\n\n## Summary\r\n\r\nIncrease retry
count to 5 to help retry on agent doc version conflict.\r\nIt looks like
3 retries are not enough for 100k agents update
tags.\r\nhttps://github.com//issues/144161\r\n\r\nThis can
be tested on an ECE high memory instance with 100k
horde\r\nagents.\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"a9ac5aeb1eac631a2c365004c3f38fdca5c33291","branchLabelMapping":{"^v8.7.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Fleet","ci:cloud-deploy","v8.7.0","v8.6.1"],"number":148169,"url":"https://github.com/elastic/kibana/pull/148169","mergeCommit":{"message":"increase
bulk action retry to 5 (#148169)\n\n## Summary\r\n\r\nIncrease retry
count to 5 to help retry on agent doc version conflict.\r\nIt looks like
3 retries are not enough for 100k agents update
tags.\r\nhttps://github.com//issues/144161\r\n\r\nThis can
be tested on an ECE high memory instance with 100k
horde\r\nagents.\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"a9ac5aeb1eac631a2c365004c3f38fdca5c33291"}},"sourceBranch":"main","suggestedTargetBranches":["8.6"],"targetPullRequestStates":[{"branch":"main","label":"v8.7.0","labelRegex":"^v8.7.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/148169","number":148169,"mergeCommit":{"message":"increase
bulk action retry to 5 (#148169)\n\n## Summary\r\n\r\nIncrease retry
count to 5 to help retry on agent doc version conflict.\r\nIt looks like
3 retries are not enough for 100k agents update
tags.\r\nhttps://github.com//issues/144161\r\n\r\nThis can
be tested on an ECE high memory instance with 100k
horde\r\nagents.\r\n\r\n### Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"a9ac5aeb1eac631a2c365004c3f38fdca5c33291"}},{"branch":"8.6","label":"v8.6.1","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>

x-pack/plugins/fleet/server/services/agents/action_runner.ts

@@ -22,7 +22,7 @@ import { getAgentActions } from './actions';
 import { closePointInTime, getAgentsByKuery } from './crud';
 import type { BulkActionsResolver } from './bulk_actions_resolver';
 
-export const MAX_RETRY_COUNT = 3;
+export const MAX_RETRY_COUNT = 5;
 
 export interface ActionParams {
   kuery: string;
@@ -113,10 +113,10 @@ export abstract class ActionRunner {
               );
 
             if (this.retryParams.retryCount === MAX_RETRY_COUNT) {
-              const errorMessage = `Stopping after ${MAX_RETRY_COUNT}rd retry. Error: ${error.message}`;
+              const errorMessage = `Stopping after retry #${MAX_RETRY_COUNT}. Error: ${error.message}`;
               appContextService.getLogger().warn(errorMessage);
 
-              // clean up tasks after 3rd retry reached
+              // clean up tasks after last retry reached
               await Promise.all([
                 this.bulkActionsResolver!.removeIfExists(this.checkTaskId!),
                 this.bulkActionsResolver!.removeIfExists(this.retryParams.taskId!),

x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts

@@ -206,12 +206,12 @@ describe('update_agent_tags', () => {
           tagsToRemove: [],
           kuery: '',
           total: 100,
-          retryCount: 3,
+          retryCount: 5,
         }
       )
     ).rejects.toThrowError('version conflict of 100 agents');
     const errorResults = esClient.bulk.mock.calls[0][0] as any;
-    expect(errorResults.body[1].error).toEqual('version conflict on 3rd retry');
+    expect(errorResults.body[1].error).toEqual('version conflict on last retry');
   });
 
   it('should run add tags async when actioning more agents than batch size', async () => {

x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts

@@ -208,7 +208,7 @@ export async function updateTagsBatch(
         getUuidArray(res.version_conflicts!).map((id) => ({
           agentId: id,
           actionId,
-          error: 'version conflict on 3rd retry',
+          error: 'version conflict on last retry',
         }))
       );
     }