added restart upgrade action (#166154)

## Summary

Ready to review, tests are WIP

Resolves #135539
 
For agents in `Updating` state, adding a `Restart upgrade` action, which
uses the force flag on the API to bypass the updating check. For single
agent, the action is only added if the agent has started upgrade for
more than 2 hours (see discussion in
[issue](#135539 (comment)))
For bulk selection, the action appears if all selected agents are in
updating state.

To verify:
- Start local es, kibana, fleet server (can be docker)
- Enroll agents with horde
- update agent docs with the queries below to simulate stuck in updating
for more than 2 hours (modify the query to target specific agents)
- bulk select agents and check that `Restart upgrade` action is visible,
and it triggers another upgrade with force flag only on the agents stuck
in updating
- when calling the bulk_upgrade API, the UI adds the updating condition
to the query / or includes only those agent ids that are stuck in
updating, depending on query or manual selection

```
curl -sk -XPOST --user elastic:changeme -H 'content-type:application/json' \
http://localhost:9200/_security/role/fleet_superuser -d '
   {
      "indices": [
            {
               "names": [".fleet*",".kibana*"],
               "privileges": ["all"],
               "allow_restricted_indices": true
            }
      ]
   }'

curl -sk -XPOST --user elastic:changeme -H 'content-type:application/json' \
http://localhost:9200/_security/user/fleet_superuser -d '
   {
      "password": "password",
      "roles": ["superuser", "fleet_superuser"]
   }'

    curl -sk -XPOST --user fleet_superuser:password -H 'content-type:application/json' \
    -H'x-elastic-product-origin:fleet' \
    http://localhost:9200/.fleet-agents/_update_by_query -d '
    {
       "script": {
         "source": "ctx._source.upgrade_started_at = \"2023-09-13T11:26:23Z\"",
         "lang": "painless"
       },
       "query": {
         "exists": {
           "field": "tags"
         }
       }
     }'
```


Agent details action:

<img width="1440" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/3704e781-337e-4175-b6d2-a99375b6cc24">

Agent list, bulk action:

<img width="1482" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/74f46861-393e-4a86-ab1f-c21e8d325e89">

Agent list, single agent action:

<img width="369" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/2aa087a1-b6e1-4e44-b1db-34592f3df959">

Agent details callout:
<img width="771" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/d1584fe6-0c98-4033-8527-27235812c004">

Select all agents on first page, restart upgrade modal shows only those
that are stuck in upgrading:
<img width="1317" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/fe858815-4393-4007-abe6-e26e4a884bd3">

Select all agents on all pages, restart upgrade modal shows only those
that are stuck upgrading, with an extra api call to get the count:
<img width="2443" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/481b6ab5-fc87-4586-aa9b-432439234ac6">


### Checklist

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/common/services/agent_status.ts

@@ -56,3 +56,15 @@ export function buildKueryForUpdatingAgents(): string {
 export function buildKueryForInactiveAgents() {
   return 'status:inactive';
 }
+
+export const AGENT_UPDATING_TIMEOUT_HOURS = 2;
+
+export function isStuckInUpdating(agent: Agent): boolean {
+  return (
+    agent.status === 'updating' &&
+    !!agent.upgrade_started_at &&
+    !agent.upgraded_at &&
+    Date.now() - Date.parse(agent.upgrade_started_at) >
+      AGENT_UPDATING_TIMEOUT_HOURS * 60 * 60 * 1000
+  );
+}

x-pack/plugins/fleet/common/types/rest_spec/agent.ts

@@ -101,6 +101,7 @@ export interface PostAgentUpgradeRequest {
   body: {
     source_uri?: string;
     version: string;
+    force?: boolean;
   };
 }
 
@@ -111,6 +112,7 @@ export interface PostBulkAgentUpgradeRequest {
     version: string;
     rollout_duration_seconds?: number;
     start_time?: string;
+    force?: boolean;
   };
 }
 

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/action_menu.test.tsx

@@ -163,3 +163,44 @@ describe('AgentDetailsActionMenu', () => {
     });
   });
 });
+
+describe('Restart upgrade action', () => {
+  function renderAndGetRestartUpgradeButton({
+    agent,
+    agentPolicy,
+  }: {
+    agent: Agent;
+    agentPolicy?: AgentPolicy;
+  }) {
+    const { utils } = renderActions({
+      agent,
+      agentPolicy,
+    });
+
+    return utils.queryByTestId('restartUpgradeBtn');
+  }
+
+  it('should render an active button', async () => {
+    const res = renderAndGetRestartUpgradeButton({
+      agent: {
+        status: 'updating',
+        upgrade_started_at: '2022-11-21T12:27:24Z',
+      } as any,
+      agentPolicy: {} as AgentPolicy,
+    });
+
+    expect(res).not.toBe(null);
+    expect(res).toBeEnabled();
+  });
+
+  it('should not render action if agent is not stuck in updating', async () => {
+    const res = renderAndGetRestartUpgradeButton({
+      agent: {
+        status: 'updating',
+        upgrade_started_at: new Date().toISOString(),
+      } as any,
+      agentPolicy: {} as AgentPolicy,
+    });
+    expect(res).toBe(null);
+  });
+});

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/actions_menu.tsx

@@ -10,6 +10,7 @@ import { EuiPortal, EuiContextMenuItem } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n-react';
 
 import { isAgentRequestDiagnosticsSupported } from '../../../../../../../common/services';
+import { isStuckInUpdating } from '../../../../../../../common/services/agent_status';
 
 import type { Agent, AgentPolicy } from '../../../../types';
 import { useAuthz, useKibanaVersion } from '../../../../hooks';
@@ -41,6 +42,7 @@ export const AgentDetailsActionMenu: React.FunctionComponent<{
   const [isRequestDiagnosticsModalOpen, setIsRequestDiagnosticsModalOpen] = useState(false);
   const [isAgentDetailsJsonFlyoutOpen, setIsAgentDetailsJsonFlyoutOpen] = useState<boolean>(false);
   const isUnenrolling = agent.status === 'unenrolling';
+  const isAgentUpdating = isStuckInUpdating(agent);
 
   const [isContextMenuOpen, setIsContextMenuOpen] = useState(false);
   const onContextMenuChange = useCallback(
@@ -114,6 +116,24 @@ export const AgentDetailsActionMenu: React.FunctionComponent<{
     );
   }
 
+  if (isAgentUpdating) {
+    menuItems.push(
+      <EuiContextMenuItem
+        icon="refresh"
+        onClick={() => {
+          setIsUpgradeModalOpen(true);
+        }}
+        key="restartUpgradeAgent"
+        data-test-subj="restartUpgradeBtn"
+      >
+        <FormattedMessage
+          id="xpack.fleet.agentList.restartUpgradeOneButton"
+          defaultMessage="Restart upgrade"
+        />
+      </EuiContextMenuItem>
+    );
+  }
+
   menuItems.push(
     <EuiContextMenuItem
       icon="inspect"
@@ -180,6 +200,7 @@ export const AgentDetailsActionMenu: React.FunctionComponent<{
               setIsUpgradeModalOpen(false);
               refreshAgent();
             }}
+            isUpdating={isAgentUpdating}
           />
         </EuiPortal>
       )}

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_details/agent_details_overview.tsx

@@ -129,7 +129,7 @@ export const AgentDetailsOverviewSection: React.FunctionComponent<{
               title: i18n.translate('xpack.fleet.agentDetails.statusLabel', {
                 defaultMessage: 'Status',
               }),
-              description: <AgentHealth agent={agent} showOfflinePreviousStatus={true} />,
+              description: <AgentHealth agent={agent} fromDetails={true} />,
             },
             {
               title: i18n.translate('xpack.fleet.agentDetails.lastActivityLabel', {

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/bulk_actions.test.tsx

@@ -80,6 +80,7 @@ describe('AgentBulkActions', () => {
       expect(results.getByText('Upgrade 2 agents').closest('button')!).toBeDisabled();
       expect(results.getByText('Schedule upgrade for 2 agents').closest('button')!).toBeDisabled();
       expect(results.queryByText('Request diagnostics for 2 agents')).toBeNull();
+      expect(results.getByText('Restart upgrade 2 agents').closest('button')!).toBeDisabled();
     });
 
     it('should show available actions for 2 selected agents if they are active', async () => {
@@ -112,6 +113,7 @@ describe('AgentBulkActions', () => {
       expect(results.getByText('Unenroll 2 agents').closest('button')!).toBeEnabled();
       expect(results.getByText('Upgrade 2 agents').closest('button')!).toBeEnabled();
       expect(results.getByText('Schedule upgrade for 2 agents').closest('button')!).toBeDisabled();
+      expect(results.getByText('Restart upgrade 2 agents').closest('button')!).toBeEnabled();
     });
 
     it('should add actions if mockedExperimentalFeaturesService is enabled', async () => {
@@ -202,6 +204,7 @@ describe('AgentBulkActions', () => {
       expect(
         results.getByText('Request diagnostics for 10 agents').closest('button')!
       ).toBeEnabled();
+      expect(results.getByText('Restart upgrade 10 agents').closest('button')!).toBeEnabled();
     });
 
     it('should show correct actions for the active agents and exclude the managed agents from the count', async () => {
@@ -255,6 +258,7 @@ describe('AgentBulkActions', () => {
       expect(
         results.getByText('Request diagnostics for 8 agents').closest('button')!
       ).toBeEnabled();
+      expect(results.getByText('Restart upgrade 8 agents').closest('button')!).toBeEnabled();
     });
 
     it('should show correct actions when no managed policies exist', async () => {
@@ -292,6 +296,7 @@ describe('AgentBulkActions', () => {
       expect(
         results.getByText('Request diagnostics for 10 agents').closest('button')!
       ).toBeEnabled();
+      expect(results.getByText('Restart upgrade 10 agents').closest('button')!).toBeEnabled();
     });
 
     it('should generate a correct kuery to select agents', async () => {

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/bulk_actions.tsx

@@ -73,7 +73,11 @@ export const AgentBulkActions: React.FunctionComponent<Props> = ({
   // Actions states
   const [isReassignFlyoutOpen, setIsReassignFlyoutOpen] = useState<boolean>(false);
   const [isUnenrollModalOpen, setIsUnenrollModalOpen] = useState<boolean>(false);
-  const [updateModalState, setUpgradeModalState] = useState({ isOpen: false, isScheduled: false });
+  const [updateModalState, setUpgradeModalState] = useState({
+    isOpen: false,
+    isScheduled: false,
+    isUpdating: false,
+  });
   const [isTagAddVisible, setIsTagAddVisible] = useState<boolean>(false);
   const [isRequestDiagnosticsModalOpen, setIsRequestDiagnosticsModalOpen] =
     useState<boolean>(false);
@@ -219,7 +223,7 @@ export const AgentBulkActions: React.FunctionComponent<Props> = ({
       disabled: !atLeastOneActiveAgentSelected,
       onClick: () => {
         closeMenu();
-        setUpgradeModalState({ isOpen: true, isScheduled: false });
+        setUpgradeModalState({ isOpen: true, isScheduled: false, isUpdating: false });
       },
     },
     {
@@ -237,11 +241,30 @@ export const AgentBulkActions: React.FunctionComponent<Props> = ({
       disabled: !atLeastOneActiveAgentSelected || !isLicenceAllowingScheduleUpgrade,
       onClick: () => {
         closeMenu();
-        setUpgradeModalState({ isOpen: true, isScheduled: true });
+        setUpgradeModalState({ isOpen: true, isScheduled: true, isUpdating: false });
       },
     },
   ];
 
+  menuItems.push({
+    name: (
+      <FormattedMessage
+        id="xpack.fleet.agentBulkActions.restartUpgradeAgents"
+        data-test-subj="agentBulkActionsRestartUpgrade"
+        defaultMessage="Restart upgrade {agentCount, plural, one {# agent} other {# agents}}"
+        values={{
+          agentCount,
+        }}
+      />
+    ),
+    icon: <EuiIcon type="refresh" size="m" />,
+    disabled: !atLeastOneActiveAgentSelected,
+    onClick: () => {
+      closeMenu();
+      setUpgradeModalState({ isOpen: true, isScheduled: false, isUpdating: true });
+    },
+  });
+
   if (diagnosticFileUploadEnabled) {
     menuItems.push({
       name: (
@@ -306,8 +329,9 @@ export const AgentBulkActions: React.FunctionComponent<Props> = ({
             agents={agents}
             agentCount={agentCount}
             isScheduled={updateModalState.isScheduled}
+            isUpdating={updateModalState.isUpdating}
             onClose={() => {
-              setUpgradeModalState({ isOpen: false, isScheduled: false });
+              setUpgradeModalState({ isOpen: false, isScheduled: false, isUpdating: false });
               refreshAgents();
             }}
           />

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/table_row_actions.test.tsx

@@ -132,4 +132,51 @@ describe('TableRowActions', () => {
       expect(res).not.toBeEnabled();
     });
   });
+
+  describe('Restart upgrade action', () => {
+    function renderAndGetRestartUpgradeButton({
+      agent,
+      agentPolicy,
+    }: {
+      agent: Agent;
+      agentPolicy?: AgentPolicy;
+    }) {
+      const { utils } = renderTableRowActions({
+        agent,
+        agentPolicy,
+      });
+
+      return utils.queryByTestId('restartUpgradeBtn');
+    }
+
+    it('should render an active button', async () => {
+      const res = renderAndGetRestartUpgradeButton({
+        agent: {
+          active: true,
+          status: 'updating',
+          upgrade_started_at: '2022-11-21T12:27:24Z',
+        } as any,
+        agentPolicy: {
+          is_managed: false,
+        } as AgentPolicy,
+      });
+
+      expect(res).not.toBe(null);
+      expect(res).toBeEnabled();
+    });
+
+    it('should not render action if agent is not stuck in updating', async () => {
+      const res = renderAndGetRestartUpgradeButton({
+        agent: {
+          active: true,
+          status: 'updating',
+          upgrade_started_at: new Date().toISOString(),
+        } as any,
+        agentPolicy: {
+          is_managed: false,
+        } as AgentPolicy,
+      });
+      expect(res).toBe(null);
+    });
+  });
 });

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/table_row_actions.tsx

@@ -11,6 +11,8 @@ import { FormattedMessage } from '@kbn/i18n-react';
 
 import { isAgentRequestDiagnosticsSupported } from '../../../../../../../common/services';
 
+import { isStuckInUpdating } from '../../../../../../../common/services/agent_status';
+
 import type { Agent, AgentPolicy } from '../../../../types';
 import { useAuthz, useLink, useKibanaVersion } from '../../../../hooks';
 import { ContextMenuActions } from '../../../../components';
@@ -117,6 +119,24 @@ export const TableRowActions: React.FunctionComponent<{
       </EuiContextMenuItem>
     );
 
+    if (isStuckInUpdating(agent)) {
+      menuItems.push(
+        <EuiContextMenuItem
+          key="agentRestartUpgradeBtn"
+          icon="refresh"
+          onClick={() => {
+            onUpgradeClick();
+          }}
+          data-test-subj="restartUpgradeBtn"
+        >
+          <FormattedMessage
+            id="xpack.fleet.agentList.restartUpgradeOneButton"
+            defaultMessage="Restart upgrade"
+          />
+        </EuiContextMenuItem>
+      );
+    }
+
     if (agentTamperProtectionEnabled && agent.policy_id) {
       menuItems.push(
         <EuiContextMenuItem

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/index.tsx

@@ -490,6 +490,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
               setAgentToUpgrade(undefined);
               refreshAgents();
             }}
+            isUpdating={Boolean(agentToUpgrade.upgrade_started_at && !agentToUpgrade.upgraded_at)}
           />
         </EuiPortal>
       )}