[Fleet] Enforce 10 min cooldown for agent upgrade

x-pack/plugins/fleet/common/services/index.ts

@@ -18,7 +18,7 @@ export { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from './limite
 export { isValidNamespace, INVALID_NAMESPACE_CHARACTERS } from './is_valid_namespace';
 export { isDiffPathProtocol } from './is_diff_path_protocol';
 export { LicenseService } from './license';
-export { isAgentUpgradeable } from './is_agent_upgradeable';
+export * from './is_agent_upgradeable';
 export {
   isAgentRequestDiagnosticsSupported,
   MINIMUM_DIAGNOSTICS_AGENT_VERSION,

x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts

@@ -7,22 +7,22 @@
 
 import type { Agent } from '../types/models/agent';
 
-import { isAgentUpgradeable } from './is_agent_upgradeable';
+import { getRecentUpgradeInfoForAgent, isAgentUpgradeable } from './is_agent_upgradeable';
 
 const getAgent = ({
   version,
   upgradeable = false,
   unenrolling = false,
   unenrolled = false,
   updating = false,
-  upgraded = false,
+  minutesSinceUpgrade,
 }: {
   version: string;
   upgradeable?: boolean;
   unenrolling?: boolean;
   unenrolled?: boolean;
   updating?: boolean;
-  upgraded?: boolean;
+  minutesSinceUpgrade?: number;
 }): Agent => {
   const agent: Agent = {
     id: 'de9006e1-54a7-4320-b24e-927e6fe518a8',
@@ -101,8 +101,8 @@ const getAgent = ({
   if (updating) {
     agent.upgrade_started_at = new Date(Date.now()).toISOString();
   }
-  if (upgraded) {
-    agent.upgraded_at = new Date(Date.now()).toISOString();
+  if (minutesSinceUpgrade) {
+    agent.upgraded_at = new Date(Date.now() - minutesSinceUpgrade * 6e4).toISOString();
   }
   return agent;
 };
@@ -176,9 +176,42 @@ describe('Fleet - isAgentUpgradeable', () => {
       isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, updating: true }), '8.0.0')
     ).toBe(false);
   });
-  it('returns true if agent was recently upgraded', () => {
+  it('returns false if the agent reports upgradeable but was upgraded less than 10 minutes ago', () => {
     expect(
-      isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, upgraded: true }), '8.0.0')
+      isAgentUpgradeable(
+        getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 9 }),
+        '8.0.0'
+      )
+    ).toBe(false);
+  });
+  it('returns true if agent reports upgradeable and was upgraded more than 10 minutes ago', () => {
+    expect(
+      isAgentUpgradeable(
+        getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 11 }),
+        '8.0.0'
+      )
+    ).toBe(true);
+  });
+});
+
+describe('hasAgentBeenUpgradedRecently', () => {
+  it('returns true if the agent was upgraded less than 10 minutes ago', () => {
+    expect(
+      getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0', minutesSinceUpgrade: 9 }))
+        .hasBeenUpgradedRecently
     ).toBe(true);
   });
+
+  it('returns false if the agent was upgraded more than 10 minutes ago', () => {
+    expect(
+      getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0', minutesSinceUpgrade: 11 }))
+        .hasBeenUpgradedRecently
+    ).toBe(false);
+  });
+
+  it('returns false if the agent does not have an upgrade_at field', () => {
+    expect(
+      getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0' })).hasBeenUpgradedRecently
+    ).toBe(false);
+  });
 });

x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts

@@ -11,6 +11,8 @@ import semverGt from 'semver/functions/gt';
 
 import type { Agent } from '../types';
 
+export const AGENT_UPGRADE_COOLDOWN_IN_MIN = 10;
+
 export function isAgentUpgradeable(
   agent: Agent,
   latestAgentVersion: string,
@@ -32,6 +34,10 @@ export function isAgentUpgradeable(
   if (agent.upgrade_started_at && !agent.upgraded_at) {
     return false;
   }
+  // check that the agent has not been upgraded more recently than the monitoring period
+  if (getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently) {
+    return false;
+  }
   if (versionToUpgrade !== undefined) {
     return isNotDowngrade(agentVersion, versionToUpgrade);
   }
@@ -56,3 +62,21 @@ const isNotDowngrade = (agentVersion: string, versionToUpgrade: string) => {
 
   return semverGt(versionToUpgradeNumber, agentVersionNumber);
 };
+
+export function getRecentUpgradeInfoForAgent(agent: Agent): {
+  hasBeenUpgradedRecently: boolean;
+  timeToWaitMs: number;
+} {
+  if (!agent.upgraded_at) {
+    return {
+      hasBeenUpgradedRecently: false,
+      timeToWaitMs: 0,
+    };
+  }
+
+  const elaspedSinceUpgradeInMillis = Date.now() - Date.parse(agent.upgraded_at);
+  const timeToWaitMs = AGENT_UPGRADE_COOLDOWN_IN_MIN * 6e4 - elaspedSinceUpgradeInMillis;
+  const hasBeenUpgradedRecently = elaspedSinceUpgradeInMillis / 6e4 < AGENT_UPGRADE_COOLDOWN_IN_MIN;
+
+  return { hasBeenUpgradedRecently, timeToWaitMs };
+}

x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx

@@ -27,6 +27,8 @@ import type { EuiComboBoxOptionOption } from '@elastic/eui';
 import semverGt from 'semver/functions/gt';
 import semverLt from 'semver/functions/lt';
 
+import { AGENT_UPGRADE_COOLDOWN_IN_MIN } from '../../../../../../../common/services';
+
 import { getMinVersion } from '../../../../../../../common/services/get_min_max_version';
 import {
   AGENT_UPDATING_TIMEOUT_HOURS,
@@ -361,14 +363,32 @@ export const AgentUpgradeAgentModal: React.FunctionComponent<AgentUpgradeAgentMo
             defaultMessage="No selected agents are eligible for an upgrade. Please select one or more eligible agents."
           />
         ) : isSingleAgent ? (
-          <FormattedMessage
-            id="xpack.fleet.upgradeAgents.upgradeSingleDescription"
-            defaultMessage="This action will upgrade the agent running on '{hostName}' to version {version}. This action can not be undone. Are you sure you wish to continue?"
-            values={{
-              hostName: ((agents[0] as Agent).local_metadata.host as any).hostname,
-              version: getVersion(selectedVersion),
-            }}
-          />
+          <>
+            <p>
+              <FormattedMessage
+                id="xpack.fleet.upgradeAgents.upgradeSingleDescription"
+                defaultMessage="This action will upgrade the agent running on '{hostName}' to version {version}. This action can not be undone. Are you sure you wish to continue?"
+                values={{
+                  hostName: ((agents[0] as Agent).local_metadata.host as any).hostname,
+                  version: getVersion(selectedVersion),
+                }}
+              />
+            </p>
+            {isUpdating && (
+              <p>
+                <em>
+                  <FormattedMessage
+                    id="xpack.fleet.upgradeAgents.upgradeSingleTimeout"
+                    // TODO: Add link to docs regarding agent upgrade cooldowns
+                    defaultMessage="Note that you may only restart an upgrade every {minutes} minutes to ensure that the upgrade will not be rolled back."
+                    values={{
+                      minutes: AGENT_UPGRADE_COOLDOWN_IN_MIN,
+                    }}
+                  />
+                </em>
+              </p>
+            )}
+          </>
         ) : (
           <FormattedMessage
             id="xpack.fleet.upgradeAgents.upgradeMultipleDescription"

x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts

@@ -13,12 +13,18 @@ import semverGt from 'semver/functions/gt';
 import semverMajor from 'semver/functions/major';
 import semverMinor from 'semver/functions/minor';
 
+import moment from 'moment';
+
 import type { PostAgentUpgradeResponse } from '../../../common/types';
 import type { PostAgentUpgradeRequestSchema, PostBulkAgentUpgradeRequestSchema } from '../../types';
 import * as AgentService from '../../services/agents';
 import { appContextService } from '../../services';
 import { defaultFleetErrorHandler } from '../../errors';
-import { isAgentUpgradeable } from '../../../common/services';
+import {
+  getRecentUpgradeInfoForAgent,
+  isAgentUpgradeable,
+  AGENT_UPGRADE_COOLDOWN_IN_MIN,
+} from '../../../common/services';
 import { getMaxVersion } from '../../../common/services/get_min_max_version';
 import { getAgentById } from '../../services/agents';
 import type { Agent } from '../../types';
@@ -67,6 +73,24 @@ export const postAgentUpgradeHandler: RequestHandler<
       }
     }
 
+    const { hasBeenUpgradedRecently, timeToWaitMs } = getRecentUpgradeInfoForAgent(agent);
+    const timeToWaitString = moment
+      .utc(moment.duration(timeToWaitMs).asMilliseconds())
+      .format('mm[m]ss[s]');
+
+    if (hasBeenUpgradedRecently) {
+      return response.customError({
+        statusCode: 429,
+        body: {
+          message: `agent ${request.params.agentId} was upgraded less than ${AGENT_UPGRADE_COOLDOWN_IN_MIN} minutes ago. Please wait ${timeToWaitString} before trying again to ensure the upgrade will not be rolled back.`,
+        },
+        headers: {
+          // retry-after expects seconds
+          'retry-after': Math.ceil(timeToWaitMs / 1000).toString(),
+        },
+      });
+    }
+
     if (agent.unenrollment_started_at || agent.unenrolled_at) {
       return response.customError({
         statusCode: 400,

x-pack/plugins/fleet/server/services/agents/upgrade.test.ts

@@ -46,6 +46,7 @@ describe('sendUpgradeAgentsActions (plural)', () => {
     const docs = (calledWith as estypes.BulkRequest)?.body
       ?.filter((i: any) => i.doc)
       .map((i: any) => i.doc);
+
     expect(ids).toEqual(idsToAction);
     for (const doc of docs!) {
       expect(doc).toHaveProperty('upgrade_started_at');

x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts

@@ -10,7 +10,7 @@ import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/
 import { v4 as uuidv4 } from 'uuid';
 import moment from 'moment';
 
-import { isAgentUpgradeable } from '../../../common/services';
+import { getRecentUpgradeInfoForAgent, isAgentUpgradeable } from '../../../common/services';
 
 import type { Agent } from '../../types';
 
@@ -76,9 +76,10 @@ export async function upgradeBatch(
   const latestAgentVersion = await getLatestAvailableVersion();
   const upgradeableResults = await Promise.allSettled(
     agentsToCheckUpgradeable.map(async (agent) => {
-      // Filter out agents currently unenrolling, unenrolled, or not upgradeable b/c of version check
+      // Filter out agents currently unenrolling, unenrolled, recently upgraded or not upgradeable b/c of version check
       const isNotAllowed =
-        !options.force && !isAgentUpgradeable(agent, latestAgentVersion, options.version);
+        getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently ||
+        (!options.force && !isAgentUpgradeable(agent, latestAgentVersion, options.version));
       if (isNotAllowed) {
         throw new FleetError(`Agent ${agent.id} is not upgradeable`);
       }