diff --git a/x-pack/plugins/fleet/common/services/index.ts b/x-pack/plugins/fleet/common/services/index.ts
index 04f74404ba382..663cd27deab73 100644
--- a/x-pack/plugins/fleet/common/services/index.ts
+++ b/x-pack/plugins/fleet/common/services/index.ts
@@ -18,7 +18,7 @@ export { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from './limite
export { isValidNamespace, INVALID_NAMESPACE_CHARACTERS } from './is_valid_namespace';
export { isDiffPathProtocol } from './is_diff_path_protocol';
export { LicenseService } from './license';
-export { isAgentUpgradeable } from './is_agent_upgradeable';
+export * from './is_agent_upgradeable';
export {
isAgentRequestDiagnosticsSupported,
MINIMUM_DIAGNOSTICS_AGENT_VERSION,
diff --git a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts
index 8a3f3ce8d59ac..ad8138abbce7f 100644
--- a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts
+++ b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts
@@ -7,7 +7,7 @@
import type { Agent } from '../types/models/agent';
-import { isAgentUpgradeable } from './is_agent_upgradeable';
+import { getRecentUpgradeInfoForAgent, isAgentUpgradeable } from './is_agent_upgradeable';
const getAgent = ({
version,
@@ -15,14 +15,14 @@ const getAgent = ({
unenrolling = false,
unenrolled = false,
updating = false,
- upgraded = false,
+ minutesSinceUpgrade,
}: {
version: string;
upgradeable?: boolean;
unenrolling?: boolean;
unenrolled?: boolean;
updating?: boolean;
- upgraded?: boolean;
+ minutesSinceUpgrade?: number;
}): Agent => {
const agent: Agent = {
id: 'de9006e1-54a7-4320-b24e-927e6fe518a8',
@@ -101,8 +101,8 @@ const getAgent = ({
if (updating) {
agent.upgrade_started_at = new Date(Date.now()).toISOString();
}
- if (upgraded) {
- agent.upgraded_at = new Date(Date.now()).toISOString();
+ if (minutesSinceUpgrade) {
+ agent.upgraded_at = new Date(Date.now() - minutesSinceUpgrade * 6e4).toISOString();
}
return agent;
};
@@ -176,9 +176,42 @@ describe('Fleet - isAgentUpgradeable', () => {
isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, updating: true }), '8.0.0')
).toBe(false);
});
- it('returns true if agent was recently upgraded', () => {
+ it('returns false if the agent reports upgradeable but was upgraded less than 10 minutes ago', () => {
expect(
- isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, upgraded: true }), '8.0.0')
+ isAgentUpgradeable(
+ getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 9 }),
+ '8.0.0'
+ )
+ ).toBe(false);
+ });
+ it('returns true if agent reports upgradeable and was upgraded more than 10 minutes ago', () => {
+ expect(
+ isAgentUpgradeable(
+ getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 11 }),
+ '8.0.0'
+ )
+ ).toBe(true);
+ });
+});
+
+describe('hasAgentBeenUpgradedRecently', () => {
+ it('returns true if the agent was upgraded less than 10 minutes ago', () => {
+ expect(
+ getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0', minutesSinceUpgrade: 9 }))
+ .hasBeenUpgradedRecently
).toBe(true);
});
+
+ it('returns false if the agent was upgraded more than 10 minutes ago', () => {
+ expect(
+ getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0', minutesSinceUpgrade: 11 }))
+ .hasBeenUpgradedRecently
+ ).toBe(false);
+ });
+
+ it('returns false if the agent does not have an upgrade_at field', () => {
+ expect(
+ getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0' })).hasBeenUpgradedRecently
+ ).toBe(false);
+ });
});
diff --git a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts
index f896d6cf97bd4..c7bd21c45af4a 100644
--- a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts
+++ b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts
@@ -11,6 +11,8 @@ import semverGt from 'semver/functions/gt';
import type { Agent } from '../types';
+export const AGENT_UPGRADE_COOLDOWN_IN_MIN = 10;
+
export function isAgentUpgradeable(
agent: Agent,
latestAgentVersion: string,
@@ -32,6 +34,10 @@ export function isAgentUpgradeable(
if (agent.upgrade_started_at && !agent.upgraded_at) {
return false;
}
+ // check that the agent has not been upgraded more recently than the monitoring period
+ if (getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently) {
+ return false;
+ }
if (versionToUpgrade !== undefined) {
return isNotDowngrade(agentVersion, versionToUpgrade);
}
@@ -56,3 +62,21 @@ const isNotDowngrade = (agentVersion: string, versionToUpgrade: string) => {
return semverGt(versionToUpgradeNumber, agentVersionNumber);
};
+
+export function getRecentUpgradeInfoForAgent(agent: Agent): {
+ hasBeenUpgradedRecently: boolean;
+ timeToWaitMs: number;
+} {
+ if (!agent.upgraded_at) {
+ return {
+ hasBeenUpgradedRecently: false,
+ timeToWaitMs: 0,
+ };
+ }
+
+ const elaspedSinceUpgradeInMillis = Date.now() - Date.parse(agent.upgraded_at);
+ const timeToWaitMs = AGENT_UPGRADE_COOLDOWN_IN_MIN * 6e4 - elaspedSinceUpgradeInMillis;
+ const hasBeenUpgradedRecently = elaspedSinceUpgradeInMillis / 6e4 < AGENT_UPGRADE_COOLDOWN_IN_MIN;
+
+ return { hasBeenUpgradedRecently, timeToWaitMs };
+}
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx
index 7b35927657959..d361349b3f327 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx
@@ -27,6 +27,8 @@ import type { EuiComboBoxOptionOption } from '@elastic/eui';
import semverGt from 'semver/functions/gt';
import semverLt from 'semver/functions/lt';
+import { AGENT_UPGRADE_COOLDOWN_IN_MIN } from '../../../../../../../common/services';
+
import { getMinVersion } from '../../../../../../../common/services/get_min_max_version';
import {
AGENT_UPDATING_TIMEOUT_HOURS,
@@ -361,14 +363,32 @@ export const AgentUpgradeAgentModal: React.FunctionComponent
) : isSingleAgent ? (
-
+ <>
+
+
+
+ {isUpdating && (
+
+
+
+
+
+ )}
+ >
) : (
{
const docs = (calledWith as estypes.BulkRequest)?.body
?.filter((i: any) => i.doc)
.map((i: any) => i.doc);
+
expect(ids).toEqual(idsToAction);
for (const doc of docs!) {
expect(doc).toHaveProperty('upgrade_started_at');
diff --git a/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts b/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts
index 014a9bec89739..b6ab67e5fb5e3 100644
--- a/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts
+++ b/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts
@@ -10,7 +10,7 @@ import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/
import { v4 as uuidv4 } from 'uuid';
import moment from 'moment';
-import { isAgentUpgradeable } from '../../../common/services';
+import { getRecentUpgradeInfoForAgent, isAgentUpgradeable } from '../../../common/services';
import type { Agent } from '../../types';
@@ -76,9 +76,10 @@ export async function upgradeBatch(
const latestAgentVersion = await getLatestAvailableVersion();
const upgradeableResults = await Promise.allSettled(
agentsToCheckUpgradeable.map(async (agent) => {
- // Filter out agents currently unenrolling, unenrolled, or not upgradeable b/c of version check
+ // Filter out agents currently unenrolling, unenrolled, recently upgraded or not upgradeable b/c of version check
const isNotAllowed =
- !options.force && !isAgentUpgradeable(agent, latestAgentVersion, options.version);
+ getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently ||
+ (!options.force && !isAgentUpgradeable(agent, latestAgentVersion, options.version));
if (isNotAllowed) {
throw new FleetError(`Agent ${agent.id} is not upgradeable`);
}
diff --git a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts
index b0cdbd9ece49e..0a3dc09692b68 100644
--- a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts
+++ b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts
@@ -147,6 +147,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(400);
});
+
it('should respond 200 if upgrading agent with version the same as snapshot version and force flag is passed', async () => {
const fleetServerVersionSnapshot = makeSnapshotVersion(fleetServerVersion);
await es.update({
@@ -170,6 +171,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(200);
});
+
it('should respond 200 if upgrading agent with version less than kibana snapshot version', async () => {
const fleetServerVersionSnapshot = makeSnapshotVersion(fleetServerVersion);
@@ -191,6 +193,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(200);
});
+
it('should respond 200 if trying to upgrade with source_uri set', async () => {
await es.update({
id: 'agent1',
@@ -219,6 +222,7 @@ export default function (providerContext: FtrProviderContext) {
const action: any = actionsRes.hits.hits[0]._source;
expect(action.data.sourceURI).contain('http://path/to/download');
});
+
it('should respond 400 if trying to upgrade to a version that does not match installed kibana version', async () => {
const kibanaVersion = await kibanaServer.version.get();
const higherVersion = semver.inc(kibanaVersion, 'patch');
@@ -230,6 +234,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(400);
});
+
it('should respond 400 if trying to downgrade version', async () => {
await es.update({
id: 'agent1',
@@ -249,6 +254,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(400);
});
+
it('should respond 400 if trying to upgrade an agent that is unenrolling', async () => {
await supertest.post(`/api/fleet/agents/agent1/unenroll`).set('kbn-xsrf', 'xxx').send({
revoke: true,
@@ -261,6 +267,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(400);
});
+
it('should respond 400 if trying to upgrade an agent that is unenrolled', async () => {
await es.update({
id: 'agent1',
@@ -344,6 +351,98 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(403);
});
+
+ it('should respond 429 if trying to upgrade a recently upgraded agent', async () => {
+ await es.update({
+ id: 'agent1',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(),
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ const response = await supertest
+ .post(`/api/fleet/agents/agent1/upgrade`)
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ version: fleetServerVersion,
+ })
+ .expect(429);
+
+ expect(response.body.message).to.contain('was upgraded less than 10 minutes ago');
+
+ // We don't know how long this test will take to run, so we can't really assert on the actual elapsed time here
+ expect(response.body.message).to.match(/please wait \d{2}m\d{2}s/i);
+
+ expect(response.header['retry-after']).to.match(/^\d+$/);
+ });
+
+ it('should respond 429 if trying to upgrade a recently upgraded agent with force flag', async () => {
+ await es.update({
+ id: 'agent1',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(),
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ await supertest
+ .post(`/api/fleet/agents/agent1/upgrade`)
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ version: fleetServerVersion,
+ force: true,
+ })
+ .expect(429);
+ });
+
+ it('should respond 200 if trying to upgrade an agent that was upgraded more than 10 minutes ago', async () => {
+ await es.update({
+ id: 'agent1',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ upgraded_at: new Date(Date.now() - 11 * 6e4).toString(),
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ await supertest
+ .post(`/api/fleet/agents/agent1/upgrade`)
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ version: fleetServerVersion,
+ })
+ .expect(200);
+ });
});
describe('multiple agents', () => {
@@ -397,6 +496,7 @@ export default function (providerContext: FtrProviderContext) {
},
});
});
+
it('should respond 200 to bulk upgrade upgradeable agents and update the agent SOs', async () => {
await es.update({
id: 'agent1',
@@ -483,6 +583,7 @@ export default function (providerContext: FtrProviderContext) {
expect(action.agents).contain('agent1');
expect(action.agents).contain('agent2');
});
+
it('should create a .fleet-actions document with the agents, version, and start_time if start_time passed', async () => {
await es.update({
id: 'agent1',
@@ -675,6 +776,7 @@ export default function (providerContext: FtrProviderContext) {
expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined');
expect(typeof agent2data.body.item.upgrade_started_at).to.be('string');
});
+
it('should not upgrade an unenrolled agent during bulk_upgrade', async () => {
await es.update({
id: 'agent1',
@@ -713,6 +815,7 @@ export default function (providerContext: FtrProviderContext) {
expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined');
expect(typeof agent2data.body.item.upgrade_started_at).to.be('string');
});
+
it('should not upgrade a non-upgradeable agent during bulk_upgrade', async () => {
const kibanaVersion = await kibanaServer.version.get();
await es.update({
@@ -765,6 +868,112 @@ export default function (providerContext: FtrProviderContext) {
expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined');
expect(typeof agent3data.body.item.upgrade_started_at).to.be('undefined');
});
+
+ it('should not upgrade a recently upgraded agent during bulk_upgrade', async () => {
+ await es.update({
+ id: 'agent1',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ upgraded_at: new Date(Date.now() - 11 * 6e4).toISOString(),
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ await es.update({
+ id: 'agent2',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(),
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ await supertest
+ .post(`/api/fleet/agents/bulk_upgrade`)
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ agents: ['agent1', 'agent2'],
+ version: fleetServerVersion,
+ });
+ const [agent1data, agent2data] = await Promise.all([
+ supertest.get(`/api/fleet/agents/agent1`).set('kbn-xsrf', 'xxx'),
+ supertest.get(`/api/fleet/agents/agent2`).set('kbn-xsrf', 'xxx'),
+ ]);
+ expect(typeof agent1data.body.item.upgrade_started_at).to.be('string');
+ expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined');
+ });
+
+ it('should not upgrade a recently upgraded agent during bulk_upgrade even with force flag', async () => {
+ await es.update({
+ id: 'agent1',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ upgraded_at: new Date(Date.now() - 11 * 6e4).toISOString(),
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ await es.update({
+ id: 'agent2',
+ refresh: 'wait_for',
+ index: AGENTS_INDEX,
+ body: {
+ doc: {
+ upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(),
+ local_metadata: {
+ elastic: {
+ agent: {
+ upgradeable: true,
+ version: '0.0.0',
+ },
+ },
+ },
+ },
+ },
+ });
+ await supertest
+ .post(`/api/fleet/agents/bulk_upgrade`)
+ .set('kbn-xsrf', 'xxx')
+ .send({
+ agents: ['agent1', 'agent2'],
+ version: fleetServerVersion,
+ force: true,
+ });
+ const [agent1data, agent2data] = await Promise.all([
+ supertest.get(`/api/fleet/agents/agent1`).set('kbn-xsrf', 'xxx'),
+ supertest.get(`/api/fleet/agents/agent2`).set('kbn-xsrf', 'xxx'),
+ ]);
+ expect(typeof agent1data.body.item.upgrade_started_at).to.be('string');
+ expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined');
+ });
+
it('should upgrade a non upgradeable agent during bulk_upgrade with force flag', async () => {
await es.update({
id: 'agent1',
@@ -817,6 +1026,7 @@ export default function (providerContext: FtrProviderContext) {
expect(typeof agent2data.body.item.upgrade_started_at).to.be('string');
expect(typeof agent3data.body.item.upgrade_started_at).to.be('string');
});
+
it('should respond 400 if trying to bulk upgrade to a version that is higher than the latest installed kibana version', async () => {
const kibanaVersion = await kibanaServer.version.get();
const higherVersion = semver.inc(kibanaVersion, 'patch');
@@ -851,6 +1061,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(400);
});
+
it('should respond 400 if trying to bulk upgrade to a version that is higher than the latest fleet server version', async () => {
const higherVersion = semver.inc(fleetServerVersion, 'patch');
await es.update({
@@ -884,6 +1095,7 @@ export default function (providerContext: FtrProviderContext) {
})
.expect(400);
});
+
it('should prevent any agent to downgrade', async () => {
await es.update({
id: 'agent1',