diff --git a/x-pack/plugins/fleet/common/services/index.ts b/x-pack/plugins/fleet/common/services/index.ts index 04f74404ba382..663cd27deab73 100644 --- a/x-pack/plugins/fleet/common/services/index.ts +++ b/x-pack/plugins/fleet/common/services/index.ts @@ -18,7 +18,7 @@ export { isPackageLimited, doesAgentPolicyAlreadyIncludePackage } from './limite export { isValidNamespace, INVALID_NAMESPACE_CHARACTERS } from './is_valid_namespace'; export { isDiffPathProtocol } from './is_diff_path_protocol'; export { LicenseService } from './license'; -export { isAgentUpgradeable } from './is_agent_upgradeable'; +export * from './is_agent_upgradeable'; export { isAgentRequestDiagnosticsSupported, MINIMUM_DIAGNOSTICS_AGENT_VERSION, diff --git a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts index 8a3f3ce8d59ac..ad8138abbce7f 100644 --- a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts +++ b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts @@ -7,7 +7,7 @@ import type { Agent } from '../types/models/agent'; -import { isAgentUpgradeable } from './is_agent_upgradeable'; +import { getRecentUpgradeInfoForAgent, isAgentUpgradeable } from './is_agent_upgradeable'; const getAgent = ({ version, @@ -15,14 +15,14 @@ const getAgent = ({ unenrolling = false, unenrolled = false, updating = false, - upgraded = false, + minutesSinceUpgrade, }: { version: string; upgradeable?: boolean; unenrolling?: boolean; unenrolled?: boolean; updating?: boolean; - upgraded?: boolean; + minutesSinceUpgrade?: number; }): Agent => { const agent: Agent = { id: 'de9006e1-54a7-4320-b24e-927e6fe518a8', @@ -101,8 +101,8 @@ const getAgent = ({ if (updating) { agent.upgrade_started_at = new Date(Date.now()).toISOString(); } - if (upgraded) { - agent.upgraded_at = new Date(Date.now()).toISOString(); + if (minutesSinceUpgrade) { + agent.upgraded_at = new Date(Date.now() - minutesSinceUpgrade * 6e4).toISOString(); } return agent; }; @@ -176,9 +176,42 @@ describe('Fleet - isAgentUpgradeable', () => { isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, updating: true }), '8.0.0') ).toBe(false); }); - it('returns true if agent was recently upgraded', () => { + it('returns false if the agent reports upgradeable but was upgraded less than 10 minutes ago', () => { expect( - isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, upgraded: true }), '8.0.0') + isAgentUpgradeable( + getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 9 }), + '8.0.0' + ) + ).toBe(false); + }); + it('returns true if agent reports upgradeable and was upgraded more than 10 minutes ago', () => { + expect( + isAgentUpgradeable( + getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 11 }), + '8.0.0' + ) + ).toBe(true); + }); +}); + +describe('hasAgentBeenUpgradedRecently', () => { + it('returns true if the agent was upgraded less than 10 minutes ago', () => { + expect( + getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0', minutesSinceUpgrade: 9 })) + .hasBeenUpgradedRecently ).toBe(true); }); + + it('returns false if the agent was upgraded more than 10 minutes ago', () => { + expect( + getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0', minutesSinceUpgrade: 11 })) + .hasBeenUpgradedRecently + ).toBe(false); + }); + + it('returns false if the agent does not have an upgrade_at field', () => { + expect( + getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0' })).hasBeenUpgradedRecently + ).toBe(false); + }); }); diff --git a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts index f896d6cf97bd4..c7bd21c45af4a 100644 --- a/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts +++ b/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts @@ -11,6 +11,8 @@ import semverGt from 'semver/functions/gt'; import type { Agent } from '../types'; +export const AGENT_UPGRADE_COOLDOWN_IN_MIN = 10; + export function isAgentUpgradeable( agent: Agent, latestAgentVersion: string, @@ -32,6 +34,10 @@ export function isAgentUpgradeable( if (agent.upgrade_started_at && !agent.upgraded_at) { return false; } + // check that the agent has not been upgraded more recently than the monitoring period + if (getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently) { + return false; + } if (versionToUpgrade !== undefined) { return isNotDowngrade(agentVersion, versionToUpgrade); } @@ -56,3 +62,21 @@ const isNotDowngrade = (agentVersion: string, versionToUpgrade: string) => { return semverGt(versionToUpgradeNumber, agentVersionNumber); }; + +export function getRecentUpgradeInfoForAgent(agent: Agent): { + hasBeenUpgradedRecently: boolean; + timeToWaitMs: number; +} { + if (!agent.upgraded_at) { + return { + hasBeenUpgradedRecently: false, + timeToWaitMs: 0, + }; + } + + const elaspedSinceUpgradeInMillis = Date.now() - Date.parse(agent.upgraded_at); + const timeToWaitMs = AGENT_UPGRADE_COOLDOWN_IN_MIN * 6e4 - elaspedSinceUpgradeInMillis; + const hasBeenUpgradedRecently = elaspedSinceUpgradeInMillis / 6e4 < AGENT_UPGRADE_COOLDOWN_IN_MIN; + + return { hasBeenUpgradedRecently, timeToWaitMs }; +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx index 7b35927657959..d361349b3f327 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx @@ -27,6 +27,8 @@ import type { EuiComboBoxOptionOption } from '@elastic/eui'; import semverGt from 'semver/functions/gt'; import semverLt from 'semver/functions/lt'; +import { AGENT_UPGRADE_COOLDOWN_IN_MIN } from '../../../../../../../common/services'; + import { getMinVersion } from '../../../../../../../common/services/get_min_max_version'; import { AGENT_UPDATING_TIMEOUT_HOURS, @@ -361,14 +363,32 @@ export const AgentUpgradeAgentModal: React.FunctionComponent ) : isSingleAgent ? ( - + <> +

+ +

+ {isUpdating && ( +

+ + + +

+ )} + ) : ( { const docs = (calledWith as estypes.BulkRequest)?.body ?.filter((i: any) => i.doc) .map((i: any) => i.doc); + expect(ids).toEqual(idsToAction); for (const doc of docs!) { expect(doc).toHaveProperty('upgrade_started_at'); diff --git a/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts b/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts index 014a9bec89739..b6ab67e5fb5e3 100644 --- a/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts +++ b/x-pack/plugins/fleet/server/services/agents/upgrade_action_runner.ts @@ -10,7 +10,7 @@ import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/ import { v4 as uuidv4 } from 'uuid'; import moment from 'moment'; -import { isAgentUpgradeable } from '../../../common/services'; +import { getRecentUpgradeInfoForAgent, isAgentUpgradeable } from '../../../common/services'; import type { Agent } from '../../types'; @@ -76,9 +76,10 @@ export async function upgradeBatch( const latestAgentVersion = await getLatestAvailableVersion(); const upgradeableResults = await Promise.allSettled( agentsToCheckUpgradeable.map(async (agent) => { - // Filter out agents currently unenrolling, unenrolled, or not upgradeable b/c of version check + // Filter out agents currently unenrolling, unenrolled, recently upgraded or not upgradeable b/c of version check const isNotAllowed = - !options.force && !isAgentUpgradeable(agent, latestAgentVersion, options.version); + getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently || + (!options.force && !isAgentUpgradeable(agent, latestAgentVersion, options.version)); if (isNotAllowed) { throw new FleetError(`Agent ${agent.id} is not upgradeable`); } diff --git a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts index b0cdbd9ece49e..0a3dc09692b68 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts @@ -147,6 +147,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); }); + it('should respond 200 if upgrading agent with version the same as snapshot version and force flag is passed', async () => { const fleetServerVersionSnapshot = makeSnapshotVersion(fleetServerVersion); await es.update({ @@ -170,6 +171,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); }); + it('should respond 200 if upgrading agent with version less than kibana snapshot version', async () => { const fleetServerVersionSnapshot = makeSnapshotVersion(fleetServerVersion); @@ -191,6 +193,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); }); + it('should respond 200 if trying to upgrade with source_uri set', async () => { await es.update({ id: 'agent1', @@ -219,6 +222,7 @@ export default function (providerContext: FtrProviderContext) { const action: any = actionsRes.hits.hits[0]._source; expect(action.data.sourceURI).contain('http://path/to/download'); }); + it('should respond 400 if trying to upgrade to a version that does not match installed kibana version', async () => { const kibanaVersion = await kibanaServer.version.get(); const higherVersion = semver.inc(kibanaVersion, 'patch'); @@ -230,6 +234,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); }); + it('should respond 400 if trying to downgrade version', async () => { await es.update({ id: 'agent1', @@ -249,6 +254,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); }); + it('should respond 400 if trying to upgrade an agent that is unenrolling', async () => { await supertest.post(`/api/fleet/agents/agent1/unenroll`).set('kbn-xsrf', 'xxx').send({ revoke: true, @@ -261,6 +267,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); }); + it('should respond 400 if trying to upgrade an agent that is unenrolled', async () => { await es.update({ id: 'agent1', @@ -344,6 +351,98 @@ export default function (providerContext: FtrProviderContext) { }) .expect(403); }); + + it('should respond 429 if trying to upgrade a recently upgraded agent', async () => { + await es.update({ + id: 'agent1', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(), + local_metadata: { + elastic: { + agent: { + upgradeable: true, + version: '0.0.0', + }, + }, + }, + }, + }, + }); + const response = await supertest + .post(`/api/fleet/agents/agent1/upgrade`) + .set('kbn-xsrf', 'xxx') + .send({ + version: fleetServerVersion, + }) + .expect(429); + + expect(response.body.message).to.contain('was upgraded less than 10 minutes ago'); + + // We don't know how long this test will take to run, so we can't really assert on the actual elapsed time here + expect(response.body.message).to.match(/please wait \d{2}m\d{2}s/i); + + expect(response.header['retry-after']).to.match(/^\d+$/); + }); + + it('should respond 429 if trying to upgrade a recently upgraded agent with force flag', async () => { + await es.update({ + id: 'agent1', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(), + local_metadata: { + elastic: { + agent: { + upgradeable: true, + version: '0.0.0', + }, + }, + }, + }, + }, + }); + await supertest + .post(`/api/fleet/agents/agent1/upgrade`) + .set('kbn-xsrf', 'xxx') + .send({ + version: fleetServerVersion, + force: true, + }) + .expect(429); + }); + + it('should respond 200 if trying to upgrade an agent that was upgraded more than 10 minutes ago', async () => { + await es.update({ + id: 'agent1', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + local_metadata: { + elastic: { + agent: { + upgradeable: true, + upgraded_at: new Date(Date.now() - 11 * 6e4).toString(), + version: '0.0.0', + }, + }, + }, + }, + }, + }); + await supertest + .post(`/api/fleet/agents/agent1/upgrade`) + .set('kbn-xsrf', 'xxx') + .send({ + version: fleetServerVersion, + }) + .expect(200); + }); }); describe('multiple agents', () => { @@ -397,6 +496,7 @@ export default function (providerContext: FtrProviderContext) { }, }); }); + it('should respond 200 to bulk upgrade upgradeable agents and update the agent SOs', async () => { await es.update({ id: 'agent1', @@ -483,6 +583,7 @@ export default function (providerContext: FtrProviderContext) { expect(action.agents).contain('agent1'); expect(action.agents).contain('agent2'); }); + it('should create a .fleet-actions document with the agents, version, and start_time if start_time passed', async () => { await es.update({ id: 'agent1', @@ -675,6 +776,7 @@ export default function (providerContext: FtrProviderContext) { expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined'); expect(typeof agent2data.body.item.upgrade_started_at).to.be('string'); }); + it('should not upgrade an unenrolled agent during bulk_upgrade', async () => { await es.update({ id: 'agent1', @@ -713,6 +815,7 @@ export default function (providerContext: FtrProviderContext) { expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined'); expect(typeof agent2data.body.item.upgrade_started_at).to.be('string'); }); + it('should not upgrade a non-upgradeable agent during bulk_upgrade', async () => { const kibanaVersion = await kibanaServer.version.get(); await es.update({ @@ -765,6 +868,112 @@ export default function (providerContext: FtrProviderContext) { expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined'); expect(typeof agent3data.body.item.upgrade_started_at).to.be('undefined'); }); + + it('should not upgrade a recently upgraded agent during bulk_upgrade', async () => { + await es.update({ + id: 'agent1', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + upgraded_at: new Date(Date.now() - 11 * 6e4).toISOString(), + local_metadata: { + elastic: { + agent: { + upgradeable: true, + version: '0.0.0', + }, + }, + }, + }, + }, + }); + await es.update({ + id: 'agent2', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(), + local_metadata: { + elastic: { + agent: { + upgradeable: true, + version: '0.0.0', + }, + }, + }, + }, + }, + }); + await supertest + .post(`/api/fleet/agents/bulk_upgrade`) + .set('kbn-xsrf', 'xxx') + .send({ + agents: ['agent1', 'agent2'], + version: fleetServerVersion, + }); + const [agent1data, agent2data] = await Promise.all([ + supertest.get(`/api/fleet/agents/agent1`).set('kbn-xsrf', 'xxx'), + supertest.get(`/api/fleet/agents/agent2`).set('kbn-xsrf', 'xxx'), + ]); + expect(typeof agent1data.body.item.upgrade_started_at).to.be('string'); + expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined'); + }); + + it('should not upgrade a recently upgraded agent during bulk_upgrade even with force flag', async () => { + await es.update({ + id: 'agent1', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + upgraded_at: new Date(Date.now() - 11 * 6e4).toISOString(), + local_metadata: { + elastic: { + agent: { + upgradeable: true, + version: '0.0.0', + }, + }, + }, + }, + }, + }); + await es.update({ + id: 'agent2', + refresh: 'wait_for', + index: AGENTS_INDEX, + body: { + doc: { + upgraded_at: new Date(Date.now() - 9 * 6e4).toISOString(), + local_metadata: { + elastic: { + agent: { + upgradeable: true, + version: '0.0.0', + }, + }, + }, + }, + }, + }); + await supertest + .post(`/api/fleet/agents/bulk_upgrade`) + .set('kbn-xsrf', 'xxx') + .send({ + agents: ['agent1', 'agent2'], + version: fleetServerVersion, + force: true, + }); + const [agent1data, agent2data] = await Promise.all([ + supertest.get(`/api/fleet/agents/agent1`).set('kbn-xsrf', 'xxx'), + supertest.get(`/api/fleet/agents/agent2`).set('kbn-xsrf', 'xxx'), + ]); + expect(typeof agent1data.body.item.upgrade_started_at).to.be('string'); + expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined'); + }); + it('should upgrade a non upgradeable agent during bulk_upgrade with force flag', async () => { await es.update({ id: 'agent1', @@ -817,6 +1026,7 @@ export default function (providerContext: FtrProviderContext) { expect(typeof agent2data.body.item.upgrade_started_at).to.be('string'); expect(typeof agent3data.body.item.upgrade_started_at).to.be('string'); }); + it('should respond 400 if trying to bulk upgrade to a version that is higher than the latest installed kibana version', async () => { const kibanaVersion = await kibanaServer.version.get(); const higherVersion = semver.inc(kibanaVersion, 'patch'); @@ -851,6 +1061,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); }); + it('should respond 400 if trying to bulk upgrade to a version that is higher than the latest fleet server version', async () => { const higherVersion = semver.inc(fleetServerVersion, 'patch'); await es.update({ @@ -884,6 +1095,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); }); + it('should prevent any agent to downgrade', async () => { await es.update({ id: 'agent1',