From 9917399e24ce607d36f7d49d24da5021292d9150 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Wed, 9 Mar 2022 08:50:05 -0800 Subject: [PATCH] support static stability in instance metadata credential provider (#4049) * support static stability in EC2 credential provider * update test files eslintrc version * add changelog * update the error message --- ...ature-EC2MetadataCredentials-f20878a4.json | 5 + lib/config-base.d.ts | 1 + lib/credentials/ec2_metadata_credentials.d.ts | 46 +++-- lib/credentials/ec2_metadata_credentials.js | 93 +++++++-- test/.eslintrc | 2 +- test/credentials.spec.js | 188 +++++++++++++++--- 6 files changed, 267 insertions(+), 68 deletions(-) create mode 100644 .changes/next-release/feature-EC2MetadataCredentials-f20878a4.json diff --git a/.changes/next-release/feature-EC2MetadataCredentials-f20878a4.json b/.changes/next-release/feature-EC2MetadataCredentials-f20878a4.json new file mode 100644 index 0000000000..40a0223b42 --- /dev/null +++ b/.changes/next-release/feature-EC2MetadataCredentials-f20878a4.json @@ -0,0 +1,5 @@ +{ + "type": "feature", + "category": "EC2MetadataCredentials", + "description": "Allow EC2MetadataCredentials to extend the existing expiration when EC2 Metadata Service returns expired credentials or failure response." +} \ No newline at end of file diff --git a/lib/config-base.d.ts b/lib/config-base.d.ts index 06e07e3c94..4f3afe562b 100644 --- a/lib/config-base.d.ts +++ b/lib/config-base.d.ts @@ -83,6 +83,7 @@ export interface HTTPOptions { export interface Logger { write?: (chunk: any, encoding?: string, callback?: () => void) => void log?: (...messages: any[]) => void; + warn?: (...message: any[]) => void; } export interface ParamValidation { /** diff --git a/lib/credentials/ec2_metadata_credentials.d.ts b/lib/credentials/ec2_metadata_credentials.d.ts index 2925a386e5..d94ebb6842 100644 --- a/lib/credentials/ec2_metadata_credentials.d.ts +++ b/lib/credentials/ec2_metadata_credentials.d.ts @@ -1,21 +1,31 @@ import {Credentials} from '../credentials'; +import {Logger} from '../config-base'; + export class EC2MetadataCredentials extends Credentials { - /** - * Creates credentials from the metadata service on an EC2 instance. - * @param {object} options - Override the default (1s) timeout period. - */ - constructor(options?: EC2MetadataCredentialsOptions); - } - interface EC2MetadataCredentialsOptions { - httpOptions?: { - /** - * Timeout in milliseconds. - */ - timeout?: number - /** - * Connection timeout in milliseconds. - */ - connectTimeout?: number - } - maxRetries?: number + /** + * Creates credentials from the metadata service on an EC2 instance. + * @param {object} options - Override the default (1s) timeout period. + */ + constructor(options?: EC2MetadataCredentialsOptions); + + /** + * The original expiration of the current credential. In case of AWS + * outage, the EC2 metadata will extend expiration of the existing + * credential. + */ + originalExpiration?: Date; +} +interface EC2MetadataCredentialsOptions { + httpOptions?: { + /** + * Timeout in milliseconds. + */ + timeout?: number + /** + * Connection timeout in milliseconds. + */ + connectTimeout?: number } + maxRetries?: number + logger?: Logger +} diff --git a/lib/credentials/ec2_metadata_credentials.js b/lib/credentials/ec2_metadata_credentials.js index 1f7f2e52f5..2aa2f5e28e 100644 --- a/lib/credentials/ec2_metadata_credentials.js +++ b/lib/credentials/ec2_metadata_credentials.js @@ -18,15 +18,27 @@ require('../metadata_service'); * AWS.config.credentials = new AWS.EC2MetadataCredentials({ * httpOptions: { timeout: 5000 }, // 5 second timeout * maxRetries: 10, // retry 10 times - * retryDelayOptions: { base: 200 } // see AWS.Config for information + * retryDelayOptions: { base: 200 }, // see AWS.Config for information + * logger: console // see AWS.Config for information * }); + * ``` * * If your requests are timing out in connecting to the metadata service, such * as when testing on a development machine, you can use the connectTimeout * option, specified in milliseconds, which also defaults to 1 second. - * ``` + * + * If the requests failed or returns expired credentials, it will + * extend the expiration of current credential, with a warning message. For more + * information, please go to: + * https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html + * + * @!attribute originalExpiration + * @return [Date] The optional original expiration of the current credential. + * In case of AWS outage, the EC2 metadata will extend expiration of the + * existing credential. * * @see AWS.Config.retryDelayOptions + * @see AWS.Config.logger * * @!macro nobrowser */ @@ -44,7 +56,7 @@ AWS.EC2MetadataCredentials = AWS.util.inherit(AWS.Credentials, { options.httpOptions); this.metadataService = new AWS.MetadataService(options); - this.metadata = {}; + this.logger = options.logger || AWS.config && AWS.config.logger; }, /** @@ -62,6 +74,13 @@ AWS.EC2MetadataCredentials = AWS.util.inherit(AWS.Credentials, { */ defaultMaxRetries: 3, + /** + * The original expiration of the current credential. In case of AWS + * outage, the EC2 metadata will extend expiration of the existing + * credential. + */ + originalExpiration: undefined, + /** * Loads the credentials from the instance metadata service * @@ -84,24 +103,62 @@ AWS.EC2MetadataCredentials = AWS.util.inherit(AWS.Credentials, { load: function load(callback) { var self = this; self.metadataService.loadCredentials(function(err, creds) { - if (!err) { - var currentTime = AWS.util.date.getDate(); - var expireTime = new Date(creds.Expiration); - if (expireTime < currentTime) { - err = AWS.util.error( - new Error('EC2 Instance Metadata Serivce provided expired credentials'), - { code: 'EC2MetadataCredentialsProviderFailure' } - ); + if (err) { + if (self.hasLoadedCredentials()) { + self.extendExpirationIfExpired(); + callback(); } else { - self.expired = false; - self.metadata = creds; - self.accessKeyId = creds.AccessKeyId; - self.secretAccessKey = creds.SecretAccessKey; - self.sessionToken = creds.Token; - self.expireTime = expireTime; + callback(err); } + } else { + self.setCredentials(creds); + self.extendExpirationIfExpired(); + callback(); } - callback(err); }); + }, + + /** + * Whether this credential has been loaded. + * @api private + */ + hasLoadedCredentials: function hasLoadedCredentials() { + return this.AccessKeyId && this.secretAccessKey; + }, + + /** + * if expired, extend the expiration by 15 minutes base plus a jitter of 5 + * minutes range. + * @api private + */ + extendExpirationIfExpired: function extendExpirationIfExpired() { + if (this.needsRefresh()) { + this.originalExpiration = this.originalExpiration || this.expireTime; + this.expired = false; + var nextTimeout = 15 * 60 + Math.floor(Math.random() * 5 * 60); + var currentTime = AWS.util.date.getDate().getTime(); + this.expireTime = new Date(currentTime + nextTimeout * 1000); + // TODO: add doc link; + this.logger.warn('Attempting credential expiration extension due to a ' + + 'credential service availability issue. A refresh of these ' + + 'credentials will be attempted again at ' + this.expireTime + + '\nFor more information, please visit: https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html'); + } + }, + + /** + * Update the credential with new credential responded from EC2 metadata + * service. + * @api private + */ + setCredentials: function setCredentials(creds) { + var currentTime = AWS.util.date.getDate().getTime(); + var expireTime = new Date(creds.Expiration); + this.expired = currentTime >= expireTime ? true : false; + this.metadata = creds; + this.accessKeyId = creds.AccessKeyId; + this.secretAccessKey = creds.SecretAccessKey; + this.sessionToken = creds.Token; + this.expireTime = expireTime; } }); diff --git a/test/.eslintrc b/test/.eslintrc index 4063e3bde8..5c1c356442 100644 --- a/test/.eslintrc +++ b/test/.eslintrc @@ -1,6 +1,6 @@ { "extends": "../.eslintrc", "parserOptions": { - "ecmaVersion": 8 + "ecmaVersion": 9 } } diff --git a/test/credentials.spec.js b/test/credentials.spec.js index a3b9861a2b..4cdb2ca684 100644 --- a/test/credentials.spec.js +++ b/test/credentials.spec.js @@ -1,3 +1,5 @@ +const exp = require('constants'); + // Generated by CoffeeScript 1.12.3 (function() { var AWS, STS, helpers, validateCredentials; @@ -1120,39 +1122,66 @@ }); }); }); + describe('AWS.EC2MetadataCredentials', function() { - var creds, mockMetadataService; - creds = null; + let creds = null; + let warnSpy = null; + beforeEach(function() { - return creds = new AWS.EC2MetadataCredentials({ - host: 'host' + creds = new AWS.EC2MetadataCredentials({ + host: 'host', + logger: console, }); + warnSpy = helpers.spyOn(console, 'warn'); }); - mockMetadataService = function(expireTime) { + + afterEach(function() { + AWS.config.update({ logger: null }); + }); + + const expiredEc2MetadataResponse = { + Code: 'Success', + AccessKeyId: 'KEY', + SecretAccessKey: 'SECRET', + Token: 'TOKEN', + Expiration: new Date(0) + }; + + const ONE_HOUR = 60 * 60 * 1000; + const MINUTE = 60 * 1000; + const ec2MetadataResponse = { + Code: 'Success', + AccessKeyId: 'KEY', + SecretAccessKey: 'SECRET', + Token: 'TOKEN', + Expiration: new Date(Date.now() + ONE_HOUR).toISOString() + }; + + const mockMetadataService = function(paramList) { + let invocationCount = 0; return helpers.spyOn(creds.metadataService, 'loadCredentials').andCallFake(function(cb) { - return cb(null, { - Code: 'Success', - AccessKeyId: 'KEY', - SecretAccessKey: 'SECRET', - Token: 'TOKEN', - Expiration: expireTime.toISOString() - }); + cb(paramList[invocationCount][0], paramList[invocationCount][1]); + invocationCount++; }); }; + describe('constructor', function() { it('allows passing of AWS.MetadataService options', function() { return expect(creds.metadataService.endpoint).to.equal('http://host'); }); + it('does not modify options object', function() { var opts; opts = {}; creds = new AWS.EC2MetadataCredentials(opts); return expect(opts).to.eql({}); }); + it('checking default timeout', function() { creds = new AWS.EC2MetadataCredentials({}); return expect(creds.metadataService.httpOptions.timeout).to.equal(1000); }); + it('allows setting timeout', function() { var opts; opts = { @@ -1163,11 +1192,13 @@ creds = new AWS.EC2MetadataCredentials(opts); return expect(creds.metadataService.httpOptions.timeout).to.equal(5000); }); + it('checking default connectTimeout', function() { creds = new AWS.EC2MetadataCredentials({}); return expect(creds.metadataService.httpOptions.connectTimeout).to.equal(1000); }); - return it('allows setting connectTimeout', function() { + + it('allows setting connectTimeout', function() { var opts; opts = { httpOptions: { @@ -1177,19 +1208,47 @@ creds = new AWS.EC2MetadataCredentials(opts); return expect(creds.metadataService.httpOptions.connectTimeout).to.equal(5000); }); + + it('allows setting logger', function() { + const logger = { warn: function () {} }; + creds = new AWS.EC2MetadataCredentials({ logger }); + expect(creds.logger).to.equal(logger); + }); + + it('defaults to AWS logger settings if logger is not set', function () { + const logger = { warn: function () {} }; + AWS.config.update({ logger }); + creds = new AWS.EC2MetadataCredentials(); + expect(creds.logger).to.equal(logger); + }); }); + describe('needsRefresh', function() { - return it('can be expired based on expire time from EC2 Metadata service', function(done) { - mockMetadataService(new Date(0)); + it('should expire if valid credentials passed expiration', function(done) { + mockMetadataService([ + [null, ec2MetadataResponse], + ]); creds.refresh(function () { + helpers.spyOn(AWS.util.date, 'getDate').andReturn(new Date(Date.now() + 2 * ONE_HOUR)); expect(creds.needsRefresh()).to.equal(true); done(); }); }); + + it('should NOT expire when EC2 Metadata service returns expired credentials', function(done) { + mockMetadataService([ + [null, expiredEc2MetadataResponse], + ]); + creds.refresh(function () { + expect(creds.needsRefresh()).to.equal(false); + done(); + }); + }); }); - describe('refresh', function() { + + describe('fresh', function () { it('loads credentials from EC2 Metadata service', function(done) { - mockMetadataService(new Date(AWS.util.date.getDate().getTime() + 100000)); + mockMetadataService([[null, ec2MetadataResponse]]); creds.refresh(function () { expect(creds.metadata.Code).to.equal('Success'); expect(creds.accessKeyId).to.equal('KEY'); @@ -1199,33 +1258,100 @@ done(); }); }); + it('does try to load creds second time if Metadata service failed', function() { - var spy; - spy = helpers.spyOn(creds.metadataService, 'loadCredentials').andCallFake(function(cb) { - return cb(new Error('INVALID SERVICE')); + const spy = helpers.spyOn(creds.metadataService, 'loadCredentials').andCallFake(function(cb) { + cb(new Error('INVALID SERVICE')); }); creds.refresh(function(err) { - return expect(err.message).to.equal('INVALID SERVICE'); + expect(err.message).to.equal('INVALID SERVICE'); }); - return creds.refresh(function() { - return creds.refresh(function() { - return creds.refresh(function() { - return expect(spy.calls.length).to.equal(4); + creds.refresh(function() { + creds.refresh(function() { + creds.refresh(function() { + expect(spy.calls.length).to.equal(4); }); }); }); }); - it('fails if the loaded credentials are expired', function (done) { - mockMetadataService(new Date(Date.now() - 1)); - creds.refresh(function (err) { - expect(err).to.be.not.null; - expect(err.message).to.equal('EC2 Instance Metadata Serivce provided expired credentials'); - expect(err.code).to.equal('EC2MetadataCredentialsProviderFailure'); + + it('should throw if EC2 Metadata service failed at 1st load', function (done) { + mockMetadataService([[new Error('Cannot load initial credentials'), null]]); + creds.refresh(function(err) { + expect(err.message).to.equal('Cannot load initial credentials'); done(); }); }); + + it('should extend expiration if EC2 Metadata service returns expired credentials', function (done) { + helpers.spyOn(Math, 'random').andReturn(0.5); + helpers.spyOn(AWS.util.date, 'getDate').andCallThrough(); + mockMetadataService([ + [null, ec2MetadataResponse], + [null, { ...expiredEc2MetadataResponse, AccessKeyId: 'KEY2' }], + [null, { ...expiredEc2MetadataResponse, AccessKeyId: 'KEY3' }], + [null, { ...expiredEc2MetadataResponse, AccessKeyId: 'KEY4' }] + ]); + creds.refresh(function() { + // Loaded valid initial credentials + expect(creds.metadata.Code).to.equal('Success'); + expect(creds.needsRefresh()).to.equal(false); + expect(warnSpy.calls.length).to.equal(0); + creds.refresh(function() { + // Extend expiration of credentials if expired credentials are loaded. + expect(creds.needsRefresh()).to.equal(false); + expect(creds.metadata.Code).to.equal('Success'); + expect(creds.accessKeyId).to.equal('KEY2'); + const timeTillExpiration = creds.expireTime.getTime() - AWS.util.date.getDate().getTime(); + expect(timeTillExpiration).to.be.greaterThan(15 * MINUTE); + expect(timeTillExpiration).to.be.lessThan(20 * MINUTE); + expect(warnSpy.calls.length).to.equal(1); + expect(warnSpy.calls[0].arguments[0]).to.contain('Attempting credential expiration extension'); + creds.refresh(function() { + // Refresh the credentials and print warning message if force refresh the credentials + expect(creds.needsRefresh()).to.equal(false); + expect(creds.accessKeyId).to.equal('KEY3'); + expect(warnSpy.calls.length).to.equal(2); + + // Print warning message if extended expiration is extended. + helpers.spyOn(AWS.util.date, 'getDate').andReturn(new Date(Date.now() + 2 * ONE_HOUR)); + expect(creds.needsRefresh()).to.equal(true); + creds.refresh(function() { + expect(creds.needsRefresh()).to.equal(false); + expect(creds.accessKeyId).to.equal('KEY4'); + expect(warnSpy.calls.length).to.equal(3); + done(); + }); + }); + }); + }); + }); + + it('should extend expiration if EC2 Metadata service failed', function (done) { + mockMetadataService([ + [null, ec2MetadataResponse], + [new Error('Metadata Service Error'), null] + ]); + creds.refresh(function () { + expect(creds.metadata.Code).to.equal('Success'); + expect(creds.accessKeyId).to.equal('KEY'); + expect(creds.needsRefresh()).to.equal(false); + + helpers.spyOn(AWS.util.date, 'getDate').andReturn(new Date(Date.now() + 2 * ONE_HOUR)); + expect(creds.needsRefresh()).to.equal(true); + creds.refresh(function() { + expect(creds.metadata.Code).to.equal('Success'); + expect(creds.accessKeyId).to.equal('KEY'); + expect(creds.needsRefresh()).to.equal(false); + expect(warnSpy.calls.length).to.equal(1); + expect(warnSpy.calls[0].arguments[0]).to.contain('SDK cannot renew the credential'); + }); + }); + done(); + }); }); }); + describe('AWS.RemoteCredentials', function() { var origEnv, creds, mockEndpoint, responseData, responseDataNew; creds = null;