From 6f0c6c7553f7b1a79276aaf42f0d11fd2ce3e8f4 Mon Sep 17 00:00:00 2001 From: Raj Rajhans Date: Sun, 11 Apr 2021 04:04:26 +0530 Subject: [PATCH 01/35] fix(amplify-provider-awscloudformation): fix tests failing due to system-config-manager.js (#7053) --- .../src/configuration-manager.ts | 2 +- ...ig-manager.js => system-config-manager.ts} | 24 ++++++------------- 2 files changed, 8 insertions(+), 18 deletions(-) rename packages/amplify-provider-awscloudformation/src/{system-config-manager.js => system-config-manager.ts} (95%) diff --git a/packages/amplify-provider-awscloudformation/src/configuration-manager.ts b/packages/amplify-provider-awscloudformation/src/configuration-manager.ts index 4fc145249c1..da9dad4961a 100644 --- a/packages/amplify-provider-awscloudformation/src/configuration-manager.ts +++ b/packages/amplify-provider-awscloudformation/src/configuration-manager.ts @@ -9,7 +9,7 @@ import awsRegions from './aws-regions'; import constants from './constants'; import setupNewUser from './setup-new-user'; import obfuscateUtil from './utility-obfuscate'; -import systemConfigManager from './system-config-manager'; +import * as systemConfigManager from './system-config-manager'; import { doAdminTokensExist, getTempCredsWithAdminTokens, isAmplifyAdminApp } from './utils/admin-helpers'; import { resolveAppId } from './utils/resolve-appId'; import { AuthFlow, AuthFlowConfig, AwsSdkConfig } from './utils/auth-types'; diff --git a/packages/amplify-provider-awscloudformation/src/system-config-manager.js b/packages/amplify-provider-awscloudformation/src/system-config-manager.ts similarity index 95% rename from packages/amplify-provider-awscloudformation/src/system-config-manager.js rename to packages/amplify-provider-awscloudformation/src/system-config-manager.ts index 4c36cd7e99c..65ab061ca44 100644 --- a/packages/amplify-provider-awscloudformation/src/system-config-manager.js +++ b/packages/amplify-provider-awscloudformation/src/system-config-manager.ts @@ -12,7 +12,7 @@ const logger = fileLogger('system-config-manager'); const credentialsFilePath = pathManager.getAWSCredentialsFilePath(); const configFilePath = pathManager.getAWSConfigFilePath(); -function setProfile(awsConfig, profileName) { +export function setProfile(awsConfig, profileName) { fs.ensureDirSync(pathManager.getDotAWSDirPath()); let credentials = {}; @@ -64,7 +64,7 @@ function setProfile(awsConfig, profileName) { fs.writeFileSync(configFilePath, ini.stringify(config), { mode: SecretFileMode }); } -async function getProfiledAwsConfig(context, profileName, isRoleSourceProfile) { +export async function getProfiledAwsConfig(context, profileName, isRoleSourceProfile?) { let awsConfig; const httpProxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY; const profileConfig = getProfileConfig(profileName); @@ -218,15 +218,14 @@ function isCredentialsExpired(roleCredentials) { if (roleCredentials && roleCredentials.expiration) { const TOTAL_MILLISECONDS_IN_ONE_MINUTE = 1000 * 60; - const now = new Date(); const expirationDate = new Date(roleCredentials.expiration); - isExpired = expirationDate - now < TOTAL_MILLISECONDS_IN_ONE_MINUTE; + isExpired = expirationDate.getTime() - Date.now() < TOTAL_MILLISECONDS_IN_ONE_MINUTE; } return isExpired; } -async function resetCache(context, profileName) { +export async function resetCache(context, profileName) { let awsConfig; const profileConfig = getProfileConfig(profileName); const cacheFilePath = getCacheFilePath(context); @@ -271,7 +270,7 @@ function getProfileConfig(profileName) { return normalizeKeys(profileConfig); } -function getProfileCredentials(profileName) { +export function getProfileCredentials(profileName) { let profileCredentials; logger('getProfileCredentials', [profileName])(); if (fs.existsSync(credentialsFilePath)) { @@ -317,7 +316,7 @@ function normalizeKeys(config) { return config; } -function getProfileRegion(profileName) { +export function getProfileRegion(profileName) { let profileRegion; logger('getProfileRegion', [profileName])(); const profileConfig = getProfileConfig(profileName); @@ -329,7 +328,7 @@ function getProfileRegion(profileName) { return profileRegion; } -function getNamedProfiles() { +export function getNamedProfiles() { let namedProfiles; if (fs.existsSync(configFilePath)) { const config = ini.parse(fs.readFileSync(configFilePath, 'utf-8')); @@ -343,12 +342,3 @@ function getNamedProfiles() { } return namedProfiles; } - -module.exports = { - setProfile, - getProfiledAwsConfig, - getProfileCredentials, - getProfileRegion, - getNamedProfiles, - resetCache, -}; From 7228ddbe2921e3aead8f21a2676cf295e6351a4c Mon Sep 17 00:00:00 2001 From: Edward Foyle Date: Wed, 7 Apr 2021 16:28:44 -0700 Subject: [PATCH 02/35] feat: s3 sse by default --- packages/amplify-cli-core/package.json | 2 + packages/amplify-cli-core/src/cfnUtilities.ts | 176 +++++++++++++++++ packages/amplify-cli-core/src/index.ts | 1 + packages/amplify-cli/package.json | 3 +- .../src/project-config-version-check.ts | 186 ++---------------- .../src/aws-utils/aws-cfn.js | 6 +- .../src/initializer.js | 12 +- .../cfn-pre-processor.ts | 33 ++++ .../modifiers/s3-sse-modifier.ts | 21 ++ .../pre-push-cfn-modifier.ts | 29 +++ .../src/push-resources.ts | 23 +-- 11 files changed, 298 insertions(+), 194 deletions(-) create mode 100644 packages/amplify-cli-core/src/cfnUtilities.ts create mode 100644 packages/amplify-provider-awscloudformation/src/pre-push-cfn-processor/cfn-pre-processor.ts create mode 100644 packages/amplify-provider-awscloudformation/src/pre-push-cfn-processor/modifiers/s3-sse-modifier.ts create mode 100644 packages/amplify-provider-awscloudformation/src/pre-push-cfn-processor/pre-push-cfn-modifier.ts diff --git a/packages/amplify-cli-core/package.json b/packages/amplify-cli-core/package.json index cc1356af0de..c2011326bca 100644 --- a/packages/amplify-cli-core/package.json +++ b/packages/amplify-cli-core/package.json @@ -27,9 +27,11 @@ "ajv": "^6.12.3", "amplify-cli-logger": "1.1.0", "ci-info": "^2.0.0", + "cloudform-types": "^4.2.0", "dotenv": "^8.2.0", "fs-extra": "^8.1.0", "hjson": "^3.2.1", + "js-yaml": "^4.0.0", "lodash": "^4.17.19", "open": "^7.3.1" }, diff --git a/packages/amplify-cli-core/src/cfnUtilities.ts b/packages/amplify-cli-core/src/cfnUtilities.ts new file mode 100644 index 00000000000..0df88f57399 --- /dev/null +++ b/packages/amplify-cli-core/src/cfnUtilities.ts @@ -0,0 +1,176 @@ +import * as yaml from 'js-yaml'; +import { Template } from 'cloudform-types'; +import { JSONUtilities } from './jsonUtilities'; +import * as fs from 'fs-extra'; + +// Register custom tags for yaml parser +const CF_SCHEMA = new yaml.Schema([ + new yaml.Type('!Ref', { + kind: 'scalar', + construct: function (data) { + return { Ref: data }; + }, + }), + new yaml.Type('!Condition', { + kind: 'sequence', + construct: function (data) { + return { Condition: data }; + }, + }), + new yaml.Type('!Equals', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Equals': data }; + }, + }), + new yaml.Type('!Not', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Not': data }; + }, + }), + new yaml.Type('!Sub', { + kind: 'scalar', + construct: function (data) { + return { 'Fn::Sub': data }; + }, + }), + new yaml.Type('!Sub', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Sub': data }; + }, + }), + new yaml.Type('!If', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::If': data }; + }, + }), + new yaml.Type('!Join', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Join': data }; + }, + }), + new yaml.Type('!Select', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Select': data }; + }, + }), + new yaml.Type('!FindInMap', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::FindInMap': data }; + }, + }), + new yaml.Type('!GetAtt', { + kind: 'scalar', + construct: function (data) { + return { 'Fn::GetAtt': Array.isArray(data) ? data : data.split('.') }; + }, + }), + new yaml.Type('!GetAtt', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::GetAtt': Array.isArray(data) ? data : data.split('.') }; + }, + }), + new yaml.Type('!GetAZs', { + kind: 'scalar', + construct: function (data) { + return { 'Fn::GetAZs': data }; + }, + }), + new yaml.Type('!Base64', { + kind: 'mapping', + construct: function (data) { + return { 'Fn::Base64': data }; + }, + }), + new yaml.Type('!Split', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Split': data }; + }, + }), + new yaml.Type('!Cidr', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Cidr': data }; + }, + }), + new yaml.Type('!ImportValue', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::ImportValue': data }; + }, + }), + new yaml.Type('!Transform', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Transform': data }; + }, + }), + new yaml.Type('!And', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::And': data }; + }, + }), + new yaml.Type('!Or', { + kind: 'sequence', + construct: function (data) { + return { 'Fn::Or': data }; + }, + }), +]); + +function isJsonFileContent(fileContent: string): boolean { + // We use the first character to determine if the content is json or yaml because historically the CLI could + // have emitted JSON with YML extension, so we can't rely on filename extension. + return fileContent?.trim()[0] === '{'; // CFN templates are always objects, never arrays +} + +export async function readCFNTemplate(filePath: string): Promise<{ templateFormat: CFNTemplateFormat; cfnTemplate: Template }> { + if (!fs.existsSync(filePath) || !fs.statSync(filePath).isFile) { + throw new Error(`No CloudFormation template found at ${filePath}`); + } + const fileContent = await fs.readFile(filePath, 'utf8'); + // We use the first character to determine if the content is json or yaml because historically the CLI could + // have emitted JSON with YML extension, so we can't rely on filename extension. + const isJson = isJsonFileContent(fileContent); + const cfnTemplate = isJson ? JSONUtilities.parse