Merge branch 'aws-amplify:dev' into fix/global-preifx

.circleci/config.yml

@@ -19,7 +19,7 @@ parameters:
     default: true
 
 executors:
-  linux: &linux-e2e-executor
+  l_large: &linux-e2e-executor-large
     docker:
       - image: public.ecr.aws/j4f5f3h7/amplify-cli-e2e-base-image-repo-public:latest
     working_directory: ~/repo
@@ -31,7 +31,7 @@ executors:
 # our defined job, and its steps
 jobs:
   setup:
-    executor: 'linux'
+    executor: 'l_large'
     steps:
       - checkout # checkout code
       - run: # run a command
@@ -46,7 +46,7 @@ jobs:
     parameters:
       os:
         type: executor
-        default: linux
+        default: l_large
     executor: << parameters.os >>
     steps:
       - checkout
@@ -126,7 +126,7 @@ jobs:
     parameters:
       os:
         type: executor
-        default: linux
+        default: l_large
     executor: << parameters.os >>
     steps:
       - attach_workspace:

.circleci/local_publish_helpers.sh

@@ -21,11 +21,6 @@ function uploadPkgCli {
     export version=$(./amplify-pkg-linux-x64 --version)
 
     if [[ "$CIRCLE_BRANCH" == "release" ]] || [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^release_rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then
-        tar -czvf amplify-pkg-linux-arm64.tgz amplify-pkg-linux-arm64
-        tar -czvf amplify-pkg-linux-x64.tgz amplify-pkg-linux-x64
-        tar -czvf amplify-pkg-macos-x64.tgz amplify-pkg-macos-x64
-        tar -czvf amplify-pkg-win-x64.tgz amplify-pkg-win-x64.exe
-
         aws --profile=s3-uploader s3 cp amplify-pkg-win-x64.tgz s3://aws-amplify-cli-do-not-delete/$(echo $version)/amplify-pkg-win-x64-$(echo $hash).tgz
         aws --profile=s3-uploader s3 cp amplify-pkg-macos-x64.tgz s3://aws-amplify-cli-do-not-delete/$(echo $version)/amplify-pkg-macos-x64-$(echo $hash).tgz
         aws --profile=s3-uploader s3 cp amplify-pkg-linux-arm64.tgz s3://aws-amplify-cli-do-not-delete/$(echo $version)/amplify-pkg-linux-arm64-$(echo $hash).tgz
@@ -50,7 +45,6 @@ function uploadPkgCli {
         aws --profile=s3-uploader s3 cp amplify-pkg-linux-x64.tgz s3://aws-amplify-cli-do-not-delete/$(echo $version)/amplify-pkg-linux-x64.tgz
 
     else
-        tar -czvf amplify-pkg-linux-x64.tgz amplify-pkg-linux-x64
         aws --profile=s3-uploader s3 cp amplify-pkg-linux-x64.tgz s3://aws-amplify-cli-do-not-delete/$(echo $version)/amplify-pkg-linux-x64-$(echo $hash).tgz
     fi
 
@@ -78,23 +72,65 @@ function generatePkgCli {
 
   # Build pkg cli
   cp package.json ../build/node_modules/package.json
-  if [[ "$CIRCLE_BRANCH" == "release" ]] || [[ "$CIRCLE_BRANCH" =~ ^run-e2e-with-rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^release_rc\/.* ]] || [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then
-    # This will generate a file our arm64 binary
+
+  if [[ "$@" =~ 'arm' ]]; then
     npx pkg --no-bytecode --public-packages "*" --public -t node14-linux-arm64 ../build/node_modules -o ../out/amplify-pkg-linux-arm64
-    # This will generate files for our x64 binaries.
-    npx pkg -t node14-macos-x64 ../build/node_modules -o ../out/amplify-pkg-macos-x64
+    tar -czvf ../out/amplify-pkg-linux-arm64.tgz ../out/amplify-pkg-linux-arm64
+  fi
+
+  if [[ "$@" =~ 'linux' ]]; then
     npx pkg -t node14-linux-x64 ../build/node_modules -o ../out/amplify-pkg-linux-x64
-    npx pkg -t node14-win-x64 ../build/node_modules -o ../out/amplify-pkg-win-x64.exe
-  else
-    # This will generate files for our x64 binaries.
+    tar -czvf ../out/amplify-pkg-linux-x64.tgz ../out/amplify-pkg-linux-x64
+  fi
+
+  if [[ "$@" =~ 'macos' ]]; then
     npx pkg -t node14-macos-x64 ../build/node_modules -o ../out/amplify-pkg-macos-x64
-    npx pkg -t node14-linux-x64 ../build/node_modules -o ../out/amplify-pkg-linux-x64
-    npx pkg -t node14-win-x64 ../build/node_modules -o ../out/amplify-pkg-win-x64.exe
+    tar -czvf ../out/amplify-pkg-macos-x64.tgz ../out/amplify-pkg-macos-x64
   fi
 
+  if [[ "$@" =~ 'win' ]]; then
+    npx pkg -t node14-win-x64 ../build/node_modules -o ../out/amplify-pkg-win-x64.exe
+    tar -czvf ../out/amplify-pkg-win-x64.tgz ../out/amplify-pkg-win-x64.exe
+  fi
 
   cd ..
 }
+
+function verifyPkgCli {
+    echo "Human readable sizes"
+    du -h out/*
+    echo "Sizes in bytes"
+    wc -c out/*
+
+    linux_compressed_binary_threshold_in_bytes=$((200 * 1024 * 1024))
+    linux_compressed_binary_size=$(wc -c out/amplify-pkg-linux-x64.tgz | awk '{print $1}')
+    if (( linux_compressed_binary_size > linux_compressed_binary_threshold_in_bytes )); then
+      echo "Linux compressed binary size has grown over $linux_compressed_binary_threshold_in_bytes bytes"
+      exit 1
+    fi
+
+    macos_compressed_binary_threshold_in_bytes=$((200 * 1024 * 1024))
+    macos_compressed_binary_size=$(wc -c out/amplify-pkg-macos-x64.tgz | awk '{print $1}')
+    if (( macos_compressed_binary_size > macos_compressed_binary_threshold_in_bytes )); then
+      echo "MacOS compressed binary size has grown over $macos_compressed_binary_threshold_in_bytes bytes"
+      exit 1
+    fi
+
+    win_compressed_binary_threshold_in_bytes=$((200 * 1024 * 1024))
+    win_compressed_binary_size=$(wc -c out/amplify-pkg-win-x64.tgz | awk '{print $1}')
+    if (( win_compressed_binary_size > win_compressed_binary_threshold_in_bytes )); then
+      echo "Windows compressed binary size has grown over $win_compressed_binary_threshold_in_bytes bytes"
+      exit 1
+    fi
+
+    arm_compressed_binary_threshold_in_bytes=$((150 * 1024 * 1024))
+    arm_compressed_binary_size=$(wc -c out/amplify-pkg-linux-arm64.tgz | awk '{print $1}')
+    if (( arm_compressed_binary_size > arm_compressed_binary_threshold_in_bytes )); then
+      echo "Windows compressed binary size has grown over $arm_compressed_binary_threshold_in_bytes bytes"
+      exit 1
+    fi
+}
+
 function unsetNpmRegistryUrl {
     # Restore the original NPM and Yarn registry URLs
     npm set registry "https://registry.npmjs.org/"
@@ -208,6 +244,8 @@ function setAwsAccountCredentials {
     export AWS_ACCESS_KEY_ID_ORIG=$AWS_ACCESS_KEY_ID
     export AWS_SECRET_ACCESS_KEY_ORIG=$AWS_SECRET_ACCESS_KEY
     export AWS_SESSION_TOKEN_ORIG=$AWS_SESSION_TOKEN
+    # introduce a delay of up to 5 minutes to allow for more even spread aws list-accounts calls due to throttling
+    sleep $[ ( $RANDOM % 300 )  + 1 ]s
     if [[ "$OSTYPE" == "msys" ]]; then
         # windows provided by circleci has this OSTYPE
         useChildAccountCredentials
@@ -233,8 +271,9 @@ function runE2eTest {
     if [ -f  $FAILED_TEST_REGEX_FILE ]; then
         # read the content of failed tests
         failedTests=$(<$FAILED_TEST_REGEX_FILE)
-        yarn run e2e --detectOpenHandles --maxWorkers=3 $TEST_SUITE -t "$failedTests"
+        # adding --force-exit per https://github.com/facebook/jest/issues/9473
+        yarn run e2e --force-exit --detectOpenHandles --maxWorkers=3 $TEST_SUITE -t "$failedTests"
     else
-        yarn run e2e --detectOpenHandles --maxWorkers=3 $TEST_SUITE
+        yarn run e2e --force-exit --detectOpenHandles --maxWorkers=3 $TEST_SUITE
     fi
 }

.circleci/publish.sh

@@ -1,6 +1,27 @@
 #!/bin/bash -e
-git config --global user.email $GITHUB_EMAIL
-git config --global user.name $GITHUB_USER
+
+if [ -z "$GITHUB_EMAIL" ]; then
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    git config --global user.email not@used.com
+  else
+    echo "GITHUB_EMAIL email is missing"
+    exit 1
+  fi
+else
+  git config --global user.email $GITHUB_EMAIL
+fi
+
+if [ -z "$GITHUB_USER" ]; then
+  if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then
+    git config --global user.name "Doesnt Matter"
+  else
+    echo "GITHUB_USER email is missing"
+    exit 1
+  fi
+else
+  git config --global user.name $GITHUB_USER
+fi
+
 if [[ "$CIRCLE_BRANCH" =~ ^tagged-release ]]; then
   if [[ "$CIRCLE_BRANCH" =~ ^tagged-release-without-e2e-tests\/.* ]]; then
     # Remove tagged-release-without-e2e-tests/

.circleci/scan_artifacts.ts

@@ -1,12 +1,22 @@
 import * as execa from 'execa';
 import * as path from 'path';
-const DEFAULT_SEARCH_FOLDER = path.normalize(path.join(__dirname, '..'));
+
+// This list contains a platform agnostic list of paths where artifacts are stored after each test.
+import { ARTIFACT_STORAGE_PATH_ALLOW_LIST } from '../scripts/artifact-storage-path-allow-list';
+
+// ROOT_FOLDER_ABSOLUTE_PATH resolves to the absolute value of ~
+// This is because of the location of this file in ~/repo/.circleci/scan_artifacts
+const ROOT_FOLDER_ABSOLUTE_PATH = path.normalize(path.join(__dirname, '..', '..'));
+
+
 export const hasMatchingContentInFolder = (
   patterns: string[],
-  folder = DEFAULT_SEARCH_FOLDER,
-  excludeFolder = '{*/node_modules,.git}',
+  folder: string,
+  excludeFolder = '{node_modules,.cache,.git,\.cache,\.git}',
 ): boolean => {
+  console.log("Scanning folder:", folder);
   const patternParam = patterns.reduce<string[]>((acc, v) => [...acc, '-e', v], []);
+
   try {
     execa.sync('grep', ['-r', `--exclude-dir=${excludeFolder}`, ...patternParam, folder]);
     return true;
@@ -15,6 +25,10 @@ export const hasMatchingContentInFolder = (
     if (e.exitCode === 1) {
       return false;
     }
+    if (e.message.includes('No such file or directory')){
+      console.log("No artifacts found at:", folder);
+      return false;
+    }
     throw new Error('Scanning artifacts failed');
   }
 };
@@ -23,10 +37,19 @@ const main = () => {
   const envVarNameWithCredentialValues = (process.env.ENV_VAR_WITH_SECRETS || '').split(',').map(v => v.trim());
   const values = envVarNameWithCredentialValues.map(v => process.env[v]).filter(Boolean);
   if (values.length) {
-    const hasContent = hasMatchingContentInFolder(values);
-    if (hasContent) {
-      console.log('Scanning artifact has found secret value. Failing the build');
-      process.exit(1);
+    for(let folder of ARTIFACT_STORAGE_PATH_ALLOW_LIST){
+      if (folder.startsWith("~/")) {
+        const normalizedFolder = path.normalize(folder.replace("~", ROOT_FOLDER_ABSOLUTE_PATH));
+        const hasContent = hasMatchingContentInFolder(values, normalizedFolder);
+        if (hasContent) {
+          console.log('Scanning artifact has found secret value. Failing the build: ', normalizedFolder);
+          process.exit(1);
+        }
+      } else {
+        console.log('Paths in ARTIFACT_STORAGE_PATH_ALLOW_LIST must start with ~/');
+        console.log('Update the path to use ~/ and make sure to do the same in the config.yaml/config.base.yaml files');
+        process.exit(1);
+      }
     }
   }
 };

CONTRIBUTING.md

@@ -102,7 +102,7 @@ and print out errors that need manual attention.
 
 Pull requests are welcome!
 
-You should open an issue to discuss your pull request, unless it's a trivial change. It's best to ensure that your proposed change would be accepted so that you don't waste your own time. If you would like to implement support for a significant feature that is not yet available, please talk to us beforehand to avoid any duplication of effort.
+You should open an issue to discuss your pull request, unless it's a trivial change. It's best to ensure that your proposed change would be accepted so that you don't waste your own time. If you would like to implement support for a significant feature that is not yet available, please talk to us beforehand to avoid any duplication of effort. Additionally, please be mindful of the length of the pull request - if your change requires more than 12 file changes, consider breaking the change down into smaller, non-dependent changes. This includes any changes that may be added as a result of the linter.
 
 Pull requests should be opened against **_dev_**.
 
@@ -135,7 +135,6 @@ When filing a bug, please try to be as detailed as possible. In addition to the
 - Any modifications you've made relevant to the bug
 - Anything unusual about your environment or deployment
 
-
 Guidelines for bug reports:
 
 - Check to see if a [duplicate or closed issue](https://github.com/aws-amplify/amplify-cli/issues?q=is%3Aissue+) already exists!
@@ -144,10 +143,8 @@ Guidelines for bug reports:
 - Format any code snippets using [Markdown](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) syntax
 - If you're not using the latest version of the CLI, see if the issue still persists after upgrading - this helps to isolate regressions!
 
-
 Finally, thank you for taking the time to read this, and taking the time to write a good bug report.
 
-
 ## Commits
 
 Commit messages should follow the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/#summary) specification. For example:
@@ -165,6 +162,7 @@ Valid commit types are as follows:
 - `refactor`
 - `style`
 - `test`
+
 ### Git Hooks
 
 You will notice the extra actions carried out when you run the `git commit` or `git push` commands on this monorepo, that's because the following git hooks are configured using [husky](https://github.com/typicode/husky/tree/main) (you can see them in the root [package.json](https://github.com/aws-amplify/amplify-cli/blob/f2ac2b27b6b0dbf0c52edbc696c35b71f539c944/package.json#L61) file):
@@ -173,7 +171,7 @@ You will notice the extra actions carried out when you run the `git commit` or `
 "husky": {
     "hooks": {
         "commit-msg": "commitlint -E HUSKY_GIT_PARAMS",
-        "pre-push": "yarn build-tests-changed && yarn split-e2e-tests",
+        "pre-push": "yarn verify-api-extract && yarn build-tests-changed && yarn split-e2e-tests",
         "pre-commit": "yarn verify-commit"
     }
 }
@@ -192,6 +190,7 @@ The "pre-commit" hook runs the [verify-commit](https://github.com/aws-amplify/am
 #### "pre-push" hook:
 
 The "pre-push" hook will build test files and run the  `split-e2e-tests` script to ensure the correct configuration file is generated for our CICD workflow.
+
 ## Tests
 
 Please ensure that your change still passes unit tests, and ideally integration/UI tests. It's OK if you're still working on tests at the time that you submit, but be prepared to be asked about them. Wherever possible, pull requests should contain tests as appropriate. Bugfixes should contain tests that exercise the corrected behavior (i.e., the test should fail without the bugfix and pass with it), and new features should be accompanied by tests exercising the feature.

nx.json

@@ -10,31 +10,44 @@
           "test",
           "lint",
           "package",
-          "prepare"
+          "prepare",
+          "extract-api"
         ]
       }
     }
   },
-  "targetDependencies": {
-    "build": [
-      {
-        "target": "build",
-        "projects": "dependencies"
-      }
-    ],
-    "prepare": [
-      {
-        "target": "prepare",
-        "projects": "dependencies"
-      }
-    ],
-    "package": [
-      {
-        "target": "package",
-        "projects": "dependencies"
-      }
+  "namedInputs": {
+    "default": [
+      "{projectRoot}/**/*",
+      "!{projectRoot}/**/*.md"
     ]
   },
+  "targetDefaults": {
+    "build": {
+      "inputs": [ "default", "^default" ],
+      "dependsOn": [ "^build" ]
+    },
+    "prepare": {
+      "inputs": [ "default", "^default" ],
+      "dependsOn": [ "^prepare" ]
+    },
+    "package": {
+      "inputs": [ "default", "^default" ],
+      "dependsOn": [ "^package" ]
+    },
+    "test": {
+      "inputs": [ "default", "^default" ],
+      "dependsOn": [ "build" ]
+    },
+    "extract-api": {
+      "dependsOn": [ "build" ],
+      "inputs": [
+        "{projectRoot}/src/**/*",
+        "!{projectRoot}/src/__tests__/**/*"
+      ],
+      "outputs": [ "{projectRoot}/API.md" ]
+    }
+  },
   "affected": {
     "defaultBase": "dev"
   },

package.json

@@ -46,7 +46,9 @@
     "release-rc": "./scripts/release-rc.sh",
     "promote-rc": "./scripts/promote-rc.sh",
     "finish-release": "./scripts/finish-release.sh",
-    "update-data-packages": "./scripts/update-data-dependencies.sh && yarn"
+    "update-data-packages": "./scripts/update-data-dependencies.sh && yarn",
+    "extract-api": "nx run-many --target=extract-api --all",
+    "verify-api-extract": "yarn extract-api && ./scripts/verify_extract_api.sh"
   },
   "bugs": {
     "url": "https://github.com/aws-amplify/amplify-cli/issues"
@@ -61,7 +63,7 @@
   "husky": {
     "hooks": {
       "commit-msg": "commitlint -E HUSKY_GIT_PARAMS",
-      "pre-push": "yarn build-tests-changed && yarn split-e2e-tests",
+      "pre-push": "yarn verify-api-extract && yarn build-tests-changed && yarn split-e2e-tests",
       "pre-commit": "yarn verify-commit"
     }
   },
@@ -91,6 +93,7 @@
     "@commitlint/config-lerna-scopes": "^16.2.2",
     "@commitlint/cz-commitlint": "^16.2.3",
     "@commitlint/prompt": "^16.2.3",
+    "@microsoft/api-extractor": "^7.33.5",
     "@types/glob": "^7.1.1",
     "@types/jest": "^26.0.20",
     "@types/js-yaml": "^4.0.0",
@@ -137,4 +140,4 @@
     "**/nth-check": "^2.0.1",
     "**/undici": "^5.8.0"
   }
-}
+}