From a0bfe8a090f0fb32e434cecb0c5ba19554740594 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Mon, 26 Aug 2024 10:28:49 -0600 Subject: [PATCH 01/32] feat: Offer self-service IDE based on VSCode (#1044) --- .github/workflows/module-test.yaml | 11 - .github/workflows/pr.yaml | 13 + .github/workflows/test-cleanup.yaml | 22 - Makefile | 4 +- cluster/terraform/eks.tf | 9 +- docs/reviewer_checklist.md | 6 +- hack/build-ide-cfn.sh | 23 + hack/create-infrastructure.sh | 24 + hack/deploy-ide-cfn.sh | 14 + hack/destroy-infrastructure.sh | 24 + hack/exec.sh | 6 +- hack/find-dangling-resources.sh | 12 + hack/lib/common-env.sh | 9 + hack/lib/generate-aws-creds.sh | 28 +- hack/run-tests.sh | 14 +- hack/update-iam-role.sh | 23 + hack/validate-terraform.sh | 39 ++ lab/Dockerfile | 4 +- lab/bin/reset-environment | 4 +- lab/cfn/eks-workshop-vscode-cfn.yaml | 578 ++++++++++++++++++ lab/iam/iam-role-cfn.yaml | 64 ++ lab/iam/policies/base.yaml | 87 +++ lab/iam/policies/ec2.yaml | 92 +++ lab/iam/policies/iam.yaml | 75 +++ lab/iam/policies/labs1.yaml | 149 +++++ lab/iam/policies/labs2.yaml | 116 ++++ lab/scripts/entrypoint.sh | 5 +- lab/scripts/setup.sh | 8 + manifests/.workshop/terraform/base.tf | 10 + .../inferentia/.workshop/terraform/main.tf | 25 +- .../ack/.workshop/terraform/main.tf | 8 +- .../crossplane/.workshop/terraform/main.tf | 7 +- .../gitops/flux/.workshop/terraform/main.tf | 4 +- .../.workshop/terraform/main.tf | 8 +- .../karpenter/.workshop/terraform/main.tf | 8 + .../keda/.workshop/terraform/main.tf | 4 + .../ingress/.workshop/terraform/main.tf | 7 +- .../load-balancer/.workshop/terraform/main.tf | 7 +- .../fundamentals/fargate/profile/fargate.yaml | 18 - .../storage/ebs/.workshop/terraform/main.tf | 3 +- .../storage/efs/.workshop/terraform/main.tf | 3 +- .../custom-networking/.workshop/cleanup.sh | 4 + .../.workshop/terraform/main.tf | 4 + .../vpc-lattice/.workshop/terraform/main.tf | 4 +- .../.workshop/terraform/main.tf | 4 +- .../kubecost/.workshop/terraform/main.tf | 7 +- .../oss-metrics/.workshop/terraform/main.tf | 7 +- .../.workshop/terraform/main.tf | 4 +- .../security/irsa/.workshop/terraform/main.tf | 4 +- .../.workshop/terraform/main.tf | 3 +- website/docs/fundamentals/fargate/enabling.md | 20 +- .../assets/vscode-copy-paste.webp | Bin 0 -> 9378 bytes website/docs/introduction/navigating-labs.md | 15 +- .../your-account/assets/vscode-outputs.webp | Bin 0 -> 17148 bytes .../assets/vscode-password-retrieve.webp | Bin 0 -> 3996 bytes .../assets/vscode-password-visible.webp | Bin 0 -> 5172 bytes .../your-account/assets/vscode-password.webp | Bin 0 -> 5696 bytes .../your-account/assets/vscode-splash.webp | Bin 0 -> 5074 bytes .../setup/your-account/cleanup.md | 2 +- .../introduction/setup/your-account/index.md | 76 ++- website/src/components/Terminal/index.tsx | 44 +- 61 files changed, 1633 insertions(+), 140 deletions(-) create mode 100644 hack/build-ide-cfn.sh create mode 100644 hack/create-infrastructure.sh create mode 100644 hack/deploy-ide-cfn.sh create mode 100644 hack/destroy-infrastructure.sh create mode 100644 hack/find-dangling-resources.sh create mode 100644 hack/update-iam-role.sh create mode 100644 hack/validate-terraform.sh create mode 100644 lab/cfn/eks-workshop-vscode-cfn.yaml create mode 100644 lab/iam/iam-role-cfn.yaml create mode 100644 lab/iam/policies/base.yaml create mode 100644 lab/iam/policies/ec2.yaml create mode 100644 lab/iam/policies/iam.yaml create mode 100644 lab/iam/policies/labs1.yaml create mode 100644 lab/iam/policies/labs2.yaml delete mode 100644 manifests/modules/fundamentals/fargate/profile/fargate.yaml create mode 100644 website/docs/introduction/assets/vscode-copy-paste.webp create mode 100644 website/docs/introduction/setup/your-account/assets/vscode-outputs.webp create mode 100644 website/docs/introduction/setup/your-account/assets/vscode-password-retrieve.webp create mode 100644 website/docs/introduction/setup/your-account/assets/vscode-password-visible.webp create mode 100644 website/docs/introduction/setup/your-account/assets/vscode-password.webp create mode 100644 website/docs/introduction/setup/your-account/assets/vscode-splash.webp diff --git a/.github/workflows/module-test.yaml b/.github/workflows/module-test.yaml index 2a1367d58..d84a25f68 100644 --- a/.github/workflows/module-test.yaml +++ b/.github/workflows/module-test.yaml @@ -78,7 +78,6 @@ jobs: MODULE: ${{ inputs.module }} GLOB: ${{ inputs.glob }} AWS_REGION: "${{ secrets.AWS_REGION }}" - ASSUME_ROLE: "${{ secrets.AWS_ROLE_ARN }}" DOCKER_DNS_OVERRIDE: "8.8.8.8" run: | export AWS_DEFAULT_REGION="$AWS_REGION" @@ -102,7 +101,6 @@ jobs: DOCKER_BUILDKIT: 1 DEV_MODE: 1 AWS_REGION: "${{ secrets.AWS_REGION }}" - ASSUME_ROLE: "${{ secrets.AWS_ROLE_ARN }}" run: | export AWS_DEFAULT_REGION="$AWS_REGION" make test environment="$CLUSTER_ID" module="cleanup" @@ -119,13 +117,4 @@ jobs: env: AWS_REGION: "${{ secrets.AWS_REGION }}" run: | - export CLEANUP_ENVIRONMENT_NAME="eks-workshop-$CLUSTER_ID" - export AWS_DEFAULT_REGION="$AWS_REGION" - - envsubst < hack/lib/filter.yml > filter.yml - - cat filter.yml - - awsweeper --force filter.yml - make destroy-infrastructure environment="$CLUSTER_ID" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e12a0cab4..919c3ee3e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -42,6 +42,8 @@ jobs: - name: Check out code uses: actions/checkout@v4 - name: Make shell + env: + SKIP_CREDENTIALS: 1 run: | bash hack/exec.sh '' 'ls -la' @@ -70,3 +72,14 @@ jobs: node-version: 18 - run: | npx cspell lint "website/docs/**/*.md" + + terraform-validate: + name: "Validate Terraform" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "~1.9.0" + - run: | + bash hack/validate-terraform.sh diff --git a/.github/workflows/test-cleanup.yaml b/.github/workflows/test-cleanup.yaml index 8473fb68b..b15005522 100644 --- a/.github/workflows/test-cleanup.yaml +++ b/.github/workflows/test-cleanup.yaml @@ -24,19 +24,6 @@ jobs: - name: Install utilities run: | sudo apt install -y gettext - - mkdir -p ${HOME}/.local/bin - wget https://github.com/jckuester/awsweeper/releases/download/v0.12.0/awsweeper_0.12.0_linux_amd64.tar.gz - tar zxf awsweeper_0.12.0_linux_amd64.tar.gz - mv awsweeper_0.12.0_linux_amd64/awsweeper ${HOME}/.local/bin - - wget https://github.com/eksctl-io/eksctl/releases/download/v0.169.0/eksctl_Linux_amd64.tar.gz - tar zxf eksctl_Linux_amd64.tar.gz - mv eksctl ${HOME}/.local/bin - - chmod +x ${HOME}/.local/bin/* - - echo "${HOME}/.local/bin" >> $GITHUB_PATH - name: Get AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 with: @@ -50,13 +37,4 @@ jobs: CLUSTER_ID: ${{ github.event.inputs.clusterId }} AWS_REGION: "${{ secrets.AWS_REGION }}" run: | - export CLEANUP_ENVIRONMENT_NAME="$CLUSTER_ID" - export AWS_DEFAULT_REGION="$AWS_REGION" - - envsubst < hack/lib/filter.yml > filter.yml - - cat filter.yml - - awsweeper --force filter.yml - make destroy-infrastructure environment="$CLUSTER_ID" diff --git a/Makefile b/Makefile index 4300ee33b..2f81d713d 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,8 @@ delete-environment: .PHONY: create-infrastructure create-infrastructure: - bash hack/exec.sh $(environment) 'cat /cluster/eksctl/cluster.yaml | envsubst | eksctl create cluster -f -' + bash hack/create-infrastructure.sh $(environment) .PHONY: destroy-infrastructure destroy-infrastructure: - bash hack/exec.sh $(environment) 'cat /cluster/eksctl/cluster.yaml | envsubst | eksctl delete cluster --wait --force --disable-nodegroup-eviction --timeout 45m -f -' + bash hack/destroy-infrastructure.sh $(environment) diff --git a/cluster/terraform/eks.tf b/cluster/terraform/eks.tf index 6949f4a45..276818318 100644 --- a/cluster/terraform/eks.tf +++ b/cluster/terraform/eks.tf @@ -32,9 +32,12 @@ module "eks" { eks_managed_node_groups = { default = { - instance_types = ["m5.large"] - force_update_version = true - release_version = var.ami_release_version + instance_types = ["m5.large"] + force_update_version = true + release_version = var.ami_release_version + use_name_prefix = false + iam_role_name = "${var.cluster_name}-ng-default" + iam_role_use_name_prefix = false min_size = 3 max_size = 6 diff --git a/docs/reviewer_checklist.md b/docs/reviewer_checklist.md index c5f58b764..d7dd5f591 100644 --- a/docs/reviewer_checklist.md +++ b/docs/reviewer_checklist.md @@ -22,6 +22,10 @@ See style guide for expanded explanations. - [ ] `$EKS_CLUSTER_NAME` is used instead of hard-coded cluster names, including referencing other infrastructure that may use the cluster name - [ ] Avoided use of interactive `kubectl exec` or multiple terminal windows (or tests skipped) +## AWS infrastructure + +- [ ] All Terraform resources created have names that prefixed with the EKS cluster name (`var.addon_context.eks_cluster_id`) + ## Tests - [ ] `bash` blocks that run commands that are intended to error use `expectError=true` @@ -34,5 +38,5 @@ See style guide for expanded explanations. ## Misc - [ ] Generated lab timing has been created (new lab) or updated (updated lab) if needed -- [ ] All Terraform resources created have dynamic names +- [ ] Relevant updates have been made to the [lab IAM policy](../lab/iam-policy-labs.json) - [ ] Images should be in `webp` format diff --git a/hack/build-ide-cfn.sh b/hack/build-ide-cfn.sh new file mode 100644 index 000000000..49586cc0d --- /dev/null +++ b/hack/build-ide-cfn.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +output_path=$1 + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +if [ -z "$output_path" ]; then + outfile=$(mktemp) +else + outfile=$output_path +fi + +cd lab + +export Env="${EKS_CLUSTER_NAME}" + +cat cfn/eks-workshop-vscode-cfn.yaml | yq '(.. | select(has("file"))) |= (load(.file))' | envsubst '$Env' > $outfile + +echo "Output file: $outfile" \ No newline at end of file diff --git a/hack/create-infrastructure.sh b/hack/create-infrastructure.sh new file mode 100644 index 000000000..14775e0af --- /dev/null +++ b/hack/create-infrastructure.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +environment=$1 + +set -Eeuo pipefail +set -u + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +bash $SCRIPT_DIR/update-iam-role.sh $environment + +sleep 5 + +cluster_exists=0 +aws eks describe-cluster --name "${EKS_CLUSTER_NAME}" &> /dev/null || cluster_exists=$? + +if [ $cluster_exists -eq 0 ]; then + echo "Cluster ${EKS_CLUSTER_NAME} already exists" +else + echo "Creating cluster ${EKS_CLUSTER_NAME}" + bash $SCRIPT_DIR/exec.sh "${environment}" 'cat /cluster/eksctl/cluster.yaml | envsubst | eksctl create cluster -f -' +fi \ No newline at end of file diff --git a/hack/deploy-ide-cfn.sh b/hack/deploy-ide-cfn.sh new file mode 100644 index 000000000..982a01409 --- /dev/null +++ b/hack/deploy-ide-cfn.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +outfile=$(mktemp) + +bash $SCRIPT_DIR/build-ide-cfn.sh $outfile + +aws cloudformation deploy --stack-name eks-workshop-ide1 \ + --capabilities CAPABILITY_NAMED_IAM --disable-rollback --template-file $outfile \ No newline at end of file diff --git a/hack/destroy-infrastructure.sh b/hack/destroy-infrastructure.sh new file mode 100644 index 000000000..6a2342870 --- /dev/null +++ b/hack/destroy-infrastructure.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +environment=$1 + +set -Eeuo pipefail +set -u + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +cluster_exists=0 +aws eks describe-cluster --name "${EKS_CLUSTER_NAME}" &> /dev/null || cluster_exists=$? + +if [ $cluster_exists -eq 0 ]; then + echo "Deleting cluster ${EKS_CLUSTER_NAME}" + bash $SCRIPT_DIR/shell.sh "${environment}" 'delete-environment || true' + + bash $SCRIPT_DIR/exec.sh "${environment}" 'eksctl delete cluster --name ${EKS_CLUSTER_NAME} --region ${AWS_REGION} --wait --force --disable-nodegroup-eviction --timeout 45m' +else + echo "Cluster ${EKS_CLUSTER_NAME} does not exist" +fi + +aws cloudformation delete-stack --stack-name ${EKS_CLUSTER_NAME}-ide-role || true \ No newline at end of file diff --git a/hack/exec.sh b/hack/exec.sh index bf38b1513..ea785e5b0 100644 --- a/hack/exec.sh +++ b/hack/exec.sh @@ -19,7 +19,11 @@ container_image='eks-workshop-environment' (cd $SCRIPT_DIR/../lab && $CONTAINER_CLI build -q -t $container_image .) -source $SCRIPT_DIR/lib/generate-aws-creds.sh +if [ -z "$SKIP_CREDENTIALS" ]; then + source $SCRIPT_DIR/lib/generate-aws-creds.sh +else + aws_credential_args="" +fi echo "Executing command in container..." diff --git a/hack/find-dangling-resources.sh b/hack/find-dangling-resources.sh new file mode 100644 index 000000000..fc52c7552 --- /dev/null +++ b/hack/find-dangling-resources.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +environment=$1 + +set -Eeuo pipefail +set -u + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +aws resourcegroupstaggingapi get-resources --tag-filters Key=env,Values=$EKS_CLUSTER_NAME --query 'ResourceTagMappingList[].ResourceARN' \ No newline at end of file diff --git a/hack/lib/common-env.sh b/hack/lib/common-env.sh index 636ec0842..42181d2f9 100644 --- a/hack/lib/common-env.sh +++ b/hack/lib/common-env.sh @@ -13,3 +13,12 @@ if [ -z "$AWS_REGION" ]; then export AWS_REGION="us-west-2" fi + +SKIP_CREDENTIALS=${SKIP_CREDENTIALS:-""} + +if [ -z "$SKIP_CREDENTIALS" ]; then + ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + + IDE_ROLE_NAME="${EKS_CLUSTER_NAME}-ide-role" + IDE_ROLE_ARN="arn:aws:iam::${ACCOUNT_ID}:role/${IDE_ROLE_NAME}" +fi \ No newline at end of file diff --git a/hack/lib/generate-aws-creds.sh b/hack/lib/generate-aws-creds.sh index 3e5728d85..dbab3dd40 100644 --- a/hack/lib/generate-aws-creds.sh +++ b/hack/lib/generate-aws-creds.sh @@ -1,21 +1,19 @@ -aws_credential_args="" +echo "Generating temporary AWS credentials..." -ASSUME_ROLE=${ASSUME_ROLE:-""} -AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-""} +session_suffix=$(openssl rand -hex 4) + +target_role=${IDE_ROLE_ARN} -if [ ! -z "$AWS_ACCESS_KEY_ID" ]; then - echo "Using environment AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY" +ASSUME_ROLE=${ASSUME_ROLE:-""} - aws_credential_args="-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" -elif [ ! -z "$ASSUME_ROLE" ]; then - echo "Generating temporary AWS credentials..." +if [ ! -z "$ASSUME_ROLE" ]; then + echo "Assuming role $ASSUME_ROLE" + target_role=$ASSUME_ROLE +fi - ACCESS_VARS=$(aws sts assume-role --role-arn $ASSUME_ROLE --role-session-name ${EKS_CLUSTER_NAME}-shell --output json | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId) AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey) AWS_SESSION_TOKEN=\(.SessionToken)"') +ACCESS_VARS=$(aws sts assume-role --role-arn ${target_role} --role-session-name ${EKS_CLUSTER_NAME}-shell-${session_suffix} --output json | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId) AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey) AWS_SESSION_TOKEN=\(.SessionToken)"') - # TODO: This should probably not use eval - eval "$ACCESS_VARS" +# TODO: This should probably not use eval +eval "$ACCESS_VARS" - aws_credential_args="-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" -else - echo "Inheriting credentials from instance profile" -fi \ No newline at end of file +aws_credential_args="-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" \ No newline at end of file diff --git a/hack/run-tests.sh b/hack/run-tests.sh index 3e96fe595..8fbe0dba1 100755 --- a/hack/run-tests.sh +++ b/hack/run-tests.sh @@ -76,19 +76,27 @@ RESOURCES_PRECREATED=${RESOURCES_PRECREATED:-""} echo "Running test suite..." +exit_code=0 + $CONTAINER_CLI run $background_args $dns_args \ --name $container_name \ -v $SCRIPT_DIR/../website/docs:/content \ -v $SCRIPT_DIR/../manifests:/manifests \ -e 'EKS_CLUSTER_NAME' -e 'AWS_REGION' -e 'RESOURCES_PRECREATED' \ - $aws_credential_args $container_image -g "${actual_glob}" --hook-timeout 3600 --timeout 3600 $output_args ${AWS_EKS_WORKSHOP_TEST_FLAGS} + $aws_credential_args $container_image -g "${actual_glob}" --hook-timeout 3600 --timeout 3600 $output_args ${AWS_EKS_WORKSHOP_TEST_FLAGS} || exit_code=$? -if [ ! -z "$TEST_REPORT" ]; then - docker cp $container_name:/tmp/test-report.json $TEST_REPORT > /dev/null +if [ $exit_code -eq 0 ]; then + if [ ! -z "$TEST_REPORT" ]; then + docker cp $container_name:/tmp/test-report.json $TEST_REPORT > /dev/null + fi fi docker rm $container_name > /dev/null +if [ $exit_code -ne 0 ]; then + exit $exit_code +fi + if [ ! -z "$GENERATE_TIMINGS" ]; then tmpfile=$(mktemp) diff --git a/hack/update-iam-role.sh b/hack/update-iam-role.sh new file mode 100644 index 000000000..0a7abdbfd --- /dev/null +++ b/hack/update-iam-role.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +environment=$1 + +set -Eeuo pipefail +set -u + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +outfile=$(mktemp) + +cd lab + +export Env="${EKS_CLUSTER_NAME}" + +cat iam/iam-role-cfn.yaml | yq '(.. | select(has("file"))) |= (load(.file))' | envsubst '$Env' > $outfile + +aws cloudformation deploy \ + --stack-name ${EKS_CLUSTER_NAME}-ide-role \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ + --template-file $outfile \ No newline at end of file diff --git a/hack/validate-terraform.sh b/hack/validate-terraform.sh new file mode 100644 index 000000000..a085a0ff6 --- /dev/null +++ b/hack/validate-terraform.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +environment=$1 + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/lib/common-env.sh + +terraform_dir="$(mktemp -d)" +manifests_dir="${SCRIPT_DIR}/../manifests" + +conf_dir="$terraform_dir/conf" + +mkdir -p "$conf_dir" + +cp $manifests_dir/.workshop/terraform/base.tf $conf_dir/base.tf + +find $manifests_dir/modules -type d -name "terraform" -print0 | while read -d $'\0' file +do + target=$(echo $file | md5sum | cut -f1 -d" ") + cp -R $file $conf_dir/$target + + cat << EOF > $conf_dir/$target.tf +module "gen-$target" { + source = "./$target" + + eks_cluster_id = local.eks_cluster_id + eks_cluster_version = local.eks_cluster_version + cluster_security_group_id = local.cluster_security_group_id + addon_context = local.addon_context + tags = local.tags + resources_precreated = var.resources_precreated +} +EOF +done + +terraform -chdir="${conf_dir}" init -backend=false + +terraform -chdir="${conf_dir}" validate \ No newline at end of file diff --git a/lab/Dockerfile b/lab/Dockerfile index ac169e511..234e41a3a 100644 --- a/lab/Dockerfile +++ b/lab/Dockerfile @@ -1,6 +1,6 @@ -FROM public.ecr.aws/amazonlinux/amazonlinux:2 +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 -RUN yum install -y shadow-utils && useradd \ +RUN yum install -y tar gzip vim shadow-utils && useradd \ --home "/home/ec2-user" \ --create-home \ --user-group \ diff --git a/lab/bin/reset-environment b/lab/bin/reset-environment index 333963db3..2c3626d54 100644 --- a/lab/bin/reset-environment +++ b/lab/bin/reset-environment @@ -89,6 +89,8 @@ logmessage "Tip: Read the rest of the lab introduction while you wait!" if [ -f "/eks-workshop/hooks/cleanup.sh" ]; then bash /eks-workshop/hooks/cleanup.sh + + rm /eks-workshop/hooks/cleanup.sh fi kubectl delete pod load-generator --ignore-not-found @@ -227,4 +229,4 @@ kubectl delete pod -l app.kubernetes.io/created-by=eks-workshop -l app.kubernete kubectl wait --for=condition=Ready --timeout=240s pods -l app.kubernetes.io/created-by=eks-workshop -A # Finished -logmessage 'Environment is ready' +logmessage 'Environment is ready' \ No newline at end of file diff --git a/lab/cfn/eks-workshop-vscode-cfn.yaml b/lab/cfn/eks-workshop-vscode-cfn.yaml new file mode 100644 index 000000000..5b664c477 --- /dev/null +++ b/lab/cfn/eks-workshop-vscode-cfn.yaml @@ -0,0 +1,578 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Creates a code-server IDE for the EKS workshop +Parameters: + InstanceVolumeSize: + Type: Number + Description: The Size in GB of the Cloud9 Instance Volume. + Default: 30 + RepositoryOwner: + Type: String + Description: The owner of the GitHub repository to be used to bootstrap Cloud9 + Default: "aws-samples" + RepositoryName: + Type: String + Description: The name of the GitHub repository to be used to bootstrap Cloud9 + Default: "eks-workshop-v2" + RepositoryRef: + Type: String + Description: The Git reference to be used to bootstrap Cloud9 + Default: "vscode-ide" + ResourcesPrecreated: + Type: String + Description: Whether lab infrastructure has been pre-provisioned + Default: "false" + AllowedValues: + - "false" + - "true" + AnalyticsEndpoint: + Type: String + Description: Analytics endpoint used for AWS events + Default: "" + CodeServerVersion: + Type: String + Description: Default code-server version to use + Default: "4.91.1" + AmiParameterStoreName: + Type: "AWS::SSM::Parameter::Value" + Default: "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-x86_64" + Environment: + Type: String + Description: For testing purposes only + Default: "" + +Mappings: + PrefixListID: + ap-northeast-1: + PrefixList: pl-58a04531 + ap-northeast-2: + PrefixList: pl-22a6434b + ap-south-1: + PrefixList: pl-9aa247f3 + ap-southeast-1: + PrefixList: pl-31a34658 + ap-southeast-2: + PrefixList: pl-b8a742d1 + ca-central-1: + PrefixList: pl-38a64351 + eu-central-1: + PrefixList: pl-a3a144ca + eu-north-1: + PrefixList: pl-fab65393 + eu-west-1: + PrefixList: pl-4fa04526 + eu-west-2: + PrefixList: pl-93a247fa + eu-west-3: + PrefixList: pl-75b1541c + sa-east-1: + PrefixList: pl-5da64334 + us-east-1: + PrefixList: pl-3b927c52 + us-east-2: + PrefixList: pl-b6a144df + us-west-1: + PrefixList: pl-4ea04527 + us-west-2: + PrefixList: pl-82a045eb + +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/24 + EnableDnsSupport: true + EnableDnsHostnames: true + + InternetGateway: + Type: AWS::EC2::InternetGateway + + GatewayAttachment: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + + PublicSubnet: + Type: AWS::EC2::Subnet + Properties: + CidrBlock: 10.0.0.0/24 + VpcId: !Ref VPC + MapPublicIpOnLaunch: true + AvailabilityZone: !Select [0, !GetAZs ""] + + PublicSubnetRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + PublicSubnetRoute: + Type: AWS::EC2::Route + DependsOn: GatewayAttachment + Properties: + RouteTableId: !Ref PublicSubnetRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + + PublicSubnetRouteTableAssoc: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref PublicSubnetRouteTable + SubnetId: !Ref PublicSubnet + + SecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: SG for IDE + SecurityGroupIngress: + - Description: Allow HTTP from CloudFront + IpProtocol: tcp + FromPort: 80 + ToPort: 80 + SourcePrefixListId: + !FindInMap [PrefixListID, !Ref "AWS::Region", PrefixList] + SecurityGroupEgress: + - Description: Allow all outbound traffic + IpProtocol: -1 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VPC + + EksWorkshopIdeLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: + Fn::Join: + - "" + - - EksWorkshopIdeLambdaPolicy- + - Ref: AWS::Region + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: arn:aws:logs:*:*:* + - Effect: Allow + Action: + - iam:PassRole + - ssm:SendCommand + - ssm:GetCommandInvocation + Resource: "*" + + EksWorkshopIdeBootstrapInstanceLambda: + Type: Custom::EksWorkshopIdeBootstrapInstanceLambda + DependsOn: + - EksWorkshopIdeLambdaExecutionRole + Properties: + ServiceToken: + Fn::GetAtt: + - EksWorkshopIdeBootstrapInstanceLambdaFunction + - Arn + REGION: + Ref: AWS::Region + InstanceId: + Ref: EksWorkshopIdeInstance + SsmDocument: + Ref: EksWorkshopIdeSSMDocument + #UpdateTrigger: + # Ref: UpdateTrigger + + EksWorkshopIdeBootstrapInstanceLambdaFunction: + Type: AWS::Lambda::Function + Properties: + Handler: index.lambda_handler + Role: + Fn::GetAtt: + - EksWorkshopIdeLambdaExecutionRole + - Arn + Runtime: python3.12 + Environment: + Variables: + DiskSize: + Ref: InstanceVolumeSize + MemorySize: 256 + Timeout: "900" + Code: + ZipFile: | + from __future__ import print_function + import boto3 + import json + import os + import time + import traceback + import cfnresponse + import logging + logger = logging.getLogger(__name__) + + def lambda_handler(event, context): + print(event.values()) + print('context: {}'.format(context)) + responseData = {} + + status = cfnresponse.SUCCESS + + if event['RequestType'] == 'Delete': + responseData = {'Success': 'Custom Resource removed'} + cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') + else: + try: + # Open AWS clients + #ec2 = boto3.client('ec2') + ssm = boto3.client('ssm') + + instance_id = event['ResourceProperties']['InstanceId'] + + ssm_document = event['ResourceProperties']['SsmDocument'] + + print('Sending SSM command...') + + response = ssm.send_command( + InstanceIds=[instance_id], + DocumentName=ssm_document) + + command_id = response['Command']['CommandId'] + + waiter = ssm.get_waiter('command_executed') + + waiter.wait( + CommandId=command_id, + InstanceId=instance_id, + WaiterConfig={ + 'Delay': 10, + 'MaxAttempts': 60 + } + ) + + responseData = {'Success': 'Started bootstrapping for instance: '+instance_id} + cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') + + except Exception as e: + status = cfnresponse.FAILED + print(traceback.format_exc()) + responseData = {'Error': traceback.format_exc(e)} + finally: + cfnresponse.send(event, context, status, responseData, 'CustomResourcePhysicalID') + + EksWorkshopIdeSSMDocument: + Type: AWS::SSM::Document + Properties: + DocumentType: Command + DocumentFormat: YAML + Content: + schemaVersion: "2.2" + description: Bootstrap Cloud9 Instance + mainSteps: + - action: aws:runShellScript + name: EksWorkshopIdebootstrap + inputs: + runCommand: + - !Sub | + set -e + + yum install -y git tar gzip vim nodejs npm make gcc g++ + + export environment="${Environment}" + + source <(curl -fsSL https://raw.githubusercontent.com/${RepositoryOwner}/${RepositoryName}/${RepositoryRef}/hack/lib/common-env.sh) + + dnf copr enable -y @caddy/caddy epel-9-x86_64 + dnf install -y caddy + systemctl enable --now caddy + + tee /etc/caddy/Caddyfile < ~/.local/share/code-server/coder.json + + curl -fsSL https://raw.githubusercontent.com/${RepositoryOwner}/${RepositoryName}/${RepositoryRef}/lab/scripts/setup.sh | bash + + code-server --install-extension ms-kubernetes-tools.vscode-kubernetes-tools --force + code-server --install-extension redhat.vscode-yaml --force + + EOT + + systemctl restart code-server@ec2-user + + EksWorkshopIdeRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + - ssm.amazonaws.com + Action: + - sts:AssumeRole + Policies: + - PolicyName: ide-password + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - secretsmanager:GetResourcePolicy + - secretsmanager:GetSecretValue + - secretsmanager:DescribeSecret + - secretsmanager:ListSecretVersionIds + Resource: + - !Ref EksWorkshopIdePassword + - Effect: Allow + Action: secretsmanager:ListSecrets + Resource: "*" + + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore + Path: "/" + + EksWorkshopIamPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: !Sub ${AWS::StackName}-iam + PolicyDocument: + file: ./iam/policies/iam.yaml + + EksWorkshopBasePolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: !Sub ${AWS::StackName}-base + PolicyDocument: + file: ./iam/policies/base.yaml + + EksWorkshopEc2Policy: + Type: AWS::IAM::ManagedPolicy + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: !Sub ${AWS::StackName}-ec2 + PolicyDocument: + file: ./iam/policies/ec2.yaml + + EksWorkshopLabsPolicy1: + Type: AWS::IAM::ManagedPolicy + DependsOn: + - EksWorkshopIdeRole + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: !Sub ${AWS::StackName}-labs1 + PolicyDocument: + file: ./iam/policies/labs1.yaml + + EksWorkshopLabsPolicy2: + Type: AWS::IAM::ManagedPolicy + DependsOn: + - EksWorkshopIdeRole + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: !Sub ${AWS::StackName}-labs2 + PolicyDocument: + file: ./iam/policies/labs2.yaml + + EksWorkshopIdeInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: "/" + Roles: + - Ref: EksWorkshopIdeRole + + EksWorkshopIdeInstance: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref AmiParameterStoreName + InstanceType: t3.medium + BlockDeviceMappings: + - Ebs: + VolumeSize: !Ref InstanceVolumeSize + VolumeType: gp3 + DeleteOnTermination: true + Encrypted: true + DeviceName: /dev/xvda + SubnetId: !Ref PublicSubnet + SecurityGroupIds: + - !Ref SecurityGroup + IamInstanceProfile: !Ref EksWorkshopIdeInstanceProfile + Tags: + - Key: type + Value: eksworkshop-ide + + EksWorkshopIdePassword: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub ${AWS::StackName}-password + GenerateSecretString: + ExcludeCharacters: "\"@/\\" + ExcludePunctuation: true + GenerateStringKey: password + IncludeSpace: false + PasswordLength: 32 + SecretStringTemplate: '{"password":""}' + UpdateReplacePolicy: Delete + DeletionPolicy: Delete + + EksWorkshopIdeCachePolicy: + Type: AWS::CloudFront::CachePolicy + Properties: + CachePolicyConfig: + DefaultTTL: 86400 + MaxTTL: 31536000 + MinTTL: 1 + Name: !Ref AWS::StackName + ParametersInCacheKeyAndForwardedToOrigin: + CookiesConfig: + CookieBehavior: all + EnableAcceptEncodingGzip: False + HeadersConfig: + HeaderBehavior: whitelist + Headers: + - Accept-Charset + - Authorization + - Origin + - Accept + - Referer + - Host + - Accept-Language + - Accept-Encoding + - Accept-Datetime + QueryStringsConfig: + QueryStringBehavior: all + + EksWorkshopIdeCloudFrontDistribution: + Type: AWS::CloudFront::Distribution + Properties: + DistributionConfig: + Enabled: True + HttpVersion: http2 + CacheBehaviors: + - AllowedMethods: + - GET + - HEAD + - OPTIONS + - PUT + - PATCH + - POST + - DELETE + CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad + Compress: False + OriginRequestPolicyId: 216adef6-5c7f-47e4-b989-5492eafa07d3 + TargetOriginId: !Sub CloudFront-${AWS::StackName} + ViewerProtocolPolicy: allow-all + PathPattern: "/proxy/*" + DefaultCacheBehavior: + AllowedMethods: + - GET + - HEAD + - OPTIONS + - PUT + - PATCH + - POST + - DELETE + CachePolicyId: !Ref EksWorkshopIdeCachePolicy + OriginRequestPolicyId: 216adef6-5c7f-47e4-b989-5492eafa07d3 + TargetOriginId: !Sub CloudFront-${AWS::StackName} + ViewerProtocolPolicy: allow-all + Origins: + - DomainName: !GetAtt EksWorkshopIdeInstance.PublicDnsName + Id: !Sub CloudFront-${AWS::StackName} + CustomOriginConfig: + OriginProtocolPolicy: http-only + +Outputs: + IdeUrl: + Value: !Sub https://${EksWorkshopIdeCloudFrontDistribution.DomainName} + + IdePasswordSecret: + Value: !Sub + - https://console.aws.amazon.com/secretsmanager/secret?name=${SecretName} + - SecretName: !Sub ${AWS::StackName}-password + + IdeRole: + Value: !Sub ${EksWorkshopIdeRole.Arn} diff --git a/lab/iam/iam-role-cfn.yaml b/lab/iam/iam-role-cfn.yaml new file mode 100644 index 000000000..5232703ea --- /dev/null +++ b/lab/iam/iam-role-cfn.yaml @@ -0,0 +1,64 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Creates an IAM role for the EKS workshop IDE +Resources: + EksWorkshopIdeRole: + Type: AWS::IAM::Role + Properties: + RoleName: ${Env}-ide-role + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !Sub arn:aws:iam::${AWS::AccountId}:root + Action: + - sts:AssumeRole + + EksWorkshopIamPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: ${Env}-ide-iam + PolicyDocument: + file: ./iam/policies/iam.yaml + + EksWorkshopBasePolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: ${Env}-ide-base + PolicyDocument: + file: ./iam/policies/base.yaml + + EksWorkshopEc2Policy: + Type: AWS::IAM::ManagedPolicy + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: ${Env}-ide-ec2 + PolicyDocument: + file: ./iam/policies/ec2.yaml + + EksWorkshopLabsPolicy1: + Type: AWS::IAM::ManagedPolicy + DependsOn: + - EksWorkshopIdeRole + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: ${Env}-ide-labs1 + PolicyDocument: + file: ./iam/policies/labs1.yaml + + EksWorkshopLabsPolicy2: + Type: AWS::IAM::ManagedPolicy + DependsOn: + - EksWorkshopIdeRole + Properties: + Roles: + - !Ref EksWorkshopIdeRole + ManagedPolicyName: ${Env}-ide-labs2 + PolicyDocument: + file: ./iam/policies/labs2.yaml diff --git a/lab/iam/policies/base.yaml b/lab/iam/policies/base.yaml new file mode 100644 index 000000000..5261d3e07 --- /dev/null +++ b/lab/iam/policies/base.yaml @@ -0,0 +1,87 @@ +Version: "2012-10-17" +Statement: + - Effect: Allow + Action: + - eks:* + - ec2:CreateLaunchTemplate + - ec2:DeleteLaunchTemplate + - sts:GetCallerIdentity + Resource: ["*"] + - Effect: Allow + Action: + - cloudformation:CreateStack + Resource: + - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/eksctl-${Env}* + Condition: + "Null": + cloudformation:RoleARN: "true" + - Effect: Allow + Action: + - cloudformation:DeleteStack + Resource: + - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/eksctl-${Env}* + Condition: + "Null": + cloudformation:RoleARN: "true" + - Effect: Allow + Action: + - cloudformation:Get* + - cloudformation:Describe* + - cloudformation:List* + - cloudformation:TagResource + Resource: ["*"] + - Effect: Allow + Action: + - autoscaling:UpdateAutoScalingGroup + Resource: ["*"] + Condition: + StringLike: + aws:ResourceTag/eks:cluster-name: + - ${Env} + - Effect: Allow + Action: + - autoscaling:Get* + - autoscaling:Describe* + Resource: ["*"] + - Effect: Allow + Action: + - ecr-public:GetAuthorizationToken + - sts:GetServiceBearerToken + Resource: ["*"] + - Effect: Allow + Action: + - kms:CreateKey + - kms:TagResource + - kms:ScheduleKeyDeletion + - kms:CreateGrant + - kms:EnableKeyRotation + - kms:GetKeyPolicy + - kms:GetKeyRotationStatus + - kms:ListResourceTags + - kms:PutKeyPolicy + Resource: ["*"] + - Effect: Allow + Action: + - kms:Decrypt + - kms:DescribeKey + - kms:EnableKeyRotation + - kms:Encrypt + - kms:GenerateDataKey + - kms:GenerateDataKeyWithoutPlaintext + Resource: ["*"] + Condition: + StringLike: + kms:RequestAlias: "alias/${Env}*" + - Effect: Allow + Action: + - kms:CreateAlias + - kms:DeleteAlias + Resource: + - !Sub arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/${Env}* + - !Sub arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/* + - Effect: Allow + Action: + - kms:List* + - kms:Get* + - kms:Describe* + Resource: ["*"] diff --git a/lab/iam/policies/ec2.yaml b/lab/iam/policies/ec2.yaml new file mode 100644 index 000000000..4f588cd50 --- /dev/null +++ b/lab/iam/policies/ec2.yaml @@ -0,0 +1,92 @@ +Version: "2012-10-17" +Statement: + - Effect: Allow + Action: + - ec2:Get* + - ec2:Describe* + - ec2:List* + - ec2:RunInstances + Resource: ["*"] + - Effect: Allow + Action: + - ec2:TerminateInstances + Resource: ["*"] + Condition: + StringLike: + aws:ResourceTag/env: + - ${Env}* + - Effect: Deny + Action: ec2:RunInstances + Resource: + - !Sub arn:aws:ec2:*:*:instance/* + Condition: + ForAnyValue:StringNotLike: + ec2:InstanceType: + - m5.large + - t4g.medium + - c*.large + - Effect: Allow + Action: + - ec2:CreateVpc + - ec2:CreateSubnet + - ec2:CreateRouteTable + - ec2:CreateRoute + - ec2:CreateInternetGateway + - ec2:AttachInternetGateway + - ec2:AssociateRouteTable + - ec2:ModifyVpcAttribute + - ec2:CreateSecurityGroup + - ec2:AllocateAddress + - ec2:ReleaseAddress + - ec2:DisassociateAddress + - ec2:CreateNetworkAclEntry + - ec2:DeleteNetworkAclEntry + - ec2:CreateNatGateway + - ec2:DeleteNatGateway + Resource: ["*"] + - Effect: Allow + Action: + - ec2:DeleteVpc + - ec2:DeleteSubnet + - ec2:DeleteRouteTable + - ec2:DeleteRoute + - ec2:DeleteInternetGateway + - ec2:DetachInternetGateway + - ec2:DisassociateRouteTable + - ec2:ModifyVpcAttribute + - ec2:ModifySubnetAttribute + - ec2:AuthorizeSecurityGroup* + - ec2:UpdateSecurityGroupRuleDescriptionsEgress + - ec2:RevokeSecurityGroup* + - ec2:DeleteSecurityGroup + - ec2:ModifySecurityGroupRules + - ec2:UpdateSecurityGroupRuleDescriptionsIngress + Resource: ["*"] + Condition: + StringLike: + aws:ResourceTag/env: + - ${Env}* + - Effect: Allow + Action: + - ec2:AuthorizeSecurityGroup* + - ec2:RevokeSecurityGroup* + Resource: ["*"] + Condition: + StringLike: + aws:ResourceTag/aws:eks:cluster-name: + - ${Env}* + - Effect: Allow + Action: + - ec2:CreateTags + - ec2:DeleteTags + Resource: ["*"] + - Effect: Allow + Action: + - ec2:AssociateVpcCidrBlock + - ec2:DisassociateVpcCidrBlock + Resource: + - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:vpc/* + Condition: + StringLike: + aws:ResourceTag/env: + - ${Env}* diff --git a/lab/iam/policies/iam.yaml b/lab/iam/policies/iam.yaml new file mode 100644 index 000000000..15641fc6c --- /dev/null +++ b/lab/iam/policies/iam.yaml @@ -0,0 +1,75 @@ +Version: "2012-10-17" +Statement: + - Effect: Allow + Action: + - iam:CreateRole + - iam:GetRolePolicy + - iam:DetachRolePolicy + - iam:AttachRolePolicy + - iam:PutRolePolicy + - iam:DeleteRolePolicy + - iam:DeleteRole + - iam:ListInstanceProfilesForRole + - iam:ListAttachedRolePolicies + - iam:ListRolePolicies + - iam:TagRole + - iam:PassRole + - sts:AssumeRole + Resource: + - !Sub arn:aws:iam::${AWS::AccountId}:role/${Env}* + - !Sub arn:aws:iam::${AWS::AccountId}:role/eksctl-${Env}* + - Effect: Allow + Action: + - iam:CreatePolicy + - iam:DeletePolicy + - iam:GetPolicyVersion + - iam:ListPolicyVersions + - iam:TagPolicy + - iam:GetPolicy + Resource: + - !Sub arn:aws:iam::${AWS::AccountId}:policy/${Env}* + - !Sub arn:aws:iam::${AWS::AccountId}:policy/eksctl-${Env}* + - Effect: Allow + Action: + - iam:CreateInstanceProfile + - iam:DeleteInstanceProfile + - iam:GetInstanceProfile + - iam:TagInstanceProfile + - iam:RemoveRoleFromInstanceProfile + - iam:AddRoleToInstanceProfile + Resource: + - !Sub arn:aws:iam::${AWS::AccountId}:instance-profile/${Env}* + - !Sub arn:aws:iam::${AWS::AccountId}:instance-profile/eksctl-${Env}* + - !Sub arn:aws:iam::${AWS::AccountId}:instance-profile/eks-* + - Effect: Allow + Action: + - iam:CreateUser + - iam:DeleteUser + - iam:TagUser + - iam:GetUser + - iam:ListGroupsForUser + - iam:AttachUserPolicy + - iam:DetachUserPolicy + - iam:ListAttachedUserPolicies + - iam:*SSHPublicKey + Resource: + - !Sub arn:aws:iam::${AWS::AccountId}:user/${Env}* + - Effect: Allow + Action: + - iam:ListOpenIDConnectProviders + - iam:CreateOpenIDConnectProvider + - iam:DeleteOpenIDConnectProvider + - iam:TagOpenIDConnectProvider + - iam:GetOpenIDConnectProvider + - iam:GetRole + Resource: ["*"] + - Effect: Allow + Action: + - iam:CreateServiceLinkedRole + Resource: ["*"] + Condition: + StringEquals: + iam:AWSServiceName: + - eks.amazonaws.com + - eks-nodegroup.amazonaws.com + - eks-fargate.amazonaws.com diff --git a/lab/iam/policies/labs1.yaml b/lab/iam/policies/labs1.yaml new file mode 100644 index 000000000..20217d34f --- /dev/null +++ b/lab/iam/policies/labs1.yaml @@ -0,0 +1,149 @@ +Version: "2012-10-17" +Statement: + - Effect: Allow + Action: + - aps:CreateWorkspace + - aps:TagResource + Resource: ["*"] + Condition: + StringLike: + aws:RequestTag/env: + - ${Env}* + - Effect: Allow + Action: + - aps:DeleteWorkspace + - aps:Describe* + - aps:List* + - aps:QueryMetrics + Resource: ["*"] + Condition: + StringLike: + aws:ResourceTag/env: + - ${Env}* + - Effect: Allow + Action: + - dynamodb:ListTables + Resource: ["*"] + - Effect: Allow + Action: + - dynamodb:CreateTable + - dynamodb:DeleteTable + - dynamodb:DescribeTable + - dynamodb:DescribeContinuousBackups + - dynamodb:ListTagsOfResource + - dynamodb:DescribeTimeToLive + - dynamodb:Scan + - dynamodb:TagResource + Resource: + - !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${Env}* + - Effect: Allow + Action: + - secretsmanager:ListSecrets + Resource: ["*"] + - Effect: Allow + Action: + - secretsmanager:CreateSecret + - secretsmanager:DeleteSecret + - secretsmanager:DescribeSecret + Resource: + - !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${Env}* + - Effect: Allow + Action: + - secretsmanager:ListSecrets + Resource: ["*"] + - Effect: Allow + Action: + - sqs:CreateQueue + - sqs:DeleteQueue + - sqs:GetQueueAttributes + - sqs:SetQueueAttributes + - sqs:TagQueue + - sqs:ListQueueTags + Resource: + - !Sub arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:${Env}* + - Effect: Allow + Action: + - rds:DescribeDBInstances + Resource: ["*"] + - Effect: Allow + Action: + - rds:CreateDBInstance + - rds:CreateTenantDatabase + - rds:DeleteDBInstance + - rds:DeleteTenantDatabase + - rds:DescribeDBInstances + - rds:AddTagsToResource + - rds:ListTagsForResource + Resource: + - !Sub arn:aws:rds:${AWS::Region}:${AWS::AccountId}:db:${Env}* + - Effect: Allow + Action: + - rds:CreateDBInstance + - rds:CreateDBSubnetGroup + - rds:DeleteDBSubnetGroup + - rds:DescribeDBSubnetGroups + - rds:AddTagsToResource + - rds:ListTagsForResource + Resource: + - !Sub arn:aws:rds:${AWS::Region}:${AWS::AccountId}:subgrp:${Env}* + - Effect: Allow + Action: + - lambda:AddPermission + - lambda:CreateFunction + - lambda:DeleteFunction + - lambda:GetFunction + - lambda:GetFunctionCodeSigningConfig + - lambda:GetPolicy + - lambda:GetRuntimeManagementConfig + - lambda:ListVersionsByFunction + - lambda:RemovePermission + - lambda:TagResource + Resource: + - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${Env}* + - Effect: Allow + Action: + - lambda:GetLayerVersion + Resource: ["*"] + - Effect: Allow + Action: + - es:CreateDomain + - es:DeleteDomain + - es:DescribeDomain + - es:DescribeDomainConfig + - es:GetCompatibleVersions + - es:ListTags + - es:AddTags + Resource: + - !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${Env}* + - Effect: Allow + Action: + - elasticloadbalancing:Describe* + - elasticloadbalancing:Get* + Resource: ["*"] + - Effect: Allow + Action: + - cloudwatch:DeleteDashboards + - cloudwatch:GetDashboard + - cloudwatch:PutDashboard + Resource: + - !Sub arn:aws:cloudwatch::${AWS::AccountId}:dashboard/* + - Effect: Allow + Action: + - cloudwatch:GetMetricData + Resource: ["*"] + - Effect: Allow + Action: + - ecr:CreateRepository + - ecr:DeleteRepository + - ecr:DescribeRepositories + - ecr:ListTagsForResource + - ecr:TagResource + Resource: + - !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/retail-store-sample* + - !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${Env}* + - Effect: Allow + Action: + - guardduty:CreateDetector + - guardduty:DeleteDetector + - guardduty:ListDetectors + Resource: ["*"] diff --git a/lab/iam/policies/labs2.yaml b/lab/iam/policies/labs2.yaml new file mode 100644 index 000000000..908e42aa7 --- /dev/null +++ b/lab/iam/policies/labs2.yaml @@ -0,0 +1,116 @@ +Version: "2012-10-17" +Statement: + - Effect: Allow + Action: + - logs:DescribeLogGroups + - logs:ListTagsForResource + Resource: ["*"] + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:DeleteLogGroup + - logs:DeleteSubscriptionFilter + - logs:PutRetentionPolicy + - logs:PutSubscriptionFilter + - logs:TagResource + - logs:TagLogGroup + - logs:Get* + - logs:Describe* + - logs:List* + Resource: + - !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${Env}* + - !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/${Env}* + - !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/eks/${Env}* + - Effect: Allow + Action: + - events:DeleteRule + - events:DescribeRule + - events:ListTagsForResource + - events:ListTargetsByRule + - events:PutRule + - events:PutTargets + - events:RemoveTargets + - events:TagResource + Resource: + - !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/${Env}* + - !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/eks-workshop* + - Effect: Allow + Action: + - vpc-lattice:List* + - vpc-lattice:Get* + - vpc-lattice:DeleteServiceNetwork + - vpc-lattice:DeleteServiceNetworkVpcAssociation + Resource: ["*"] + - Effect: Allow + Action: + - elasticfilesystem:CreateFileSystem + - elasticfilesystem:CreateMountTarget + - elasticfilesystem:DeleteFileSystem + - elasticfilesystem:DeleteMountTarget + - elasticfilesystem:DescribeLifecycleConfiguration + - elasticfilesystem:DescribeMountTargetSecurityGroups + - elasticfilesystem:DescribeMountTargets + - elasticfilesystem:CreateTags + - elasticfilesystem:TagResource + - elasticfilesystem:DescribeFileSystems + Resource: + - !Sub arn:aws:elasticfilesystem:${AWS::Region}:${AWS::AccountId}:file-system/* + - Effect: Allow + Action: + - ssm:DescribeParameters + - ssm:ListTagsForResource + Resource: ["*"] + - Effect: Allow + Action: + - ssm:PutParameter + - ssm:GetParameter + - ssm:GetParameters + - ssm:DeleteParameter + - ssm:AddTagsToResource + Resource: + - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${Env}* + - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/eksworkshop/${Env}* + - Effect: Allow + Action: + - ssm:GetParameter + Resource: + - !Sub arn:aws:ssm:${AWS::Region}::parameter/aws/service/eks/optimized-ami/* + - Effect: Allow + Action: + - s3:CreateBucket + - s3:DeleteBucket + - s3:List* + - s3:Get* + - s3:PutBucketPublicAccessBlock + - s3:PutBucketTagging + - s3:DeleteObject + - s3:DeleteObjectVersion + Resource: + - arn:aws:s3:::${Env}* + - arn:aws:s3:::${Env}*/* + - Effect: Allow + Action: + - codecommit:CreateRepository + - codecommit:GetRepository + - codecommit:DeleteRepository + - codecommit:TagResource + - codecommit:ListTagsForResource + Resource: + - !Sub arn:aws:codecommit:${AWS::Region}:${AWS::AccountId}:${Env}* + - Effect: Allow + Action: + - codebuild:CreateProject + - codebuild:DeleteProject + - codebuild:BatchGetProjects + Resource: + - !Sub arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${Env}* + - Effect: Allow + Action: + - codepipeline:CreatePipeline + - codepipeline:DeletePipeline + - codepipeline:GetPipeline + - codepipeline:GetPipelineState + - codepipeline:ListTagsForResource + - codepipeline:TagResource + Resource: + - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:${Env}* diff --git a/lab/scripts/entrypoint.sh b/lab/scripts/entrypoint.sh index 2697798b1..6cfcf1e94 100644 --- a/lab/scripts/entrypoint.sh +++ b/lab/scripts/entrypoint.sh @@ -5,13 +5,12 @@ set -e bash /tmp/setup.sh if [ ! -z "$EKS_CLUSTER_NAME" ]; then - use-cluster $EKS_CLUSTER_NAME + aws eks update-kubeconfig --name $EKS_CLUSTER_NAME fi if [ $# -eq 0 ] then bash -l else - source /home/ec2-user/.bashrc.d/env.bash - bash -c "$@" + bash -l -c "$@" fi diff --git a/lab/scripts/setup.sh b/lab/scripts/setup.sh index cc426ac33..568c528be 100644 --- a/lab/scripts/setup.sh +++ b/lab/scripts/setup.sh @@ -14,9 +14,17 @@ if [ ! -z "$CLOUD9_ENVIRONMENT_ID" ]; then echo "aws cloud9 update-environment --environment-id $CLOUD9_ENVIRONMENT_ID --managed-credentials-action DISABLE &> /dev/null || true" > ~/.bashrc.d/c9.bash fi +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) + cat << EOT > ~/.bashrc.d/aws.bash export AWS_PAGER="" export AWS_REGION="${AWS_REGION}" +export AWS_ACCOUNT_ID="${AWS_ACCOUNT_ID}" +export EKS_CLUSTER_NAME="${EKS_CLUSTER_NAME}" +export EKS_DEFAULT_MNG_NAME="default" +export EKS_DEFAULT_MNG_MIN=3 +export EKS_DEFAULT_MNG_MAX=6 +export EKS_DEFAULT_MNG_DESIRED=3 EOT touch ~/.bashrc.d/workshop-env.bash diff --git a/manifests/.workshop/terraform/base.tf b/manifests/.workshop/terraform/base.tf index 5786d75d9..3041f4b45 100644 --- a/manifests/.workshop/terraform/base.tf +++ b/manifests/.workshop/terraform/base.tf @@ -2,6 +2,10 @@ terraform { required_version = ">= 1.3" required_providers { + aws = { + source = "hashicorp/aws" + version = "5.61.0" + } kubernetes = { source = "hashicorp/kubernetes" version = "2.31.0" @@ -53,6 +57,12 @@ data "aws_eks_cluster_auth" "this" { name = var.eks_cluster_id } +provider "aws" { + default_tags { + tags = local.tags + } +} + provider "kubernetes" { host = local.eks_cluster_endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.eks_cluster.certificate_authority[0].data) diff --git a/manifests/modules/aiml/inferentia/.workshop/terraform/main.tf b/manifests/modules/aiml/inferentia/.workshop/terraform/main.tf index 5b792c9f9..a995dadbb 100644 --- a/manifests/modules/aiml/inferentia/.workshop/terraform/main.tf +++ b/manifests/modules/aiml/inferentia/.workshop/terraform/main.tf @@ -22,12 +22,26 @@ module "eks_blueprints_addons" { enable_karpenter = true - karpenter_enable_spot_termination = true + karpenter_enable_spot_termination = false karpenter_enable_instance_profile_creation = true karpenter = { - chart_version = var.karpenter_version - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + chart_version = var.karpenter_version + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + role_name = "${var.addon_context.eks_cluster_id}-karpenter-controller" + role_name_use_prefix = false + policy_name = "${var.addon_context.eks_cluster_id}-karpenter-controller" + policy_name_use_prefix = false + } + + karpenter_node = { + iam_role_use_name_prefix = false + iam_role_name = "${var.addon_context.eks_cluster_id}-karpenter-node" + instance_profile_name = "${var.addon_context.eks_cluster_id}-karpenter" + } + + karpenter_sqs = { + queue_name = "${var.addon_context.eks_cluster_id}-karpenter" } cluster_name = var.addon_context.eks_cluster_id @@ -49,13 +63,12 @@ data "aws_subnets" "private" { } resource "aws_s3_bucket" "inference" { - bucket_prefix = "eksworkshop-inference" + bucket_prefix = "${var.addon_context.eks_cluster_id}-inference" force_destroy = true tags = var.tags } - module "iam_assumable_role_inference" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" version = "5.39.1" diff --git a/manifests/modules/automation/controlplanes/ack/.workshop/terraform/main.tf b/manifests/modules/automation/controlplanes/ack/.workshop/terraform/main.tf index c51341b56..d54a4ff57 100644 --- a/manifests/modules/automation/controlplanes/ack/.workshop/terraform/main.tf +++ b/manifests/modules/automation/controlplanes/ack/.workshop/terraform/main.tf @@ -35,6 +35,10 @@ module "dynamodb_ack_addon" { # Controllers to enable enable_dynamodb = true + dynamodb = { + role_name = "${var.addon_context.eks_cluster_id}-ack-ddb" + role_name_use_prefix = false + } tags = var.tags } @@ -68,7 +72,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } cluster_name = var.addon_context.eks_cluster_id diff --git a/manifests/modules/automation/controlplanes/crossplane/.workshop/terraform/main.tf b/manifests/modules/automation/controlplanes/crossplane/.workshop/terraform/main.tf index 76fdf7cf3..86094a6da 100644 --- a/manifests/modules/automation/controlplanes/crossplane/.workshop/terraform/main.tf +++ b/manifests/modules/automation/controlplanes/crossplane/.workshop/terraform/main.tf @@ -44,7 +44,8 @@ module "upbound_irsa_aws" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "5.39.1" - role_name_prefix = "ddb-upbound-aws-" + role_name_prefix = "${var.addon_context.eks_cluster_id}-ddb-upbound-" + policy_name_prefix = "${var.addon_context.eks_cluster_id}-ddb-upbound-" assume_role_condition_test = "StringLike" role_policy_arns = { @@ -128,7 +129,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } cluster_name = var.addon_context.eks_cluster_id diff --git a/manifests/modules/automation/gitops/flux/.workshop/terraform/main.tf b/manifests/modules/automation/gitops/flux/.workshop/terraform/main.tf index b7c048bdd..881a442ad 100644 --- a/manifests/modules/automation/gitops/flux/.workshop/terraform/main.tf +++ b/manifests/modules/automation/gitops/flux/.workshop/terraform/main.tf @@ -545,7 +545,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } cluster_name = var.addon_context.eks_cluster_id diff --git a/manifests/modules/autoscaling/compute/cluster-autoscaler/.workshop/terraform/main.tf b/manifests/modules/autoscaling/compute/cluster-autoscaler/.workshop/terraform/main.tf index e9d792963..150aca529 100644 --- a/manifests/modules/autoscaling/compute/cluster-autoscaler/.workshop/terraform/main.tf +++ b/manifests/modules/autoscaling/compute/cluster-autoscaler/.workshop/terraform/main.tf @@ -7,6 +7,12 @@ module "eks_blueprints_addons" { cluster_version = var.eks_cluster_version oidc_provider_arn = var.addon_context.eks_oidc_provider_arn - enable_cluster_autoscaler = true + enable_cluster_autoscaler = true + cluster_autoscaler = { + role_name = "${var.addon_context.eks_cluster_id}-cluster-autoscaler" + role_name_use_prefix = false + policy_name = "${var.addon_context.eks_cluster_id}-cluster-autoscaler" + policy_name_use_prefix = false + } create_kubernetes_resources = false } diff --git a/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/main.tf b/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/main.tf index fa4e672ee..b914858aa 100644 --- a/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/main.tf +++ b/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/main.tf @@ -13,6 +13,14 @@ module "karpenter" { enable_pod_identity = true create_pod_identity_association = true namespace = "karpenter" + iam_role_name = "${var.addon_context.eks_cluster_id}-karpenter-controller" + iam_role_use_name_prefix = false + iam_policy_name = "${var.addon_context.eks_cluster_id}-karpenter-controller" + iam_policy_use_name_prefix = false + node_iam_role_name = "${var.addon_context.eks_cluster_id}-karpenter-node" + node_iam_role_use_name_prefix = false + queue_name = "${var.addon_context.eks_cluster_id}-karpenter" + rule_name_prefix = "eks-workshop" node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" diff --git a/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/main.tf b/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/main.tf index fafee4603..04a445064 100644 --- a/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/main.tf +++ b/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/main.tf @@ -10,6 +10,10 @@ module "eks_blueprints_addons" { oidc_provider_arn = var.addon_context.eks_oidc_provider_arn enable_aws_load_balancer_controller = true + aws_load_balancer_controller = { + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" + } } module "iam_assumable_role_keda" { diff --git a/manifests/modules/exposing/ingress/.workshop/terraform/main.tf b/manifests/modules/exposing/ingress/.workshop/terraform/main.tf index 601a2d4a3..f0f2c7602 100644 --- a/manifests/modules/exposing/ingress/.workshop/terraform/main.tf +++ b/manifests/modules/exposing/ingress/.workshop/terraform/main.tf @@ -8,5 +8,10 @@ module "eks_blueprints_addons" { oidc_provider_arn = var.addon_context.eks_oidc_provider_arn enable_aws_load_balancer_controller = true - create_kubernetes_resources = false + aws_load_balancer_controller = { + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" + } + + create_kubernetes_resources = false } diff --git a/manifests/modules/exposing/load-balancer/.workshop/terraform/main.tf b/manifests/modules/exposing/load-balancer/.workshop/terraform/main.tf index 601a2d4a3..f0f2c7602 100644 --- a/manifests/modules/exposing/load-balancer/.workshop/terraform/main.tf +++ b/manifests/modules/exposing/load-balancer/.workshop/terraform/main.tf @@ -8,5 +8,10 @@ module "eks_blueprints_addons" { oidc_provider_arn = var.addon_context.eks_oidc_provider_arn enable_aws_load_balancer_controller = true - create_kubernetes_resources = false + aws_load_balancer_controller = { + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" + } + + create_kubernetes_resources = false } diff --git a/manifests/modules/fundamentals/fargate/profile/fargate.yaml b/manifests/modules/fundamentals/fargate/profile/fargate.yaml deleted file mode 100644 index c3e438940..000000000 --- a/manifests/modules/fundamentals/fargate/profile/fargate.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: $EKS_CLUSTER_NAME - region: $AWS_REGION - -fargateProfiles: - - name: checkout-profile - selectors: - - namespace: checkout - labels: - fargate: "yes" - subnets: - - $PRIVATE_SUBNET_1 - - $PRIVATE_SUBNET_2 - - $PRIVATE_SUBNET_3 - podExecutionRoleARN: $FARGATE_IAM_PROFILE_ARN diff --git a/manifests/modules/fundamentals/storage/ebs/.workshop/terraform/main.tf b/manifests/modules/fundamentals/storage/ebs/.workshop/terraform/main.tf index 59152013a..21e3a0272 100644 --- a/manifests/modules/fundamentals/storage/ebs/.workshop/terraform/main.tf +++ b/manifests/modules/fundamentals/storage/ebs/.workshop/terraform/main.tf @@ -2,7 +2,8 @@ module "ebs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "5.39.1" - role_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" + role_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" + policy_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" attach_ebs_csi_policy = true diff --git a/manifests/modules/fundamentals/storage/efs/.workshop/terraform/main.tf b/manifests/modules/fundamentals/storage/efs/.workshop/terraform/main.tf index 15fd12452..1e225b177 100644 --- a/manifests/modules/fundamentals/storage/efs/.workshop/terraform/main.tf +++ b/manifests/modules/fundamentals/storage/efs/.workshop/terraform/main.tf @@ -2,7 +2,8 @@ module "efs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "5.39.1" - role_name_prefix = "${var.addon_context.eks_cluster_id}-efs-csi-" + role_name_prefix = "${var.addon_context.eks_cluster_id}-efs-csi-" + policy_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" attach_efs_csi_policy = true diff --git a/manifests/modules/networking/custom-networking/.workshop/cleanup.sh b/manifests/modules/networking/custom-networking/.workshop/cleanup.sh index e3370e0f1..80913ff3a 100644 --- a/manifests/modules/networking/custom-networking/.workshop/cleanup.sh +++ b/manifests/modules/networking/custom-networking/.workshop/cleanup.sh @@ -2,6 +2,8 @@ set -e +logmessage "WARNING! This lab takes additional time to clean up to ensure lab stability, please be patient" + logmessage "Deleting ENI configs..." kubectl delete ENIConfig --all -A @@ -23,6 +25,8 @@ do aws ec2 terminate-instances --instance-ids $INSTANCE_ID done +sleep 30 + custom_nodegroup=$(aws eks list-nodegroups --cluster-name $EKS_CLUSTER_NAME --query "nodegroups[? @ == 'custom-networking']" --output text) if [ ! -z "$custom_nodegroup" ]; then diff --git a/manifests/modules/networking/network-policies/.workshop/terraform/main.tf b/manifests/modules/networking/network-policies/.workshop/terraform/main.tf index a98dc1783..dd0ce05ff 100644 --- a/manifests/modules/networking/network-policies/.workshop/terraform/main.tf +++ b/manifests/modules/networking/network-policies/.workshop/terraform/main.tf @@ -3,6 +3,10 @@ module "eks_blueprints_addons" { version = "1.16.3" enable_aws_load_balancer_controller = true + aws_load_balancer_controller = { + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" + } cluster_name = var.addon_context.eks_cluster_id cluster_endpoint = var.addon_context.aws_eks_cluster_endpoint diff --git a/manifests/modules/networking/vpc-lattice/.workshop/terraform/main.tf b/manifests/modules/networking/vpc-lattice/.workshop/terraform/main.tf index 2c16658fe..f39380cae 100644 --- a/manifests/modules/networking/vpc-lattice/.workshop/terraform/main.tf +++ b/manifests/modules/networking/vpc-lattice/.workshop/terraform/main.tf @@ -4,7 +4,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } cluster_name = var.addon_context.eks_cluster_id diff --git a/manifests/modules/observability/container-insights/.workshop/terraform/main.tf b/manifests/modules/observability/container-insights/.workshop/terraform/main.tf index 7989393f4..0bd35ac44 100644 --- a/manifests/modules/observability/container-insights/.workshop/terraform/main.tf +++ b/manifests/modules/observability/container-insights/.workshop/terraform/main.tf @@ -12,7 +12,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } } diff --git a/manifests/modules/observability/kubecost/.workshop/terraform/main.tf b/manifests/modules/observability/kubecost/.workshop/terraform/main.tf index a620cf90e..8d617efba 100644 --- a/manifests/modules/observability/kubecost/.workshop/terraform/main.tf +++ b/manifests/modules/observability/kubecost/.workshop/terraform/main.tf @@ -2,7 +2,8 @@ module "ebs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "5.39.1" - role_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" + role_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" + policy_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" attach_ebs_csi_policy = true @@ -36,7 +37,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } } diff --git a/manifests/modules/observability/oss-metrics/.workshop/terraform/main.tf b/manifests/modules/observability/oss-metrics/.workshop/terraform/main.tf index dd5b8be1e..b32e4da31 100644 --- a/manifests/modules/observability/oss-metrics/.workshop/terraform/main.tf +++ b/manifests/modules/observability/oss-metrics/.workshop/terraform/main.tf @@ -4,7 +4,8 @@ module "ebs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "5.39.1" - role_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" + role_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" + policy_name_prefix = "${var.addon_context.eks_cluster_id}-ebs-csi-" attach_ebs_csi_policy = true @@ -38,7 +39,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } } diff --git a/manifests/modules/security/eks-pod-identity/.workshop/terraform/main.tf b/manifests/modules/security/eks-pod-identity/.workshop/terraform/main.tf index e6dd8c94e..44fd60ac4 100644 --- a/manifests/modules/security/eks-pod-identity/.workshop/terraform/main.tf +++ b/manifests/modules/security/eks-pod-identity/.workshop/terraform/main.tf @@ -8,7 +8,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } cluster_name = var.addon_context.eks_cluster_id diff --git a/manifests/modules/security/irsa/.workshop/terraform/main.tf b/manifests/modules/security/irsa/.workshop/terraform/main.tf index 8c1a531e1..1067420fb 100644 --- a/manifests/modules/security/irsa/.workshop/terraform/main.tf +++ b/manifests/modules/security/irsa/.workshop/terraform/main.tf @@ -8,7 +8,9 @@ module "eks_blueprints_addons" { enable_aws_load_balancer_controller = true aws_load_balancer_controller = { - wait = true + wait = true + role_name = "${var.addon_context.eks_cluster_id}-alb-controller" + policy_name = "${var.addon_context.eks_cluster_id}-alb-controller" } cluster_name = var.addon_context.eks_cluster_id diff --git a/manifests/modules/security/secrets-manager/.workshop/terraform/main.tf b/manifests/modules/security/secrets-manager/.workshop/terraform/main.tf index 8a6f62316..c1e93da3f 100644 --- a/manifests/modules/security/secrets-manager/.workshop/terraform/main.tf +++ b/manifests/modules/security/secrets-manager/.workshop/terraform/main.tf @@ -44,7 +44,8 @@ module "secrets_manager_role" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "5.39.1" - role_name_prefix = "${var.eks_cluster_id}-secrets-" + role_name_prefix = "${var.eks_cluster_id}-secrets-" + policy_name_prefix = "${var.eks_cluster_id}-secrets-" role_policy_arns = { policy = aws_iam_policy.secrets_manager.arn diff --git a/website/docs/fundamentals/fargate/enabling.md b/website/docs/fundamentals/fargate/enabling.md index a7a78bb8a..cb626c681 100644 --- a/website/docs/fundamentals/fargate/enabling.md +++ b/website/docs/fundamentals/fargate/enabling.md @@ -9,13 +9,7 @@ As an administrator, you can use a Fargate profile to declare which Pods run on If a Pod matches multiple Fargate profiles, you can specify which profile a Pod uses by adding the following Kubernetes label to the Pod specification: `eks.amazonaws.com/fargate-profile: my-fargate-profile`. The Pod must match a selector in that profile to be scheduled onto Fargate. Kubernetes affinity/anti-affinity rules do not apply and aren't necessary with Amazon EKS Fargate Pods. -Lets start by adding a Fargate profile to our EKS cluster. This is the `eksctl` configuration we'll use: - -```file -manifests/modules/fundamentals/fargate/profile/fargate.yaml -``` - -This configuration creates a Fargate profile called `checkout-profile` with the following characteristics: +Lets start by adding a Fargate profile to our EKS cluster. The command below creates a Fargate profile called `checkout-profile` with the following characteristics: 1. Target Pods in the `checkout` namespace that have the label `fargate: yes` 2. Place pod in the private subnets of the VPC @@ -24,9 +18,15 @@ This configuration creates a Fargate profile called `checkout-profile` with the The following command creates the profile, which will take several minutes: ```bash timeout=600 -$ cat ~/environment/eks-workshop/modules/fundamentals/fargate/profile/fargate.yaml \ -| envsubst \ -| eksctl create fargateprofile -f - +$ aws eks create-fargate-profile \ + --cluster-name ${EKS_CLUSTER_NAME} \ + --pod-execution-role-arn $FARGATE_IAM_PROFILE_ARN \ + --fargate-profile-name checkout-profile \ + --selectors '[{"namespace": "checkout", "labels": {"fargate": "yes"}}]' \ + --subnets "[\"$PRIVATE_SUBNET_1\", \"$PRIVATE_SUBNET_2\", \"$PRIVATE_SUBNET_3\"]" + +$ aws eks wait fargate-profile-active --cluster-name ${EKS_CLUSTER_NAME} \ + --fargate-profile-name checkout-profile ``` Now we can inspect the Fargate profile: diff --git a/website/docs/introduction/assets/vscode-copy-paste.webp b/website/docs/introduction/assets/vscode-copy-paste.webp new file mode 100644 index 0000000000000000000000000000000000000000..01933604f5e40c2724efae2b32b797d3b086d304 GIT binary patch literal 9378 zcmZ9PWmFu_lJ*A+GWg&Y2ohX_yK8WF*Wm61C%C)2yC%2~?(PuW8C>4~z3<*VyW5{y z>h$TVetuQm%F+@N85973hPa5Lx*`{8%s+i}8mMe2T2h#2D1msXJXy-3!qTGKcDyT0 zL@T?O*LDz}`nkvNVZWJyH{DXE7KgYm;kLH@ht|GqkdN1?;%<}=t9STee{%inx2b7C zYQM_2+VrhSf9wy1*W5zi*Vpn7i+QHi-1F9FAwjD*D5miPR$iu8<6IQ^JZ+Yr^tVnH zr=v#w{;rPPY>@rBo$ijdtn}v_O^3~Enjd6mm3VOX)9LGuB&jOY&J&OHEZi0q1HEu) zFl>Qgfp692muE+YwZyv97t1;w_G7JC-hX|O3;_vPJx~-h0nkz^0NA(^03xZsX{R(V z)Vff``1Ai+B9jB4>SHwnXNdDoG?N%~Y6?li0ohef9wR)T1w#Lr2keH;l2BAH=(m74 zX!pRIG6qaEza+bGbD(N9DXev;>%H4OUTJ=#SQ`J$#T*R>&h$k4Wl<|C8Vi1lalwS1 z;bEl`_{#7!4z5f;U?R<^2p@4EcWdF>ITN$g)S%N5!%`qZf0PzP45|Y|sF4aQ)SOKA z^6@GLg6;qiXPEwY5W=6f+ek4AzHt{%?-lVe&FO;P=?lz80k-GOC;zKBUu2o!&|TL*!N}Dl^9UE| z(dHxt!g>>%(>PR%LYhl)pjdLFzXC*Koi#v=%gWk_bG?g_JnD3Za?_dp z@}};qyjo+EHv;ds(u{?kknq~`>6l4!6|$+NI7kX#*u-dVbzkboDE>%Op;D&pCW|eV z;2$UoGj=Jl=C4v~;?l5;p8gwB@TB}(1>cn`#E$~_ZRtg#m2BkMR5HgLks|Cg*~a|RV66Hi59J+h$T0=eVPIrK{&x6yOkCHyL@&YTvc}t4AuG^JXh)U3b;m<;Zqd-RgwgzmM1(8bXcxrkcmD-TF!A+x zHSkl8r3Ze-f8hCt5KNKO4jGYm&RmNC!8SLG>3@*=_iP$~B7_+r^|`pXO6`B2@V~(O z3{_1pSO3=EDDaP+^#3GN`9UJN67_POTb9duh#A0LwYbt4TK;1C^5$)$u^3d!kxGY( z;d;SI_yxB66lG;5E2ltBn=l_<>gM@MM5&rS(zf@i?fTWPxr zx`-2sDlvMJgG|Hl18VzEtrKQe*@Yb#_o%RNQYoKfEngaMcoK5BbOEv%K!H@_dozC(&w)-Dc<4f<1z1= z?>j5G|Jv{0I?|6xxOP;%!oWno@aELY715~owjRv$e)(URKy8=D(EejC2l>C!`*{AW ztrGCxJ2IsWc}~V86IXDCe>Wd^DC@$cO6|~~wEIsT@qd*{8_Q&-n>P@|Rd(|;swxsR zHxytJyFjMBpQ^ZCD)E)8;GQxN&N-6dzaD28|NoAMFd0Gmhd5B?DW(MYD_*!3do2QMTE&3zk+}Gk0bxC>b$c`Rh^Uu z%+zpKd`4&~-AHZ!GmHJJ7KR!XBZyxrm6GQ1Gu{q(g-xi8D~gSttGvz9`2hH;$MNsZzafvdMa|aAgG=Kp@Qua)XBa-x8#o^P;rodmIOAfcg7G`E4C|2DE2i5C*PF z2VYH7oqZA--v<1veVU@G!}3!arZY=M0KlYe69K*EOVF~&93n~@2m+u0GOa^os!tmt zQ7Gn}SV*##=RHIu_4ZDwG(y3&a){lGD2Xgi-7})IPjiF}dOdwP9^x#d-DEM?Ev)8g zF!iiz=5-PL+C+HsW5(}CPYD`f#<)k-3B~4nEUvu2+P&k^a6KjLY!$w59hFrasV^fF zbSG18#H7awC;MWx9AlnjD#1<_*kT1g558Xc&slux7jeK~#cV0VZiMh*hUfZE=>+X` zUwGvvVo-|Fd41Sz5Bij&+F~p}-5zCgiJ-u*J@u_kR^CHqn*!+-*kwbGiNx1LLt=5T zeUTvy3IP~XhA^is6@W7V?(3|_lySesDVT(WQI?_@QD zYM3;Qmz2$~b~AQ4AE~v=CZJeS11il-Twidp5R*HV0BOq_J`Tl{M$#B69MMc7s!&3) z%h7^H1PjlN+z#FssgwJ^vl)!xs{VO($y?r1s5@imGm}rk%!~N6Z^kYwhufbLjF_y- z%p3Y!*o}y|*%?Ih8Un-yIpZcg{H7D%qon>|Mw=OgD5xP<^+D4@{uZ4KDGoY>NSYD} zv3lq`@6U3%6D?dPWTfBkYA{(-;5~|5Ta=Dt&XjLp1GZDb8y9K6Wb+O4^SA!|5`Vm$ zQM?^qqGc#pjzor#TW$I`Z|!v5DOuc)D+KHWGMyM8{M-RM^G3il-G>U{(QAFVC5bE$VJvr7$<(U#X!0qq#ok82s~!+r~Jmn1B*qoxG`z(`%uecq>GRyoo#)>Y>=iO zr{>c8kE&FS8ok8HxbooaL%9`2J4f|W9WXmJ4Mlg96cv|eU(r6m^!5!yfnho&K&`L$ z+*YAje4%ogc4Vn9{20A-D@yd<)YdA@FOSM`5cdI!Ax(lbxV6!h4crtp0}R^YBOM6L zJhgm8bItoIFCNs40R_As3eYnye%lh%V*&el z(fZ$F?I@h$q0X?&I=?Nr5K?`^SU`IhU7$S^%`1gAq~FK=Q-PyZ;2>eS%G!?)I|TT% zQgvgRC5E(r1UB=Zie!oXDD)i|Exil_22XeB??Kspe16<;Ly!|&&cDGj{~ZJ_k~$x@ zrODrvGU1lHJ*LyCX-hNtrI2jmLlV0NEhr?2*NtPPl`DY(Y~js3Wo^#u^s)G!b~BdI zLn2?S*b+A{_rRpTQ>^8+17T4tJr%Ac$1^V>x5>&<#E2ID-8WHWWjly*bPQ5~A)pH(Pbb`9S)Ywiyz zlOEPSp{4eCf{oFapM~tRdBMc$Ia<>-0ulC=8DojP(66+AWc(hAgMBu!Ot#E273iC)srJVVq3EbFyRYMor=;J|VO5Aaedb>cyJpjKwaaK51!J zP@oYR_M?~ljfmxET|-hvmSV91Da}iYzz7B{Xm{+6fxK1GjnBxsdu$wxaa7htKBA<5 z_o~8pVmL1cVE|c;T)H=xMEE`kgzrzm@_$S;mbJ(37;=kpl3xc(`4{qGeDWBw|dBOBDU!+t1+EB(5#`$gEUR`0fp8F zom|IpEmVEel)8piDfSqgN+?97-&Rcw$)qepw>-YcRMn)GR={9V;thKbu*8Yg$!(6& zrBSS?p+|HFj7S(&I{=vjb>nc7Y4fj`G*qwgPdSn1$uF;*;Z~-iboB$I7W1s&+zyU; z(_B7P4IeuzeK#rLKpO9Wq2=fN@GRCA$v|s6dn02d=f`QL9}U#9smy~-C2mf5n)2N6 zh0RaDY!m~z_$Ud7p;V%qOMJ_$&)bkpleO1wbLp#|k?GBHitpInuB%&`OX@aPJ57$c zDlxNicL|n*_naYYl=CRT>f_2{>4GAmzx{T8$?$i~j5Ou5Cg=kB%31{O7Cu=t)+J?N zi#t=a2?N(v@N=`Uq|XRd9e7S5yB#!+%_bvtb&6-su(L66-bBz4Fq`o%sS<0O20Un0cFsth9}O{IwWD?f&pO?1td?I z#oes6CEarl@NMpLh-$tfVi7P(_2%lQt@rT1u%QlMpG`iGlifQtOiO@{aK`tPusfm} z*y>U{)c$;VodmNr;3n2#MDmV9=@4t_EVEwnn3x`tgH`Umb1s-Q^7y5V<}^u~(vl=I z4f1Px4XSULUa7yeU?s|ROscNO6F{ePSP}Lx0BZ#AC-A&k57>DqM0o;W38ZbdPB3S1 z75jnIYY2vQua^$Rzrs5sBCC~46v+rCU(>CHwrH411^_L5^ph;RImK|qq$1G0dpI9B*wE zVd2+h&n`COBNQUk6^MOGd6|$~V2}iL_RwHUnPXvcor#%m7rU&2Mi4b$+9@Y>*u9pQ zr;vpszNILo{4s#I7Ro*dz|g;!li!6Uvavz$z_$(NsiUx6ply$5_Wbk8hkyLs9u9!< zmoa4v#VJ|dK`iQY1E9>@MK;^;4RA!Rc71N; zLuZ<@56NzVr9*JJtL~MLPXJ+F;~*>w)cv&7X0^Zj;>hOoCljI~wTpRk&GLNeZ5Rf= z4aVbWN%IWVGtK2vq8YQ>*|_#tR%n^#BYe5E?(R`H*oKffL&8hdZ_Bw-2{`#&y2N~$ z&l^Z>CWCQ_ZhEqa4KWYxwcX?O4J>Ji0l`F4!RwBh zO&w=&EhNt2~YMZ%+ zn)SNAK__%l&cK!)8NNDtbHMN^!e0C{!-n5TzG>VBQlJel;`$ggilYb*LZ=OA&;~Ne z%ZusB9QEO0=k&`oq@8-hPcB!S=is~U%9EhO~M zU1_v3cMn7^YL=K-jrbGrIUo${(4SOgHAd4Pr$5M(3vNo`+;~*u8L#6)=g%Lle4M&B9TxOS$Fo;MVD>pUleAi+yAAFw2+uio(u|abXqMHn zP1<~o8LUY4K>72GpTD$Apri|5X9wXa1-3(*2U1J2!o`eVrb37=Ed6WnjeAeRsEsu9 znU^OsZqp!KjtPE03}!S2My^htizPHiM6D#ccUHW9!TXeWM~~nS+LL_SQ9~H^NVwZ0 zfYVQ&398F+81}DQ`ly>XVFMivkkzZGElW3McTCY#l)6BJYxE7O#jy zi?;xDbj3iKAGRE0#ckUktlKwAp~iMP7Xvmad(ADg*9;bLz3iv)6{u{se9S+nvlMoU zg=Hip+ug&gn6#lBH91pq9VzrKun9tWDRoEFq8w>Z*W{q@3u5rGo}^xGc4sk{BClA} z2i|&lV^9yn3ksJFMQ8rp8tlI27QD@EtpA)xI*UKs>=?Q1B;i;*z#H^l>L{EU zuyncGCvw$eqkprd@*9kH2_tQ*I#?5<1nnwkCP*s(m?(hL04a9iH^u4$Y&J5OU#wkb zB^Tc*;*JQA;U!ImZ8Uq|{dm8<3|igbtnai9eCd)?(_ZO~(2E!p#z^Ke-8VV}eZNOcAe{}dlP z`e1Qu?X0rNDAyI{b}w~PRg){OuX5Lxrg;5(2n;V-KLQw#+C@jwE|STO(~OdqaN`)>5IldRBR=GN5x3SpDeAlWy}dHsC9Kz=7|bU%{r!a z_tRrLG5I>IAaiE$T8m&jp40h-H8MRb;-<#VLi1j`WQ=sVhjnkwFA+<{s42<%UI4!Y z%K{An5MM6{oC}Obb`sUus*W5ZCGCudhSq!FdJmU4qFHAtyYwe9aW&{PJPHGO$r{@+qG7Tvci9VS$4;K1D7cG>Sb$?ggwX9F5AU2mQs3$_2Q(~_=&OzEl%Ms3P38Im8E}iKQ%bCb+A_di zQnZ|(fJ`|@7#DR51|Rm3@chBtr+lRIExiuwEpD_kGyLt+2iG=a$h$&a;;O?hl-}K7 z0|r1nn_gOJuI!Zc$O6ou?4gHQjFWc_**`4DTw9IAQqax>XRcp8AtGn>=pQHHhkg-y z3^rp78W+ODmY?&xC+c#$)VwlXa)g{m4j5rVSzsHxe+)07rq(l7@z(`6yj5D7@>%?T zH1_L5E&f)iiSG9B8%FUL$vw-&)HYxd3O}m)GZw$I^WCYUEFlX2rkA>9;0JA3w$MY% zCMdYp2LgOOIQEFf$|>AChRWKHMk$Yhi?6~L8#y`z}=Wuk8x$W2_;azq;`I-okPLF$3 z8q3YrS^Eo{rd8_tUiFr_I59csTTo<0o^U%!bJRwJHol^YYc^ToPt`w(<>K8mh_M4( zQYXN&<|jstbDckH;eE&S1b-e?qb}rCZY*;@G_6H!FsEvuSD-`yQ&&u}c~c_kv5^~l z^-CA#DQIFBpVM=VB#~|00#7zLq;g#$_gE!n9r%}D(m6Q&FDVqisFnmH3+o>2C?3$- ziq&5<;>-R*nME?uXh>zf*Yn!x!BX06Q0{t1@wJ=U^9N`9*uF$tI+No^eh*CPjwNd! z%84!)l<4c%tc6o-R1OFXBsab|_W*iD4CxSw-mm1_;tqsU zSuPYx(x00z1&^1NDE$wXaiTY6!omst#upW`fFCsK8 zS*Q9X?%rY!XHvF$>>JIRs}OA=>4qQdAHnJ>^+5?8NV%g7e(r#>`w=fk!_w{VB56-- zQUiO8sW{;gM^`h%pZ@!B1L-JoNOa6I(sg6R1jth9P8ieY;nnXgz*YgFqr3tRON}9B ze(GE62{)+6Ruga;(%Oa#x%rIZL89Q9RL_3qoaAI*yN;{jq(GO6DyUO1XD||)nuT14 zGB|chBqBoF7OjccvYTsica=)`v&!Tfe$y`nx$k((!%ipBaNqRZF7ZI?tczatUIvO^ zQ!A9!Hh-aeD#ViB4iz5HSgGfx|ElN^^@q(~ZGkt11fV|5LI9o?^dlJ6c1;L*AzEDF z@$|Gvi150iSf;vVw#H8DNMj#+#O?vT-{aOEWjEG0$S5KaXcEflP6N$QH2kBQ34S~2 z`EQ1!Bh}w)%C+S?*e;@(0zcs;@% z)m3w?X3c+pZ2Eu7d&*xb-GsR+7x>6sY{oes)?B)T96WZ!Rx)?*p;p8%S0Gl|#{nn| zDp1~4yc`^t#KBlB*Ng8nXqj(rsb!i-E>7GYVw4-dVb0U>Pd$T`T`<8M?FA;McRuo| z`gKgRb<2$M{Y-P7ysROx57?KjZ^kpIR{N;B>$eDnN@>_)c`wh5By4ACBqX!t)LO7~ z=b7TrXc|DK-8ZQaU(pthJ%p-E|*l9l};BX)tRPbz48~ z1tbsvsRn8M z zlIrW|(GQ>o=%mo&KDs<4Z;RMx|Ex`1#Zm@0zQYa;6dF_~sagk>VIm%z&wqw`d!;7_ z5>2irh=E9c#&4p4SdLeB)%=!xUcy*=Et~?L7zn=TxuNC-n?g}aPm&)LycR@=j0l(r zoXxc%G1Cr~#LbDY03vHkNO_UoyZ!j+r5!cmA-o( z=WdR?HXHu{3Y#q%cGgs+sa$2_CQ5T{XoQm7Kum=sd}`zobAY&EpbzcxEYaJ@#ZouT zcB0@jm7DR3_6?G_IgkI_o<#i=LYDxSU=wkf^THlB${I|h#og>h2xdDWS6{O7;jME3a`o>Lj>+6rq1 zFQC@etd5f96bCMZw*lGZl3*?Ow{_8J{JUAg{mV>7pD3G&RxPnM#`8~ + Once you have accessed the Cloud9 IDE, we recommend you use the **+** button and select **New Terminal** to open a new full screen terminal window. @@ -33,6 +37,15 @@ This will open a new tab with a fresh terminal. ![Shows new Cloud9 terminal](./assets/terminal.webp) You may also close the small terminal at the bottom if you wish. + + + +Depending on your browser the first time you copy/paste content in to the VSCode terminal you may be presented with a prompt that looks like this: + +![VSCode copy/paste](./assets/vscode-copy-paste.webp) + + + ## Terminal commands diff --git a/website/docs/introduction/setup/your-account/assets/vscode-outputs.webp b/website/docs/introduction/setup/your-account/assets/vscode-outputs.webp new file mode 100644 index 0000000000000000000000000000000000000000..0fc1b1e7b0bccf7ee3c3086b11ca408bc24cf59e GIT binary patch literal 17148 zcmZU(W3XsTvMs!9+qP}n_Fm>*wr$(CZQHhO+kU6JZ@>HDPen$}m{AxbM`l%)lBAf} zlMn!ax~PzXngR#G**~_O3t%=N6#;Mo5MR7Rj&yMmQ9*&?c~~71q`A$$^L+37PX^~) zR)iCA=P=yM!y4jlE<^P9>k@k!I7jX_yrQlJ|Gl@gujP6C@8A{o_v8=w@9&Q<@NX-w z1^%Ho^+%6~IeT-Z-Cyu@1W1}d*j>b41e;^?PvD)=Z581&BspQm+QB!U~lsG z(l`HW?gL-K@50aHTjM$X(CpQ>&@bCh_)qK5?APy;_jG6KN9+dwP0yZgpLgId_`B_+ zkLvH8Z`!l?lit1Fd++2Ips(o9{rRYGY1t>JPwTVsFZ-1G>d#<6EK67mcdc@k3a-|> z&pVA@wAFvxHPY@@|A+5~TeETFNj6q5-07;-Hv&XX&)?oTO4rs8)4bg=0q zs-l`H_@`JI6FYlc0(Lug0Ck$fR)p^P2{v_Nr8ef?`0Fs}9sJ*#Jc82=!IY#5i$8i0 zZ&OlSl{^o{Mdx-=2ksFBQ2%XD7xDzqD=<~I2oC9CBJNd z-4vfTX)JeQ;fo7gO3z0^g}e_!>F*ZKjII=o)b565*u7mkd+Z4aVwO&H>24XDd%z_f z9-XUkuC;jqQq9c5Xt<#d`Kc)ROeHscuoh3b7mNP3na131Y2QSCen@nE;V!(q^3jgth22Ozi93iFa3`?A z{S>uZfIrrGyIf@sd`y0g{?kn!1WBnTe-fvwvvNmLtjV7t=Es zT=HvMM=TL(^g1(on;pB$jo;(K?{($#zV`iSm$*^GOpFyAmt#!}lUBG(8m8bTr;=0ivs(LF><4NCOMdi`vfARi*C%hKab98>$g0yAb zG+ogI`kD!sMXM}M1-(y(P>o}f+!~`ZHv5d>aTHwf_gK)2WB|1=s-fk6vK`BrWnHpx z0FFC>7jY~8EaoUy#FfN@xSe<&_rH*HaV)tm`LtI!pM4CZwfq}0M`uS^PIa`s+Km%%PN^MC2lo**! zBVXnmy??>q%i&GuFGSQ1!4ul%7wH4RkznyrG4z$wr_lrq73KX{|1nAa>MMP6xu3J` zDW{-D`VhmpCWr!M9M0Kw*j&+q;<~m2$io{eUveOnd0LB>8~dC2qIdGI)ojAhaD#=+mmpSHA$9Q?yXXNZG9oCbkk`HNNtKWtHX>MQ$^^pn%xXm~@`>Q8Sm6_C| znnVr69(9Rm|A(mxcUlGg0@H{B!ZA}j?ck=FV_&95?2Zh){{MNGjR~#(F-arifd|sIG7Ov zTec?f8!B}r_H@XPZqyr{KU!Pfezf(Qt`q8?-V_sOxmwZcpp)CKV#O{8=&Po0+sv+390akYpF!R;u zfIuJEqQ+c>(7X8tv`xNdW*L8XKWlCfTtu@{{d}uSVz~&j;=-gde+RPrUYdPcyn`pBkbaQ-dJi~*t<+`Sl$>hr~Z0RNe zIk&d&KVqWSMLcnI$9xOn7vTZg;>lcDADHs?%P*{}H@9Xh&w|i9S5^mH%+{AJ4Ne>!fWq`7!#h=ZTAB7X+0pWx&-d7^NW37DUUpEoFYj1S}#Pkqk&PcYk!I0vrBO45Uc zgWM#WgjF@f$#;Xf-08ghM`r&KH>vf?G{Qs)Ag;3g_LAiQKf!0S@u|NExRi5`EaT38 zj2o%HU(*d>XutO1>hIEz(+ts$sw61aVB*}##C(M}(yi*?E}i!Iuj9ua-BYD=*X4hV zBuKirkvPNxtkdsw%HU9@7ZH9pkdHXo4oiQ1T=;G8Hpu@g`E;KM`XX^4EUZFJ285U+ zyh&kMbU75;MPl*XB!K@wXoKXbO;SC<>)8Gg^vZo?Eo3j($FvsR_fo0%k6I%7!GyB6 zF(P}&KoV{D%#}q+5MF9w`d`G04I^E)N{mDS^ApTWNq*iijyTs{ehV@126Y+Nlzk~cId62&YS}02cX8}xm8-2 zkChf`iIbkiKf^xUa;06;GSU$?q#G$6jByHajg};FgE$S!9C6qx}~G z-6QuiX~GsP7w1fs3t=RO&f#Rn9=CKEh=0--uHgwgB?IsO_1Lu#jQ^xZ{|TfA&;JJG ze{-Dw!t(z~eg4~>;s@~aE4P<}W_KrF_=x<28y|ewF!oIdxklry_UZb3CkCYD>q!tB z&%Gb}aVk>FjXWYQYvPT`9sx@6Czw3HyFP$Lh&hV|q#wTK4i%R%QQ2cR1qPsY3twRl zqkA=e!k6kVLQ6k@{sOP57@lm?APB80Q_|AS^hnBIi&jD4K17J?HnE&A-)5VKk+$#3 z{e$FjY}j*UR-zI1p6+~hjj8QbAf_GOlrq-Pn=O9+U#dAV*8vNmvzuGhw~|-=4c_wZ zociJ8+tGzclpYd1FILplzv%iYITa$yy8fZXLfVJ|r3b~O)lVTlWjUHTk1kY7dYV%aW0>P{#Y_@g zeDefLl@K{&UEsOO1rXo&#KpuY^`Pe6!lpe6fc1W@F+~wqcP@zVut>Hc?y_rr79>$s zu79GNceDrh5Zqz+Qa&*%Z_O^as#;vbq`!dZ$ef~Xtk@S@$mesEL8kE$iOnHai_X1m zxaRpgL>eVakn6^3je-htLG59RbJNHY!u)FuyUI<414CO?mpG==k1ZiC3eSJ%fkDD- z7JfP#qz8v{7uUa7P5H}ukgvg?BYn}4l#HqB$PD-De~HHj9GLm8`F8ytRPb~8qanM= z9)YV}UK?R=v6_1*L?a{Y&+UW8z*vJHT8>`g0Ip@Q1JtD_ya{#9D*%IbgbHI$P<+w` z8aEm*Z?U_2Mp(W8AlJ|7Oa_wiGzJ#hun9=JioTul*Neyrf%Qt4@`<>uc#zxi?c(z-lJ5WjYyTR(t1_4r zku$d@uZ94d<82(9@EC0RNLfQgJEUnxQI?37Ti&jd_%4cupAXV3a7H|X`aHjyqFChuZ%butD@rOEbD)CuSFzyBKHt?jh|MAut2(?}G2MfS(}*mr{$0w*_}TKH3KMx7Fu zMWHVdwyBg_y+lF*xWuG(S}pBJ!FO~hw>RUC6x8=4Xq$O7eesLiwQAhRcg|w zYYTfd`X2C{TjDt7=7j*a!(|L-L{kF+;o20P`A@z={L$E4$5Vt#kv(_D^`6zp321Vu zl`lwitp=cqo&E(@oFu%}13lbsF)0NUaiFaJH28a{7{rY?icM02@~z$ZbA(Qw!p>?|sg*SdlFpp+u3tsv0WC5W zg)rvQr>Ffo)#SX91)ox}6fa(HIdptpYi*I2SZLZpFZp>j6LOdO+o6SNqM_`%&9+ew zIku;vj5b5+oxM|Bx!2`lalnW|Ac&ZAtHyNYKI(L=x}wRq8>Pg}H$=%}9*C4bAth&c zKmENzleM5`a1i)9YW~k5d5(sTc26d?Cl>L6cCVO_ro8Tqu;b34^HdN!wit(n zEnb?Op?_Z!a@3%NvB*_Y11D|g0p{V4_fd0Hd zSIqw}I7I0AZM8T0FdsWhPMBsv4(fi%FqJR@B9i$hnc8GOmyx&)@Zs_>Gh=_xZpvxT z^Nr$m_@zGnww0m6V^uC1A#h$cNs5#iG(ml2I+exkZ#BgYDEfVW0Pz-DIR^p9?C%}| za2LiM82yiKZJuxAjP%JA(9ybtWLIfN1#3!XD@*w-KutTC55#M#s4rH2Mq2DCGt?Cf zc@~x*m8Db>ucRBA%2f?Gw)xpB@tEN0N&p7pcvPr0Z@1#Y=^l=hS{<- z{I*{MeJpshJ0W6yTiP*nR)1mH8#bsw6#6Bhi5X0$c_mfIYCl#`a2_fDNS|Ic-_#S zQU^AZJT$2dn`Rj}MEyR$d@ld&_DiAJhFM?00735&$eig*Bbr(XJF0vR^Hv(NQ`m- znzq?1lHY*VUVo81u5`*$0q7k~Mj5)r*+n)2TMu6I74iFG_r&jr-w?kbegGLp|6_ep z^RfT{0Cwx~hfWFimZVDX7RoK*nwY6R*@LuXH-OwwV`J}DtLxCJaBeVr=mg*i-4dX` z-3Xb8Z|vy@uaHC~oqji4TsuD@NgacM0cS^SVgRm2i!y2{RUIkexlaYa#s>!Azik5m)FW*_o(}W-p57=P0=vUjI>ejHWT&7B z5~NcD=x-ylM5|)Zd}G$jTqzMR^igKQtREvqzP~^Fki@IE^{WY`&=Ei19{}$t0>5jN z%wK*$XcjjQ5B+1=;QJ6m>FaO>?sm|1|3I-&02{k`5(LVIy`3xyxKu~HYM?UMPa3-|vERun+oZp9203!fWH&}U zRNP}+)DoOtTKZQyHAT@AX~EQSw7J196ZdEw`L&D4Xp_ETa~mE&yT2FkrZp&JC~~b- zcemw7$26IAZ75@u9V4v1$8`>8e4ZzT0M)g!gNNyP)4D#T`VH@hk{>h{*se=>z?9CO z0Z$pQ?2mV`Oi!vtdaWWA(@oP`XBYm+2oGQ^1R#=Pvy`)=9{_7L9wlK2UE{OAiaQg% zl-#E%-WUNymIB+WZ3L1IsRs*;%8Vp)+>ug#4MSzeCWxcwsTbw7=K&*4M;XJf^N3qC z=8g}7{(|MEWtR;gcuT#`^TeXS8Sry$HZ_%bTAk(elXJBO1O6OJL}xPht7;U<)8=j$ zBsJSC$XC0!cJvjJFcwH0>juZ@xT+;hpFW;!${^Ef6f@6!jk<@DQJw7B@+zDPrWmUF zGO;Anrxz+g-FEm$x-r&@FNtcW{FXD-g7x)sRgO_H2q9yYOiQ{ME6xu_w9dc~4n5OsfXw zC#xaB8@erv9G;7TjLV4pp^d4_W?1BMrGpulZs=^48*=PeD};ZF+p9MaM?zQcU4Eu0 zvcYJ-x9-<*R8|y3qhzyaZE=R~wOfj+cmagslj@^#N==YzO_Lgapd9e`q~Ins0faL` zrnKZpW9g5F2<%>nBuz}p>>-ky3gKh-66njOF_KOsFPx6jR+u2~ieB1i={;x_hW8XM zWWa#5g7(xOHoS@STk}OF79WDbI?FkG*gCkB7(%CBelc@knUVflGMQKH?RrPHKE%_r z$s(e3;6p7VjTnHFr>Lnf&C8fdSpr!G^-I=_Y12l{suj!s8cM8v=^CgN*trczsmTBU zP}m&+^W2R&*ya|deIh+37K#(Q%`m97?Q4*R;tgcX+<$M*t8O0_Yy}6nknURKW8qd& z@(R5SJO{qf%kEbnGiG*m!nbNA4Wx4!8cFWu2Mk5#D60%Gj*I?oH8IE_1g8A?+ukMV z4Tp4C6*ZXRoQ}u|$6kd(>236(O(3uCEIc3Adc&}8N5|Xc%H3kttVZM4QEJn@r=lt* zAn|SaBikbSvZ;?Y_KXuO?Ze(Es6Dlp~*a205*MU zE8@fryCwVt1tmsTa;GK|WuSIetZh*fMG!9Bn`LQ81>ko6w3Yj0#eynv%FMvGn*Gn! z@C%9M<&^?3lDgDZe&Wu$vCzlHL&0>go5H?AkE8@)Nqy7|;Au4}ZwaH=PjT9p?p~AN z`|{;Q>oBPkcN|n3WRd{^EgZqRtl4s5ak!;QCudZfBGfiIvUs(N*I6K9+;znEdu7Ch zq%H%iPtV?aUaE}>jfuf?;FHw|Moqn#Wc5N%9coELLiJk zqDx*gKpB{++h*B-4{jmZWJx}rU~O0umgC061ro6&hUY61m+G+tMpJnfi}I%C9$tp6 zD6`pQnC)Y3Z=4h z+LU3`<Luu{F#Vejt_7b=X&dcXRI~oT5AaXvvp# z)93K|5M%fICZRi%VRjD(?*xpjm(0lX=jp;uQ1SJJDp&yyq?2nE%KToRs2s+itbql{3CDb<(wD-5O$rLS*C1d^&5T>K#^WIaNgXT_yP>}l?* zwtjiwu@3j_;?g-m7JGw_3K<8_JpN9r17ck}rq~5(sRRT;%*2M&>VtaExuV|J>xadC z%o6GV9AtNtjCQMjJ;W3DZx_(t&|Pn97ibB61iaLq#d>@^?)xtkO$<<4^OU5u)^CsK z*P}I5fhoaia+849!SZNXA{fsWpyBQRs9U4ci!0aXyhc0t>Q^lhi=EQhGN zDJrkwRojX<;^?q3OuzM788p%Qs%o)*FYXQfjYiT1M9hRpI>b~$k*o7W@k5*;(yKe+ zNc0obXM)r1HWGGn%hrqO>MyxPJ`iI@pdFGy)lZjf4X*L*Y(UQr^MFEYgwq(*TBzp~ zatk8g3mR5r5!eb)3Ey0aB?)0NwJbno zTUp>4M$9_RN9%|NIV1Ox_6)wZ#E%I#WL&Vr7tez{dp^pOMHWyo50bGZnY@o~o=Hfx zlvMmqn>BovPuAW~FdfKGvEP;MgwBy0BZcv><@uoJ}DvOz8|2zdr- zMuk2V*7U;!MTJ|jUwl0``c#XU6X!0>+;!uVpMbw1z8w7pbk>y9*V^ygXHNU}yVXg6 zk~_J3`pBUfsGSVYpne0p1*MqbiA2>e>Z}|!c+EXZXHA;0)DnhQ zp&ECQP@8*{;THq#(kvt?h}mHg^j?#E$c6X4<3Mo-P-J(J0wjCXv8{H4_I!GYQLlxf zma-VQe5vSXw5@ljlPuQ3&1@iz@zK>&Dga zZM+vZ5I|lTZqm!H*>6U`znm+aS=nVHj1z}pB?r~<&ld$THM-swG;gcsXN!9MMcD-Z=>Dsfx<%K*@4N>882z4`j-3ZS^UcGHa}Snx5EwfgH+j0mD5&wO znkmeKm4%;r?v@>DxY=fqA>AL2TOAM17)|#gXm~ATTcnAjDZknGUEBL*(**|(@{2g* zLr*Mebs`~YNkN5=6U{NQfU>8{Wzdg6WoVntm18Ln??2=^0djTtk-i?uwO_<)b$@Yx zFpvk2lpD*|pga8{GCTZ6Q1th&P-Kw|-Vu|&`;MoI6&x}Mj-FD(fm|NF{2p(3n9z$7 zwA6mX9b@7}3wxbsRnH2~(-YWH0;Zr{g&UXoEvaB|nS%bO?m1S$bvQ79NkV+dap;vN z2&3NF&>$7illpJ=zKK^p{dnZQ?LKEdlNVClbzCquyHR#vM0%bF4Y%t~?OT|`z6T$F zG2eRMxO}p;v(=>0iB41dJOpsI9SeEuTrto2A{AanForG0n7oiU|8s za7!gQqj=4j=T!~X+q%7D@okewtJ278+ZN!|`h^R4)ZDLOiRom?+28K_s}y_Ode&|b z6O@DieLoX~Syr%~nP;DA7)W(EN|7}j5W_#3#2vB&X-rb?UpmZPfQ$%7crSh`=9fmG zz0aQ~%KWyp2tX?Ap94||zcv2=4Kb*{?p+BJ;637mXQ4wW+LP{$zD$d7<9 zH#QVYn0t_Wp@7g}YGT_+ov$Pl!C$yUA9&e<)EiuqSu(RK!-mNoR}HChUF&9XFFd)n zaG^-m3}0p|K?jeR+OI#bUaWJ_psgqrp1@X!O0zdT4(5mWhKPepT}nuL*bAp>&`CVB z1H=mT+e03PEerk`j5tZqfyQBj&83@R#@jQ8M)7}BjOY^NUo@kA>WQoDMyy0)a1fQ4 z=eM-pN6!G1d0RFPKF48-F`b~}pQV>j#1_=I4DL(vK!=|uBRX12-pedadkV+E%k)>S z#&QQL7yLUPgi|Dm*Q$^bd^oUdDecX+*+n-!__;yHK~TXFqiJ6DMRMCqI(L)x6c`85 z=8g_J`v%#Z&Q)LiJUHkPC#iP?=d-gEx^t6%gy%IWGPy1uvf>iE&a+cv{4J90m|7d! z1?;;=3l-dM3h@SEGOvpSJ*t(<2)8ENYzGlpjpHC5b{Af}ZJ>#WfT3B(sT7I0HZza* za>Cxt;y`-^;I@8k-cB+GpLr{P=lNitHws~A%RulEaHJP%AQQ(Q{(ATEj&n@hG4L!4 z$$M8^57KTwa7qY9@&v&%jW)Rw&;U|Hw$%X22Vh$Citjd!h5N7T$n@o^R?yMukqFJq zfC9>yoj6_sRE5ihIW?a$_{IHJ#)LZ&9EAl)vYY*6q-8I3!_?o93&#Uy=qQrpZZ`_? z)Z)K&P1$f|jxLI3`xs!ZQ{(2ItKs>r_NSmTR4A_P8-M~IMiOmBywR0HEp$s;#PpkF zZ@PIhKC9M~Sb1RQL-WW`#M{Nv7pCR|@e5dNjmkIvjeNjunGWDGB`1o=w)k;@2kVqe zKeFN}*@-=B9hg#FvM2GqVrjo3;gKiHNk5%DbA7c`-9#iIM2GR^lsrK`c}|tcRGSMJ z^qR)fqr8){eKyyo7Z_N!d1k2`{Q5j~kJyf0?kPIF3W2|gc)}4pM9+1?narI5B5Twj z@ZD|izuZ1PLpt>yrl_{k&nTmzxX8W(Jd}dgA2_=D8AaNnw1*8FRM}J#p_n$vUXmV4 zwx5?}Xl|J7s9VpGb4nlMaGy{U-C){@@`?dVf0kj`s=M{D|SU$4ZPB;H2*^&)&R z@+dsrD5}=@CEk9+5*-!(YvO2r0~>ANB-USy2dDQ`Cfq)n=H-p?%7N6_|pPctaFR zxuchvuH_^nGvin}j&k^`C^&Y?`rHDkvh>WlE4FMfdI%>kR10aGu#b?&^(USSv92=# zRv;NI{5LyVc= z7mx~&d}~=Kdmum>c=yY|I(U5OSObixQ-A#G;tTT+`py-<`euzHxUq=XfZawIJUTGG z2tB1OVfy|HuyD=}b@9OPSU3lb7MZXc&Tg-iU_7?-7oX`E+4HNK&f^Y@HTL6FTZ9i% zG%_nYl_@52L=uCk-Iddl8obp7hU})%@(@afAX~>nC(?${>6LkA0>ATuj54AQx^UN) zaV4c4>tCy!#-_bh32{es3h`mn3tsg0AqfpC6JeV;*mC?zTGWV@pox*P#5u?j&w?Z1 zhLcH0vm(@69nkk|3``f5=BUrXJz`&+9eDR5q58(VKDX8~DCy-mW~$9UK39O4L=I$O zRS3v&Ntw>^M!a{;Y3%-rVyBodnXpG*&lm=BG>bgsog8CS84!Up;SCRZZbYRa_6bY( zB#)BLN_S6Wh0im8+e2cd{aBVH!;82E^u_GyOS1LrB$~(Gl5vPDZhGdOXEIIJfmL}l zR1IK3vr^S2HQEE3`3hFVrk@loFDeSEuD8j-D;V5kE;qS*U@*(ecQPvg&#o1l{_I!e z$o$G1%#g^Y-dxx%1~WqqlKEjf<5cR-rr2P=a}Ez)B?R`FS_IZ9jB3CCZ5BdMAP6=C z8!1xG>n6r)dkirBr{bT=cl%4)cry)I@$kV<_wzY*YsZdP^x%SxLGK(#mMH#q8~+LO zoD}uO5}ql*JLLcJK%6;KCxmX!vA9;Jpdovj`LJ*6;?;|iJjFFX9&W75lc{V#HsO1C zs|!Dl7DRMw6nUZSk~-BCC5HeNI&W@o8dxAF_mC^Vd-)an3iiZC7eB*lKA#Nt_8c+X zQ4i+!c2He~<|D+&VqG~U_3ng6J_{YXQ^-HpN;282=1eV2PwvTOVWnmYkpzc+|1%Db zyo>Dh+go7c+#3I_H1)|h^KsANP(NWscV;!1!E624YD+7bdOC|+l=@f$`zIsu7UoFe z#C6i3IpF(Y7+833qiq;Q1| zPXbVBg@Iof$xR@c=b>bsN5bN)2G{Vz6lg_6NI_J`pfMpWZ1Uz*qiz#-MWZ+lz%Lza zHfsX0jPn9Db>!^ZJ%K1LaRW8E2SYxr?hpkAE0Q!D8s${%;@-Xdhl7;1dbb$Ak^L2o z!E%IHV}Yoqf+LVrUMp+0ro8`_MEQI7c(RO~U4w#~)T8QSzkDitNZ0?~_30s(+ZCVeQeq{K1RRf8lw&#lX8G zPJNU5Sg?>NUObHz?wytf=oRqI*yk{N;!Al0U#+|}fdh1*>ELo(A96pSh9{ur(6lL@ z>0ltbZ7PHXvSqc00 z65ocj`!aE9KI7!?YP~rI)jmmY__oVBnObcau#z1<6=mELnC-K*j93V|bS~>?d%#DEyJk^O;`#x+l(Gfye+bR%Nq3vH_giG9K}VjmJ(xek72bex$IG6A6!b_ zvH+X{vv$POc1AS8hM`kP&8s2<-M#%)k!7j?0G2QyLA%?G?);+!6ZlD!Xy2R0h9_O- zda5xT?rO=y7asGUVR-ZD;F{Bc_MGZFvMOL!goGbFISY4_`wg_Yp^IGZz=q^jFT1m4 zpsJ`J$|w!gH4qa+RMv-t)hwmmngNO(ADNH`%$i_iBDDyGR!v^Ah4BJ24ZWrlx|y*N zC4V?d=V6apj{#t^1#WheCVAR^gQ25h31Z(cK`0Q6>3hxG`Xj68f@ek}jI!!mDm^rk ziW6vur?8tOfIT#emf7YnQ$Ml@g*S&dkfO^WUwHfUI<<^zdp>Uif7lNnyI*&`Uw2`u z&j0`bF!p!6g^BcXLcxpHr4Hrrc+Ye#=Q&d?sEnktimF5Hu2Y5k6lrC4R)o5Lvo%xM zYvghP#f}46OWhM_ckSqNyxW4v*e$U>_6T28_I!f2^0#yO84*y0c6cAIqp;BD2Rrm8 zw1M5_kqSgm5NuoEWg%$H^04&I$ZWP#3tafPd2(Cv+J8Mt$T1pKFt8j~!K{r>J{eYT zcYT+_HA$ecW61RnFvCdm(& zY#>TWbib%7UIS93JX|MG>N?eAHUa`FXZ`m>Xo8ZeOcgdz4PdNS<#FdyI|gEx4Q6V~ zb|_$NbCRKH!lUBIfpfgk3Ov3oB{1~RwBodNqdh#}Hi;;5V?KOJoM>H6KrkQ$4>vAP z&99fXFF=)dPheC$e}eHxw@)PlQ$2oN1C4HdWs@DMjaSy< z&F1jU_As2TD?kDKca`(K1yjj_F_f8HH`L1%Lbne!!c%a&Nwhyket-F(o4ZXJJwp5-=gMXa#? z)B>}^zz!V31!Em?;R7EtGz?;Cl(u-Eq)^|%TWbS}XMy9al;Z^efJHfCW`DcvN5Y({0b2jcMQ&n_;;+(I??~c6htF3Zl(&*eF64lN$}4EBE24*( z#~7|*D{8L=>q@8bo2D+xS|aVCStW_rH><5rf^*zq-#3>-bPW#ZkpA)yG227N#sYGm zT>N*%H1pXnS_Ap)NAM@_chV|6BPerK-Umv@l}zVMKhKP6gRjw@w81I5kmL{a__gv)Tk9-92C*F`uBlb;SDJ_(XHl#@kL&T=E7((9d>1i`-0^oaSl9q`$ zV^^qxA7f(+ST$R1eJ=!$O5neF&Ts(6*@zYG)rY1oDJxkS`y(dJ4yu>R-cq@Jz@T?B z%K^YCdp$fpg4zKoMb6WUblk+(XllnCTcB{4GnuLM@tHdrP(EQ_T|)YHFN@7Ns*CEJ zJsncJ;A&}%3a6^?Y4w(*Yo!70Ak24Hb+Zxhpv$s;8IZ_jg!(I(S!>>-eR)E6f+?e+%+zNnzoR1uVGJmjf zBzrO&JHIKw!+D!9hvQJuG~4qDB%ut4sJB!lk`6AcExOBIO)M1tWVl+|2>aLrno>VJ zR57kKkITcQvX%-p0*kw$<#I9GWTIxu`Q5Lq%IQ>?7o2<^fczfiDO-lL4K=a^>F+l%JmNGI1W(a>sC~E3+=Q0(tP`XTh9~$W#1YJI z&xvv8P+~Sj%6-MW>-Yjerl+UfDlVwG(FBWtjtdB?u2sho+fAchQ6cgt%687nRzm~! zB>IFq-S4UACSIWf zn5}_E&vxDG3TzTMPy(`Lw6&7VxhNOAvI=CDCVDRIZ#+?n-W`R#yy2xRgTB)p5q3?T zwDv_F)r*-l<9{d~oyNgQ|ttvf?r%?$OGXM$3hTbq;9y1?%Q0 zJd&Qpc4RQ0kfZ%aTshd%&I0`aqel5pl9Y5p=^<)(cH{gN9m)cvN@0D;oiH5Jg<##( z*1AblRRz1-;k#Z*r2ydm@jv1fiBI-#dcjbpA_awbJlCK=0D>`@Sl5%crUG>v`o_C& zzm2OAf$gNF82jF<_R5T)YSe)QySybq+jK;(-ISQi*UCTj(8Q+#I!cV>PYsGt9@T7S zy|uqlJ23HXWun=wj3fH6B~T-1!UT=yEBxJ$5U*7W5R(jncU0QPY-6M-)XV^G)q6+& zg6?qi<=V9e1%EU&#`E`bL?PBtV4D|nP5W*EAt_)Bq9-rnd8HT(@i>qE5LOLW-bU~| zD_;cwbu!R_EvHjvXHd0flr}FIiaWf8LimdC9|)AC#Wli*@^-1L5RRt!GKjFlZAhtH zZc&JElmyL>VvP_8tw2g7OzM^x&$Kzk)a&>PNg8?z08KnXKfkv{Y->uuFP&d=pa9IQ z&9|toY1!!_vNNh|H==Bh#_!1x5yx7Ene2?8m*mXZCygEDR_lCw!wVsI#Mv|p*Jf-c zdbmgfX+!XI3}>^rK$bdBj?8|K#YepJePHQqlTFLt-0$Md^kYYxoQW4ugOdD783A36 zX^oCazIaW^Tng{K?=nOGRRfEt77u~m=`062d_ z!=&`5hwl>UT4Bs$F^72#Q_i3j8F=|_ho+ke$|Pt1HP-C;kd>Q6p_alI<1wkId$ie} zlE5x*BcWxLxvn--&*P205csXMr;CXJZrj;UJ~pWq6b0>*Bhz|`H^4v9W&&WF0otqWl!R}@#z3@bW@=XA(r)vnM_Ar~sK_g7b2NGI4o?+@JEs)QEy_NO>76Hg&QsR^O+8o>ZN;u_tSY z-M0ujK+D%&s%EH8-F^3PP*nD6y4h+Mi9gDrj~>>s+4 zWVNPqiMFYSp*V+G6pOz~!{M@=7?ty7vhN{7(M#u}t|L+0!3vr`OuTHX>+O*d9nKh- zdvSwl6Nct7B{v;dAyv#ylB-+vQ^l&+tmVVYUP@wo$9CWqDGBd*_fWLjA4QT=8C z?B}aY7p^VH*r&C}nJJHT#bQZ0G_suuzbQ{w6TZ~CPQ;wWcM%X2J ze-ij!A7b2XEl#jYc3_ccugb&G<&0Ol07dHy02%e=<-+=x z2bS~UjWU0pDS8b&6_DEk-MSd7Y7x1cQt{S_pzOKd{`4?8ut zPIq_()bKG2J=X*fI0j3EFkxYOyQ^|E3#)o!Ngg6N!_M|$m%F~${3 z=_2ttCm8$<^|IrTHwkxd1CAzsrG!SWdz9~({Y}2{Swv{ERrU}(5l2e&6_9MQdGzrbZmdc)?SY4PFFNF0;}7{P%YY^iPF)neiw?1 zv}2pdqyx+4Ds#ENJBYmo70jB;ZJ)1xe+(wUx(Im{PADwNXYhWL&iiRyVf-S`wp~ZDs<^w%6A(8lUdQLQ$AXeKDn& zMdUqpt@~LQnDv7hAvhgryel$M0E8FBeHG4Z(E|#G1?jB@^PYGuz{#bQ>ZQYo? zTY6i5JBbUf9PXVX{^xdcg`|DAdpiTUySBko0sssmK`;6w8S3}jTAY@^^U}dhFMTqF z)-uQS-K%_fk3PHIgM-y_NYWwoc-8(q|KB&(NGx9CZ{h(X4wzlpEYVam&@%ive_B~h z1EO?AW1i(KY+&TTu)!_Hg~yH6`Mwa8kc&+&-x?$k0rwzdVtg}i-bal%vu%*2XvSDK z#?$b!qam}uyB|I~ILeIlvo1dRoNoZmohC9(4E5)0?*w@*&=jTcl3wqjD&j8_e}bi= z*fIa9Ncr|zI#htCypYa;jDk{h67-1Hrgg0=yIJv}RF{Hx)yE2m#77?1+kHO#`eaUr z-#`on!bqbcF&mx+yJ8rdA_<=<_s({HO~YVN`A@-eeKV?#_(aQ-%*tb}JhLScLSpXg ztFC(**=p2Uc!XNzSVRH*Z7)@W-M6FN%D#?VY|)e`NgHjaF%f^XmH70lBl4RD$Tmu2;!~eBfo(4u?FkLLF;*`J4FwN!e^|SJ`nIKXJn@%Xr;h)6mKdbOeN+{M zvVhRH7f$ZD1DpK$BwZlL_Cq)lgrls}adsA0F>8o<+8gzZu)#)sT(9uE->tPCM84w8 zXrKS~Zz~#70r(w?#O2dyZ`zF2U+=X-yZaS7nbwf*g zVj=;RV!9L_@Qe}SZ;^+Rp!P+)K-`A+^I}7+k7~O`IEA{#s#SxJ2!OwpY<;cExx?G{1xS1{l)YcQ;i$I{3Zsbf^{>u(L(bU{^*w_; zgYMKLaENS{T2YOp;#|S5uOxI0+xYs9f4<_?&ZKP~l z3R#1r#6!f&hHOvOu^-^wcfmf|wvK_>bFM?TU&GbLg@6p1l8+DZktJj3+B}$a&L}w!rB}DHDR_|?DUGy%x zuzJ{##s7KUFaHnk%=_WY+&Ob*&i&23`sy%PDi8oLQdQD5)Rkg;|Ib`y1NcI~%SMy~ zkWW#|)+{XG$jz@~9Ipc1eeH^WZY;Tfca+vA>>bYa`}2FAm)pLEZ3>bZ9m|^&Z-Ld& z!xg4KkW4qBZfnYI(6uevG6cHfANo!jUyOG>YtO%l9;cXM&A#owIl3Xm z;P7sW^5wx5IKYkRwd~EalH1Kd=BLqfz$>9Ytn)|SN!X5y;BUW~&Y=fJojO=nGd+z= zvRa6L&Mv1O<3UxIyJV0^J)^)1*G*XVe28q8uhgLjl8DlC`>kAAJ%EcMa4tNag05(V zaJuVGvdqILo;5k*$QlkSWbbA+!Qhy2S8CdNL zAo`18f~Ae7ey{ja^jlP+{W?qIjDpS6{plRGN4smd$5)+J(zoMZC?r>j{s|@Su#Czk z1@1rc=Y-XA94fVQLwdV+ZY=32)~y4zgkmC3g#TmJqcBny+SU`c+BO_I|1EHwQv6#4 znvb#2mcGK~Bgx6_74lbiDmTn}uWGGSGEpULF)y5Vo|$O^57$-m>)=Hj5Gm+5Q@f-b z`D2n_^Cth`Oo5I+Ulb<&XCo$|!srMN+&X7=JoW1v6B)mf1A0n3DL%v}iG&6N!V2)j z9u@TLF$ET2DdXmODyxua<3dZr^%+rbZ{vQx_j_o{Lmzk?3?llfcCft!#J41LML2T{M>rhQ*Oz36^$OKduFb7c?QZSMB8d6G45 ztK@qg?QtBiXlnm_9)Q^H+%(fpbzwwUH?FEfe}J6SKXw{fEK1Uj8IXN>5ms*8PWur$ zZ1Vps|8{;@gp&#k+=SJN9yoXYYyA5p|2fge`z;N@JeP-eO+bPg|4W9ec;qg9~XJyY?=1*Qr)nXx|5(sl6Ph!N%1%&__1V1+M2!*k#wCG#x&2r+QJSn zOJb4qL~77xtf8{B9CEuaP~mYVLu`%}L;1;F60AwwBE*q|Equ~H$p!J-=P`Y)pRjq& zlU@%Woc0%TZ}eu9U~OQ_cP)sm2vKeUvBNhSJ_kD21911Y;mFqDzr0&Q;V+8iC~9r2 zOm>txuY?;%aQC@blu}7-=&8U-;(9Oi(kPZ4@CW|Sod322gATEFYXxa8B?uh}P3|2g5D?k~xRoY8(6;(a*_F6+ zp4W&LQ&VPnYag8=g3Czvh2CO4D_IVaX$;a`0p4bR-%b1&C~A0bYpo$x>r5#xU2}@M z0Z+qO4UpgGFZMQsZC6L`mQ6+ID29bl6uTRW{4HPe@Ed0B0`a&wwOEu|mRotr#n~2` zXsxrK`!^rqDs6x#-@Xvhgm)P@p#_rY|7NxO5<4si##xk2@cnYb2{n;f*0Il0Th6|E z279#a0TsxLN@T1Uo&9(-oDY7HjzbMIQ4UHsf3P5*IUk0gbb4Y%4G1Jl!uq|qi6A{X zI|9a7D0R7L=sg1{mF-;T5^yFlxz3$BXUBPV`sG+d?k9G+rKT9xRe!(2z9gfA26(~~_*{!`#L)(8k_pC{s^v$)ocik3!4OGp( zA45fsAE8*Xl6b79N7a~gO9}&kZxvQ98S{yvUeR@!ucM;6&6;9$sYZbQGC2!;Vx=W~ z1CfM9y{Tgk{A5&DYDg3?FaoY$srn(Z1l;B|N7KNZRm#;i8+cyxci5}6ZjZBmDCcYB z&*=Gr!5{mXV>UyikRh zHyC@KL@e>qgn;Ks7YD;YLL13ChZCr9bjW3l9PC&u7vr+rvbQ`7j_Ta35Ax$2Iomw8 zLTDW`7F5@tPv#4^x{#B^%|B_f2FqDA)BucnB3bx$89}lgjB4CJ|Fj+9`oam1NM0K_ zMi5jKO^z=4UNf~C@;nz5PP-UA!eyb$JG(tdod?CUU5PV&D`!rfE}L?hJ?Ar|j;==B z1vJ*ipZbbbOW-F$-js({K1>oK$UCgujyt|^NTSRS3cilsF6pNF;+*rw=cQ$rp8aJ2 z=>DF=yGlwW@pIG6Zs$nFOy@;|Ujcc2jODr~D>O37vO_Jx9!0!k51x=f=PC7dGZ60)&vry$!@S}jh634e#U>Qz@E^TkA@J}5L^3$)d3&-4k@cvL7{a73%9q(o%BzB#A*Rt9wf;Yod{&H$(FZ}k;oUIQel%d$T zK1Lek&d-Q9sBh6ssj~RSceB0GJj$G94#9mzAZn$f^qWJldi!YqHhb*jQVKLX6BW6O?`Y)LtBlMNPL;*GOBRpQx6^ zI^@C+X*Cn~X(sIg=^$oLu6f2TKI*uC4@sxQAoh)^Z~QpNmv=AmDjR~93|umn^npr( z+~p`fs77CV1+4>X2MAfp(I47%kW3qBo&4J(*EruNO=#YH7)7(o*RjCH!y9y-a^RhkoG)vh7AjX zR_~!lw&>uN=_CpX{;2J2vIuOB=bSRWl2`_$y#%KJU1q1-kY?kUrt!@cfXT&h$4 z?(V==U*42&!sJnYc~(+n!u31TaS3sbV}y!yK;9Z=;c>?9NOOQZ;)x^VHgrUDDe+wO z9FynO7UV{~Y}sZ%Pf)IG>Th~>8k^Na#cLQAUHL_}c>?Xs!0_Vhh{b-je5AjLP9UEL zFh+8zYr$WKpvULC*%8TVTm}n8wp8#EdPDy8`>Dj>j?JVtN9Zj~BHQjJ0yBWryezcC zz$4_{TM5A8{li%ccj+RA`-oWOseY~%4x1Z45XqMZni%kE8KtK8f)1}GnleZ{Cj~}1 zK00q>A7(d|A_-~Y5F2_CZ6Tt+%5?9Qg{=UF7)~_hD2SW`v^im0&O+^*tB->A>QMXv z6+F95^Hk88(a4KQ85@t;B9kPnzkMACyH~7}mF?E)u_4EXIWV4pv0!8QuO4F;gN6lW zx0WM|zt+^(?p{)BT!&xsj}vqQT8M427|S^1DP=SiJ4AREWBqN_&o%>oRs2=>=Lwy1 z#zF70*D@3Z`M8F7;6g0W&GuHg!6)R!OEZ$KOzF*XCRQImQ9GXB>GMP@A<5vGK>#{& zBWToePOMsXf^f>Tfty2e?fnwu4_Cw4Z`YVvgQHxEiq0+=hvSkXlx`~j8>b~_WF|5g zzXrPhy-t2jH=!RUQN^sKkM?qetE6Cemi_$H4|7EdCMD+!^b}5lNWak`uhoZE=Hxqu z&O76)D8r?+{5W+X%D;ui1z#71(iG?k-F(M2i>udR$>W670^`v4wc+MYy1GeV-hg=kZPh*?f!BxL3&hg zm41DBIP+P9fdV6EE}SXmY01b?EDU~2i@k8;%paVh@9xEIk(o4oeib5J^Nyb8)tDI! z49pDler2j5Fe26oG(5{>ZRWx8!nWq#v4e}sx(>9B&SX?qzeQiF&-JwbnyO&Rnarv8 z=g?-;S|D?8=LB<#{c_Cv1&sn@FQ0s0*QsRUvnN;~Jj;|RWNqR;ph{QwOB(7M$`l)2 zFaEsoV#4Y!Y`m&_)1b5|_u#A?S6P&K>?UQ*mo_14e#&;Id}@S`97aQ?`nrIkuwaL^ zK)K`&+>$6Q&pg7SvI6kPuRh{U2QgoERBDE04ZnBKA9hSu0lrJI|9=05pk4xs`2ko> zdFm4X@KusfS7E0%_E}7nH2ip3V*MI!afTHAYu9O}Sj5W#cDHS7D1Lihqknsl59O>C zeer8^aJvgAH#Vbv@}a||QlKSI)`!bNihQCwp{#1st~o;zYb^7rtr@mRNHW9_CExQI z#=Ke+yzbq+G@a@E@K_%I2Qybk%Q3slZ41A|GkguV9!#fJc~gBD9^V*Gyxld|yE%|~ zwwm?jjQjhyoDX{axuyN|o9!*DTkC6nouLk`pxP2XMyBSBNf_dr0alU_h{ zkaQ5!{ThfS(_$J()o7DP|EdhgNf-i3RxMhyd1`@H(MeoFT$r6|t1RWUlDd&{;g9~- z9bVP3IC?FxwT;rx^j;&V_V-mq_J_^l_~>2X!9 x!j@=X2h+Vj*P`#}958c+wrasw0D{KHfKAF;bw4Bem*decA&&<{Pyhhn{{Tqg*&YA@ literal 0 HcmV?d00001 diff --git a/website/docs/introduction/setup/your-account/assets/vscode-password-visible.webp b/website/docs/introduction/setup/your-account/assets/vscode-password-visible.webp new file mode 100644 index 0000000000000000000000000000000000000000..5dcd587682cd6df8aa1df9e1e3041fa5396445d1 GIT binary patch literal 5172 zcmZ{oWl$W-mVgI$7zj=X?ykWH9V9q}VSofD1ef6M8r*|haCb6jaDux_@Zf{nyZ7$a z*1p=e)jz&IKTaL5I^CLzU@$u&005DbR@YG%qNe`qUK0z*LSUsv!a$ISSIAXnD5RsM z3pmW?^{&sKlCG`2=A@3+6 z+PC(K?hdr7w(S=UfA>rh{DORidu?%?26%dV|MG_^1~So6<5O@A*jN{pzn8Fj?!s~A ze4G2dmkQgP=WO6VA;w0bOxHWN_EhTsIZ(I2mt~y6#NI7mh*qAu+2Kk{aMlWJyy_O1 zW3|zF=uwffaz2k+9_p($bO&Pk{g{&iuE!pTGOH*llp{X1IOyG?3-fUgI7^b6bM=KH zVyKk%#lCzaTS#A@WiRGa?r|#MT3h9r&+~% z#EA{gMpl^&M5@UCF;ORXiGSS;`LI6v>Qi7mD`pHO1p6^3@!~HRZg;;@i$V3-9*fUV zVcz8DIdXPkm6pl*IuUvd{|<{bQh58SpUczOy5ZlLzxj7S1p2M1$Ng3_Q+n=G^532- zN4XLy2foJS1h<#N1|*%?G%!8~bc&p393|3@6|{)y=f~c{3qA`*WI7i0mTej%u{bKM z%~e#qV`AR0B&{~xZBV9B=cEKxDe-D7WK^mBovOcG9-)Kmrnhn=S-w&p`z^nQ);mo5 zCiT6KA1*?14KZSu&Ybr3)b_#F$?`DD&EMf(uGt;8 zH{JZNs2!OSb-eYel{6*}-U6?UU$jsfrPvdV30n2y?h#K@FGE41wbze6enHXCjyih&RiQ~9eKk&7$CoNqkc^Y3MzIV}*8q>W1;>QzEA-ZIq#*rLC72VrPoIVS;&j-c4 z_R*>Px+v=^!5j64;4cl>1$EAW14>KO5y{|6UixSE_sqBe)CXiLU2jqC;uylA=`x^! z(st10gF5#9v(2+wcu>`7W=#CV%>h-X@6*G7VUxkT%NcUIE&!grRQLX$_WAFj`=|Nu z5-4s?*Z)!Zzqs%RJipYH7a^3@&TC6LSe8~b=itu;La4ue=mSJW)}R9rtZ4WFATRFD zCu__1dq;bnut~I1MpheDmOc-nK_lit=#!@`okWJ~F;OK}{v#zF9g)gns}E1=Lv3@s z*BmhE84Ny0q0N*SZEvvrR`b4q^3RNPZCs|}1lLzPvc%jJO<#kJ0Whv-c{7s}J$oPw zouwspCDPpLl!YDYRpf~iGvGf(h%&bC)?Z*pmhz_Paxf?G-i^74!78hQV_Lx3JX@D) z)Kd{5hHeM&=R%!2+QhR+-LtcYg*ic1&{H!|cr@gARG`6n$US*Br%1pSWwmQTf$yEC zjw2Os^lkY%ezXmNP!-I~xtb376G;*o9Kp@7qaY6B#`>gTJa5dAZ5Y+SQ17& za`aAe>+y_`%wYB!m#o?mWk@|A;&vOGNgQY0NL3l`7WTlmLAR_9n0Cx6_=B6Nq-zmt zFW*y}C%vNNnD>VM$Xeeyb>D20k+Ak=4RD9VVVCjM&G=SX8p`ZRSL@1{L(ry-*}TNI z*(XiaEiQv)wcQ_?D4J!ad9 zPB|RhOvx+CWvHgUKdbO&DhC{vr-&R}gC+&jx}*H)mrr(ZZV|_D8wUw1mPZa!VLvOo zb+#6iyeP0pF?>+If|QfWqvyRHX0? zjR$A=4aJ)Vpvw`>yt8bZgn%lk{Igjf5Rm}qlMJQ^(NL%kHhTx4AJ8^JGy?*TGoTtI zVkHzqo8dfs#Z~cUGwv zgRUh*G|$&_e4Z}9`DL`|tGOX9>-4=vZkqwAc5aqZ?^E<;%?No|96R0~5~>JsTf=7f zw{g}g*3b#M{P$2$Q{D$j{<;Y2Uv>wvguPu@y_5_BH2zGqYS{aj)$zv8f)lzIaD$nr zXqV=RyR8Bwb}7(Lj29r;aZ57V=@(`%wG zefQyj%^f+IvvELju}XX-*x#}(_u%>_kVduE%y@Auy#rEN)wf~J=SfqD27o*TWxvzh zO^a=tP&n&LGw5IwMhdEj$;(2Ru2IBcc!$mP>2i}gv6#|r>~!51X?vj{>-r(Fj4_(j z%pt1e*B7EW&$-CBILto=$7uG6pICZCBvpghx4wz^4U1D9G3*2|b(Y!uDz8*6&=EdjA!b-LsY97$}raMBvE??L`J0&0e^MGFGAi#(!_6;>H_|F|{qVY8#+#;`Ygt6!6t#d z+ORy!XD{3b7TXo|ip+UdlEw?5M|Jy1tfpb&bUC{DYUj`Xj{OQs`4AFfgf2&gUZ0V* zwOShR941iWRML!53D%|2zHZHllK}nP@+w|%n#9v0;nx=gO79K5kWTS+2{ErCPt7Xk3!aI? zv0D7escgqZ^+p~7;#Hc{$?m;I2-@5N5y9bTI$uIfzGubAeH*=>x^5 zt=8_6fvcT=PLm#c%}*6q5{=184eM{jU@|=&i+MYEkzZ7b@a4+O8radvOdpX;X89!r z3~U*fO&LbHy{#~GdRU#o88x?GcF7B>=p$ZlLTEiERa2CM^$AJN6`kli>+sDKf?OMZ zco6UsCLA3?1WOv}b>ng5meZ4?h(I>cH;j)Z|Oh zy`)K>H9F~Y+#cRI+u(_QqZiXK%8{YCAZ`cE8ew$gMB+o?QyLugj|=Lk9u}FX{6c48 z@1@xIAc--cd?CGVJ$6>r4_>Q%S+QSIw5LUKFTclT1yX2K;e^|0sTe4i79W&UK9vyO z9-IOvvKNKItjj|mM!%f)dF&6pnx_+%UAAU_-LMgCnbuPxuu7G-(98uc?xoC_6Z-uu(_0#QKI2kb;YH2#3&d59FUJs@w5nD#+33Mx&qMBsf9Ib5-t$3yo$5N$nCH{wrXor!WN!chK( zSdO@}r5!Y+N+n5;ulh{#9lcwF4kvodRV(w&vyW`=1H+VOBL^SYNuDBdJOMu}I8fH_ zru4{dm$BMlUImbG%dy))vAxu$Iu31ZWJP}`xyLO~F{gvLt` zA3S~C|GAZP^E8~tLm7UjC=vb0&x7O=;MhOMuVz>k4Ms=u8;4lOZ(+?>SWBN8q{gnt zwg^kyMc+XtW~*7BfW7UoXC30bR})X)#C{OX`47QcX38A94HkG1f*+yi#!39|D4eWw z;2e~vt%4>V`n;<{ilZj&VcMeIAK_UNQJM}LcGjmxss?!y$6DmCj62$@3p;3i4`j=8 z8>sv8>q8B>O=nE38M69@8h&&UPyJ}H#kehgAWYZR@8l)hmMKBF= zqs41lnqS~`N8`1VDH0ZG?=D8Ww%@uN9GiO?LRVV0yAc-b93y~6D#4z!wjG>F0xoaf z)f`Ci7OwY8wdzCj+9{-K%u+WncO~1s=-ZdV`kX$>YmL=}>4rt$LU+!WWHHq=dkf_7 zei}a5mKy0z{1|+FQ~FAnfBYzna)Rhog(VQr~wCN5Yk&joK_Kq6RmrSOnODv zmy~KN0*ss7y6b|o0++_GS`$vjye}Y&I4A^N_rfb18=A!OJx*-o!tn#aGbr=fgEXOXB$mMxCFZc0>Oz|B&-19VUkf@?JqpBj0C*YbE zOQPn-YS}g{KN5bl#FCI9bHV?JvKa@Yi1tIcuVfPq zB0xv5JFp^FrOI9+*0Q{m$6bYwf!7=rsD?~#FBsV*Oc_j~aqJJk1b(4Lpa!zKoKr^s I^^^eo3**WZ#{d8T literal 0 HcmV?d00001 diff --git a/website/docs/introduction/setup/your-account/assets/vscode-password.webp b/website/docs/introduction/setup/your-account/assets/vscode-password.webp new file mode 100644 index 0000000000000000000000000000000000000000..ca5c43dbc72ae9ba0c7d274b6234c34cd78a6766 GIT binary patch literal 5696 zcmds*RZtvUlZFS^Bp1wGG>MAPYKmfo<1dz1frbb2Xa~M}Iffv*?a3^5SYNftr2UAP~#{2icmX6A1SKRuGNIhH=JAC||?IWD4n{^f1+BzJR75syj3C_GzPFA;F zyou%+cY3}lk%UC9h?2AW1j`iTd7WAga&oUXn+Uw#k>1SdUUu z!esh-TF*+wLD*clbf06S?VlhA(M`oR5qR(=k1a`e`?NJC96>$8SKGcJJpyphf&EcO zC+2iwk@}jeamKPPGhUKBWg$!F5kAMqO!Ptj%uUPZ>ZQfaTzsXDHDcPLr&g*D-B3^sbAhz%Uvf2DSnGk=r3 z8)`5+As+DwrL$Nt;`tL}2kTC3IC}0EeDo;aU3A+TgWq2>Z%?FTdS|ML&&BmWbA6vb zW%;$g*0rfPL|nln`i3IF?|QvdZd0P~&tN&LzhXOnIne!j7le)sA1`&+A&MbCEl7!? zgT6<~0w$6ya~EDgIh;h4zq0)e%=v@C6@r(-5v<{Iw=Dh7MyS(L`(Z0wO@viyp-TI` zF|POd89Z62zuAhvx-01!QS%hkdrF+~0h@N{6cRlx_BVT2-JRkmLzbl9#KZRq)iYm| z?oL}@5T<^oeYVC)*ByDLk23edjADZQWl79Dl$?}fT35b3Rk!q5wJs;lfuS#?uiP7Q z1vvmeP)9yA{WrNt;QtTG|5uBu0sweG_b56Z!h2oVp}h7^`>7ncvY|_jZf@;M0e}{zNo|FS zOAUWNhOkiq=O{Iq6@=dG7JA6}f*c2SfwxQGj<29J(M%(qrg-A@k*iWo_L4ZkQZ=ZFPixm)Mf#qLn~21B@H4;1 zrhZLMsYVE=pIz!J-Vx-8kB})$&ZgkDE%2pQ--@CzyoUw}K7PKLm+u#6@G&10!{}u7 z75W3M&?trd>atXKkmD{w))_Ret$t*m{oLko@PYR&hru^tE9T-+&kh zD)QmzPtRA>XBydN5m z(kY1%kok@0KpCW%m)w&5U`&AiD$Bix)aJ%oM*Q(G{K5QvqD2 zM}2j4c`r9}50!%ySm)PR8)bTZI4pQhYL=vgl#O&BM&49CHW3t=0rS^Sah3DsJp@Vs zFqhX}iDgQFL8|*kJ3P4-$araKr@Y^YGhw!@hB38~Lr7BlE8F%ans+A(=1L6IQX)ngtW0$h&bU`c~;| zP;4Z4a}&5N)bs~6EWgjCtm1wINxt7b&F$H!@GwwEcUsIec2eLn0Quv@xVlPFQ0zlM+!^T^vbCHM6nk9ZoIrI zwiISc@yb_wwkh%AoHUc)^fOVbEN~*VSmFV;aEwkn>rKFHG%Z^4jU})_EJ6*hZ(Luy zpWG4+7qc^UHTq?MKG=LoAK)n|#nuhZ)QuA6Wc}h~=r|xff82nH+{yW-+4@n1GAxDw zP2S@2sR!c6^I^Y@`sW01q)VGG*V}i-tb^|+qOC=aqli5YL6%6P+9neBax!%WZc#tY zpvF&B`NLKU8+&TU@py7BS*Eb*iBe&8tF6Riex@ibHKDdCr^(_*wM~SCp)(3~&?hWU z3dd41heoF-mF+LHf(m5rq_{^IdBr^)Z*m2f)EY}WI_vD%I`Z+XoygUlcTa&rlLo{4 z)fjwgi58%U1Xar_>srU&Vj{LLe@^XO2{H|LkEv(VE~UdGKxix5IIjJEs>$c^Hs{+c zdQeNb?CDw_Ye>;k-h4`n6L@L7MeoN_^=Vb$M zrm-s8ibrWh63Z*0T_eQa+y#88P8-w_CT8;{yJ47D;DGlJ(63PNS+f#?R8c`FGQ(8b zua*AkAn$c;4m%cqm#_QEyMcAPU2G7}bSvWR$c|mWFZ(0qqghoCqiky#mxkCdVgQ-g z6t~=Hga}(XbKK3!8JI9PeHr(1suA$Xe2NP6qc{HUG1o+v|CL5{dTq)J7~8C|oT6Jg z=I+lyLh@Rvb0zxQ&E9C}+pOiC^2$jTmSq6*VL`=u$z>@S>%;#;~=!j|# z&XYd1@*1cl3XS>WM*0TjODzT>fW6~U-X^?r3oX)v0jJYl=coGK^XbvJQF$}rOoPG9 z@|^n$>ajE3a9Gad4=7Fl1L=bE!alWLOZzZ&>3vj*Agi{W72pZ#kqSQ9t-kW=fS1?k zijQN(w52u2lkve{f};#f&PD-~=T*n9`@hB^r-!AY!^xgK?HsWvLfMDjQ>RC^aC(87 z#e=IP93`ZhfNcrr~vwlHRaVskZplj*}iYbUE(I^}IOLE|}_#O;!s^`F!Kgkz$_n zc0{U_f~E7*wBWEPn=>$iAear-8-%hy34W8b-VIBO@luM}!T?iSC^ z`SFHY515j7@C$pfN6{RhQ9g2XKWe>Riu#RN zhz}>wK>jYQh&YmSr>hrj*6^XhAO_1H>c=j^8Fo~R887QW)X^wHn*hO#_c6%pFesA& zKQ8BqeZP3K2cQCPCvEJ#e0sB~F+=^@iv5?5{f4V2kZI;+AB-auz+Yqs*>B28DriW$ zx|d45>IS^3Il`TaV0`ES*Hq(G)o(J~KTuquUX+w9J)w4TJfOxkmSaBGZt(P=vFon- zTH(~>4?yIY*2Os!8icZe5!qjn3U%6n92$6HS}xJW?gJ;Cr4ORbk_8N$&5k-Ldf3e# zro*}PE{Pmf?o29o$vqtRst%5!_6UA^ycMA{tA?s2N@e*S|M=#sB`@-fvC}?L53xG% zW**zl~=0h{@$0gf=`GsLcOrdRl0;j>!T3jau1tUrNY1nK68MQ6nk;3Zoi>4d42G9PXHm;n3Vh8@Gxe|Q!n(@WMMB*FL^a!vy;U0<2NAt|~y+cG1 zvAEXZqkN+&Ny$p$9h@Bd#p9kzljKh;oKQkW)xafNl=!+N5yLiKK8-F|DW_Iu)WJyl z2Ryn>qkUbMvcRgCI&%jZ9vr{Lagj?_(Q_ZR$P=#`mBdl+cpWYiAs3={GM_3BR(tbE zvr?lHRTWw;IF4Fcl8~UB2xQR@DE4Pf%Amo^b;h2|i8#-_ve?GXK|#Ao*A3bXg5`j( zgrt12t%`PWJ9g}M0i>_9U9IQx|A1)29DB{B`tH7&Tts^@yY$+mw2q3x(z;M;$i*e|1@U&bK&S|1Bh?{jW(DL_AzXuT?-B*>< zp2}1_*t2P`v@JHn*ecgf1sBrVpcSe<#7MPQ&V!^z-G-0NtASiVW1xGmka%3ZuW|d~KC8-~52JI3YewU4}j9IK$ zENB9HTPL~p@zjK6qcs76s1&vVRh$CZ-6x3+$Rti>#ln;MXT~f7-k3n&t>z@GLncO| z36`o$>=!Gp!>=qKlcxM~6ojX2BpIX;vdv&;cO5gr(0L8fLHCnC*-U(K$w^QF*^jrB zNo{P~Q6c}3+97_UM?V7h$DPJD@Qz4uF~#bx{6jZY*}T8T?du(HsURA!R60ztx6c;q z(oO3OZUXn8pmE`&KVKlxdvDW2FT9?wCjmWj1wLG6Tm5!TduzChzKE4vC;WB{C7ju@@q9KQyRwq^vt?Z#UPMmP$@g!m~)V(zStKIRg$_f>&(b#u(xQuZn}n_2qshP z@PG0UxRFxYy>F@3ba?(lL%oe%Y`;VRTGD6F9RE{rRfe7XcA|Nn=Y8>WYJ3tGb`9-7 zWZXmkGEh0qgylYVR(CQqF6A>&aSy{ubp-rX5${Ft++0fe1x7 zUG9UpOIWbess1Nx6yMAsS?EyMhZaCo2U7>5c&uawY&5 zjrr}KP{zew+ygAUUf;yPbZ5>%t8#16sWnP7YbRW91VOC5biJc}M`ADHNQ&=#)+HyP zjEQg@3KXDFC`V~0sPSXj*r}kevs_mejtkusjf@otKL~P-BdDos51UmJN0|&A8#UMQ z)B{XOmF!c0TKjSuTBCcHRGsadmwpl&&Ue0-vQl>nn>Pz(IE`PEE80xAAE7-rIm(QG z!mb^yhlZ`*a-_Km3N9KK3y`zF*EWlgL@mn7BczKc-D%NW(qFD zIp(%9QXFRN)|9C|3BTi0aXUrgvMxPY_2}HzwbUNv&QHgHPR$?sZ5F;`X7qa|o^iN3 zrq%P3%n~1Ex1G7prjs4A9vw$uH>BP*9HvXR>1=DR3ZeKMJ+Zidfdg^19K^GyELbnt z+==~&6Nk6Vbn?rBxi!$`?1--_3_-1&a?}3ekgDIRFNOr=Rh&8(Usp5iPXE*6FIi<^ z>blRc-|XrSmvTQNC$T89EB2^4(OQ+|KJVHLCZr-4^fq@ZY3YDxD;wblKY#mtbkS^=eJ3U@;gn_;Ctb_0*WRwW-P`D001KfTY)+Rvgt01DV#73 gY&Xkkx&4Rkq!+zTTEZS6fF8y{rhk7p{EzWJ0IhH|Qvd(} literal 0 HcmV?d00001 diff --git a/website/docs/introduction/setup/your-account/assets/vscode-splash.webp b/website/docs/introduction/setup/your-account/assets/vscode-splash.webp new file mode 100644 index 0000000000000000000000000000000000000000..df06190c700da82c9baeeada3761ee615a638cb7 GIT binary patch literal 5074 zcmY+Dby$?^*2aepDPidD6p-!#=?*2NQ@R^O8U+LiNol0JhXDa;+@u0Bbay-CjKc@} zoc$g5``2?_>v`6Fuiw31Jyj*8ZzKSKq2er7L53Du+~E9PM*;(0NOa(Gen3t=&7fr1W0uwy&JzLLfXWKIZ($s56Bv3e$ zmpR3`CzGx|{p8im`x3SmfN=J(u5|?7*=DN0wOKUg!!+f#U%XUjtr*=Lr(z+pE*YZx zE1<*rD87|<4(zISbNNJ1?W?>C`{RgxNdHK{nLtKczK$7z+x~Zdcm>$%&N-z)vN3() zFYRId#4#EW=@o6ArI4y^qjea=g>CI?g>tyn>gko*!-XCkN^h;={BlSo@zV?b03!bDawdTg>P$b*_t@! zXM_6|eZ$x|SVQBCUiv+1g_-G`;)as7iw7#q{?q!Gr%8?!*8huN2y0$=_?U)Y+pIiRUjNS9|J z{$L9LB-kn5z#cKomSbiM0IJn2cz*rgtoIv4wgX}7{faq?a*;8EdNv$hdM-Hy;tH2y z7Zgx*<*xatRFyTl+2P>h^0&?Z-*ufA34&}w_1Q8CKCN zttd{;F(HS9yG+DA3~3#1lKKT6oo^ogv5m89#Qh=U7EOIqPWab{a)9MbYUDXY1{R?; zRvXNnH|izOu_|pI001EGTAv1-ooLKIxvTxL2}d5CN)4>TYTU8v12kI>d=e)Z`+o&io7C@Bjg)M!|r02kjaxav8$-kDoflwc9^Fo}L%9vM^ z!;V2OD1<6X)^J^_04}Prbq6>f?q;0&(C2>5$)S2r5Ke(fE`q>GHgQa1xNHzQlH12t zZ82_?qT`+JzT($CXm8@EuTBScz#~c9HJt$6W%v7 zY~h4u3K(^tZY*f!LGO!~?I@%V@P3upvPLxVmZ*1{CU6@q!;AEfMpP^Vs&03fdUp~E zlbNW=rQ;=N_^;W0RiS@TGc1v;W2xsa$?-r3BB|w%)MA`KbIXDI~502q>KbIzOfwocwaLz18 zB3uf|!zmSs<+m1B2DAJ*{)&pJM(@gM=1EPwj6xz6Wlma5HrC5ptxvj4b?Ui4^)!1~ z90%-YC{LNfT)%yODWe1{BMo0fk{)Iar%!h6fcX-C{jd$7IY33368n?_fa!;0W3HMj z-0FKD@}1P8(U0cz!h1rGc0RVG`mi!Z8D|NwbfnQFh`T7QVtpa8Z}f9S{5Y^n(%rF9 zFeg~pF##vf+myZT4+dmFRWN2k^-uJxxj~ru)}oOWH+uec?>TYC(vGNN{%FNB-$l zN*iKE=Vr^@T~(YW#;oI>0K`PSMLdOqRH}q^*PCC&gLk2B{Kn=H1!!e;3)N5wu+VH565#1zR{nB=ndgDUqh1(N!$@Tful z&IBn;k--@=8ji+%?dR27*k8^_p$5vg;A{zNuV8c%e)OAXl;VbdDVOe#V=(vJ@RjhN z)YY=_9`_rp9hMsvN7P$TSv0OxJsJ4E#M+uK8+Z-)7jKjsi zsg|m~Gi{u7y&w6zeIZJ*wEQc@+JYRmBvGeEpJM@&NIdU{OB~-|iCOi0W#`LjqTsHf zfj!qbJ=M5nh}@uN0kr|Q$}5)!V-YE*B1II{x6+@$?g`^EQe>k)teI3rgaQf77QW9Y ztR&=;YynHZMYHJpQA=Y)Q@Wmv4>;sRzI#eMm{ne$SA)B`%tvrH$hCGwsi$FP@G(CR zPOUX`u}3B2!W&pXIGiIOF!RdqY>OtS7!jwe$}#p_N%F?WV;9NVu1ZpLQc#JQG8C$= z9on|^Z1UL?Kx+tq+Y%;;39Ynr>rnb7K%+Vek8xlBc63s+#mHduVjLyP=#RHswPAj0 zOK;n7{0ZW+semK>3V4k z4P?bh5=r$i1E;;VG;QA|DHP7O=(dg=5p49AZSN&}OiMP^bKtn{vuIholw^;q&WeuP z2M%6evai#4;(j6Pul{IPzj{JT$t5o=yeL=2ClS&1acCGukIM+F(xX=J51mp{cKbEVOC-=a7t&s2) zFsFn?GJ-X0c#H>_)UK<+CFht^rfImN-8)qPHN!|+yo1CuwV>=g>z?$=joY8=!!=aj z28uP&eRbNrmHM{6jesZU&K=KQ35OGB&KHQLGs*@vq zTxaan=-@@FtNWIT?7nh#%KO z8yOf2Hy24`x5&yGD^wUe_=tnQ=-!K$+2v<8Mzsb<1nU(epdh}~4*V~SX71rPgF5lX9L(%%`RL3z&n{Mkm)KKpw8WWL(7n$*LyfnQ{XB9A*bo?pS(KAzySEH7QmU;_GHCK{^n z#7OO(JleEkkk~fn>!e$3m5czp9H*}Yy(X4ee8dGi=8TUA1Aja}-~!_gnH zpM)&Nb!k>V0{{RhLSF-(vG*!zGrXpReTMs2h4l8mxVqI+cd*pUqb*RU?-B!LL)?&* z({yTqJ>IaG^+pQC-Nf*V7!wWA*J#sBVRO8$L%R^#8aH2#pVspZ;Up=zE*}6Jtd^upEHz%yHOcE3~RIl%((s}f* zjNKr$29A=lkFcXHF5`-u@LY<} zmhep&J@Z0ngF+&jO+j!ielN(L{yO`@@f^TOg4;4fo_+f^lQD*QEtA)wn{3=G>t}1@ z)jXP|3l6vUW9AcRp;cVuwk`oouU783;-@J?Zf4afxX6dr)`hL=s8zlcX8CSYy|R!7 zO*r#Np^iaWEVR|R*r}n+ljvQ2>vu!d`C%0OeCx=@ROu3Oh=3>gcrxm zO*y>gmLy+g{@TFIJFPy3OYZx~Wi;$;K6&7S*QX)U;dIN1Gt^%gLq8+!}fHt>* zR(;1NYV+U~kM;&DE)yzxT>;rMMg7u*!Q~UPqw}&EBJt4+yyY*U@{0A?YtB_uc!iUp zQ?nhKv7f~+$IBC~DY@|U94Z>K9Q4eE-0?|q%;=~bx@%nc-;S$IPy+76v9emAn@+E+ zX02n5u^j0>56hwk;0)E3ei9DXDw#X@>67fv+=WK(nU_NUW4;bdE++h5;RPGb-QD$< zHAeBDnw>sRJ1p+3fS)b#S`IfD2sp_abh~Cam@my{tx!7%MhA+qOusct_-gNW(M^=$ z3a!dNJ~qzJr3qF~+Omx<(h1iiRI>0~jx){Y8xM7SO58UX~TV149+#V34 z?nn3KvBBF)TT~LFeoJi^C$S+n)U*kfJu>3EC^<)$fq4OrWllTTP3taA#M;4D7T@2? z2TmjQP#N9Na?0TO+|)}MNl}*K3K&{7@#B<(prdZPR2@ ztw{JyNyKd#R=Hr>UJ;b@(Z`+_(Wd0?$LNC3UB!P(_OZ7K_fWJa9%pSc=9VjSeQJT= zhe-pSv8MK>wo498`4Je|k8TcFF#;=4O>|9-V7e#k&I<0qA9uv J-=ARs{2#>-xw!xU literal 0 HcmV?d00001 diff --git a/website/docs/introduction/setup/your-account/cleanup.md b/website/docs/introduction/setup/your-account/cleanup.md index 20c0f4ba5..6204c9525 100644 --- a/website/docs/introduction/setup/your-account/cleanup.md +++ b/website/docs/introduction/setup/your-account/cleanup.md @@ -12,7 +12,7 @@ Make sure you have run the respective clean up instructions for the mechanism yo ::: -This section outlines how to clean up the Cloud9 IDE we've used to run the labs. +This section outlines how to clean up the IDE we've used to run the labs. Start by opening CloudShell in the region where you deployed the CloudFormation stack: diff --git a/website/docs/introduction/setup/your-account/index.md b/website/docs/introduction/setup/your-account/index.md index 3877da221..3fed9a532 100644 --- a/website/docs/introduction/setup/your-account/index.md +++ b/website/docs/introduction/setup/your-account/index.md @@ -3,24 +3,48 @@ title: In your AWS account sidebar_position: 30 --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + :::danger Warning Provisioning this workshop environment in your AWS account will create resources and **there will be cost associated with them**. The cleanup section provides a guide to remove them, preventing further charges. ::: This section outlines how to set up the environment to run the labs in your own AWS account. -The first step is to create an IDE with the provided CloudFormation template. The easiest way to do this is using the quick launch links below: - -| Region | Link | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `us-west2` | [Launch](https://us-west-2.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-pdx-f3b3f9f1a7d6a3d0.s3.us-west-2.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-ide-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF) | -| `eu-west-1` | [Launch](https://eu-west-1.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-dub-85e3be25bd827406.s3.eu-west-1.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-ide-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF) | -| `ap-southeast-1` | [Launch](https://ap-southeast-1.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-sin-694a125e41645312.s3.ap-southeast-1.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-ide-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF") | +The first step is to create an IDE with the provided CloudFormation templates. You have the choice between using AWS Cloud9 or a browser-accessible instance of VSCode that will run on an EC2 instance in your AWS account. :::tip + +After careful consideration, we have made the decision to close new customer access to AWS Cloud9, effective July 25, 2024. AWS Cloud9 existing customers can continue to use the service as normal. AWS continues to invest in security, availability, and performance improvements for AWS Cloud9, but we do not plan to introduce new features. + +If you cannot access Cloud9 in your AWS account you must use the VSCode option. + +::: + +Use the AWS CloudFormation quick-create links below to launch the desired template in the appropriate AWS region. + +| Region | Cloud9 Link | VSCode Link (Preview) | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `us-west2` | [Launch](https://us-west-2.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-pdx-f3b3f9f1a7d6a3d0.s3.us-west-2.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-ide-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF) | [Launch](https://us-west-2.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-pdx-f3b3f9f1a7d6a3d0.s3.us-west-2.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-vscode-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF) | +| `eu-west-1` | [Launch](https://eu-west-1.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-dub-85e3be25bd827406.s3.eu-west-1.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-ide-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF) | [Launch](https://eu-west-1.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-dub-85e3be25bd827406.s3.eu-west-1.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-vscode-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF) | +| `ap-southeast-1` | [Launch](https://ap-southeast-1.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-sin-694a125e41645312.s3.ap-southeast-1.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-vscode-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF") | [Launch](https://ap-southeast-1.console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateUrl=https://ws-assets-prod-iad-r-sin-694a125e41645312.s3.ap-southeast-1.amazonaws.com/39146514-f6d5-41cb-86ef-359f9d2f7265/eks-workshop-ide-cfn.yaml&stackName=eks-workshop-ide¶m_RepositoryRef=VAR::MANIFESTS_REF") | + These instructions have been tested in the AWS regions listed above and are not guaranteed to work in others without modification. + +:::warning + +The nature of the workshop material means that the IDE EC2 instance requires broad IAM permissions in your account, for example creating IAM roles. Before continuing please review the IAM permissions that will be provided to the IDE instance in the CloudFormation template. + +We are continuously working to optimize the IAM permissions. Please raise a [GitHub issue](https://github.com/aws-samples/eks-workshop-v2/issues) with any suggestions for improvement. + ::: +Now select that tab that corresponds to the IDE that you have installed. + + + + Scroll to the bottom of the screen and acknowledge the IAM notice: ![acknowledge IAM](./assets/acknowledge-iam.webp) @@ -43,8 +67,44 @@ You can now close CloudShell, all further commands will be run in the terminal s $ aws sts get-caller-identity ``` + + + +Scroll to the bottom of the screen and acknowledge the IAM notice: + +![acknowledge IAM](./assets/acknowledge-iam.webp) + +Then click the **Create stack** button: + +![Create Stack](./assets/create-stack.webp) + +The CloudFormation stack will take roughly 5 minutes to deploy, and once completed you can retrieve information required to continue from the **Outputs** tab: + +![cloudformation outputs](./assets/vscode-outputs.webp) + +The `IdeUrl` output contains the URL to enter in your browser to access the IDE. The `IdePasswordSecret` contains a link to an AWS Secrets Manger secret that contains a generated password for the IDE. + +To retrieve the password open that URL and click the **Retrieve** button: + +![secretsmanager retrieve](./assets/vscode-password-retrieve.webp) + +The password will then be available for you to copy: + +![cloudformation outputs](./assets/vscode-password-visible.webp) + +Open the IDE URL provided and you will be prompted for the password: + +![cloudformation outputs](./assets/vscode-password.webp) + +After submitting your password you will be presented with the initial VSCode screen: + +![cloudformation outputs](./assets/vscode-splash.webp) + + + + The next step is to create an EKS cluster to perform the lab exercises in. Please follow one of the guides below to provision a cluster that meets the requirements for these labs: - **(Recommended)** [eksctl](./using-eksctl.md) -- Terraform +- [Terraform](./using-terraform.md) - (Coming soon!) CDK diff --git a/website/src/components/Terminal/index.tsx b/website/src/components/Terminal/index.tsx index 735152f2a..6e88346f9 100644 --- a/website/src/components/Terminal/index.tsx +++ b/website/src/components/Terminal/index.tsx @@ -16,7 +16,7 @@ export default function Terminal({ output }: Props): JSX.Element { let sections: Array = []; - let section = new TerminalSection(); + let section = new TerminalSection(0); let appendNext = false; @@ -27,7 +27,7 @@ export default function Terminal({ output }: Props): JSX.Element { if (!appendNext) { if (currentLine.startsWith("$ ")) { - section = new TerminalSection(); + section = new TerminalSection(i); sections.push(section); currentLine = currentLine.substring(2); @@ -40,7 +40,7 @@ export default function Terminal({ output }: Props): JSX.Element { } const handler = () => { - navigator.clipboard.writeText(`${allCommands}\n`); + triggerCopy(`${allCommands}\n`); }; return ( @@ -80,8 +80,11 @@ class TerminalSection { private inHeredoc = false; private commandString: string = ""; private inCommand = true; + private index: number; + + constructor(index: number) { + this.index = index; - constructor() { this.context = this.commandContext = new TerminalCommand(); this.contexts.push(this.context); } @@ -127,24 +130,37 @@ class TerminalSection { render() { const commandString = this.commandContext.getCommand(); const handler = () => { - navigator.clipboard.writeText(commandString); + triggerCopy(commandString); }; return (
- {this.contexts.map((element) => { - return element.render(); + {this.contexts.map((element, index) => { + return element.render(index); })}
); } } +function triggerCopy(text: string) { + navigator.permissions + .query({ name: "clipboard-write" as PermissionName }) + .then((e) => { + if (e.state === "granted") { + navigator.clipboard.writeText(text); + } + }); + + window.parent.postMessage(`eks-workshop-terminal:${text}`, "*"); +} + class TerminalContext { protected lines: Array = []; @@ -152,7 +168,7 @@ class TerminalContext { this.lines.push(line); } - render() { + render(index: number) { return
; } @@ -172,17 +188,17 @@ class TerminalCommand extends TerminalContext { return this.lines.join("\n"); } - render() { + render(index: number) { return ( -
+
~ $ {this.renderCommand(this.lines[0], false)}
- {this.lines.slice(1).map((element) => { + {this.lines.slice(1).map((element, lineIndex) => { return ( -
+
{this.renderCommand(element, true)}
); @@ -199,9 +215,9 @@ class TerminalCommand extends TerminalContext { } class TerminalOutput extends TerminalContext { - render() { + render(index: number) { return ( -
+ 
{this.lines.join("\n")}
); From cc232c37c6f2ea3ce451e4a92a16b8e5040e7072 Mon Sep 17 00:00:00 2001 From: Pranit Raje <46365177+pranitr@users.noreply.github.com> Date: Fri, 30 Aug 2024 01:12:22 +0530 Subject: [PATCH 02/32] fix: Add cluster admin when using Terraform (#1057) --- cluster/terraform/eks.tf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cluster/terraform/eks.tf b/cluster/terraform/eks.tf index 276818318..f3f4afc72 100644 --- a/cluster/terraform/eks.tf +++ b/cluster/terraform/eks.tf @@ -2,9 +2,10 @@ module "eks" { source = "terraform-aws-modules/eks/aws" version = "~> 20.0" - cluster_name = var.cluster_name - cluster_version = var.cluster_version - cluster_endpoint_public_access = true + cluster_name = var.cluster_name + cluster_version = var.cluster_version + cluster_endpoint_public_access = true + enable_cluster_creator_admin_permissions = true cluster_addons = { vpc-cni = { @@ -56,4 +57,4 @@ module "eks" { tags = merge(local.tags, { "karpenter.sh/discovery" = var.cluster_name }) -} \ No newline at end of file +} From 8a3b4351c1d2e74de5c4f139f2c8968593f97a67 Mon Sep 17 00:00:00 2001 From: tzahimizrahi <5658093+tzahimizrahi@users.noreply.github.com> Date: Thu, 29 Aug 2024 22:43:05 +0300 Subject: [PATCH 03/32] add internal elb tag to vpc private subnets (#1055) Co-authored-by: Tzahi Mizrahi --- cluster/terraform/vpc.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/terraform/vpc.tf b/cluster/terraform/vpc.tf index a387e13f2..50f1bf0f4 100644 --- a/cluster/terraform/vpc.tf +++ b/cluster/terraform/vpc.tf @@ -38,7 +38,8 @@ module "vpc" { "kubernetes.io/role/elb" = "1" }) private_subnet_tags = merge(local.tags, { - "karpenter.sh/discovery" = var.cluster_name + "karpenter.sh/discovery" = var.cluster_name + "kubernetes.io/role/internal-elb" = "1" }) tags = local.tags From 27a1d59c7ff265ed7329b0375242891b7006b069 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 29 Aug 2024 17:32:30 -0600 Subject: [PATCH 04/32] Fix default branch value in VSCode CFN --- lab/cfn/eks-workshop-vscode-cfn.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/cfn/eks-workshop-vscode-cfn.yaml b/lab/cfn/eks-workshop-vscode-cfn.yaml index 5b664c477..569e06a19 100644 --- a/lab/cfn/eks-workshop-vscode-cfn.yaml +++ b/lab/cfn/eks-workshop-vscode-cfn.yaml @@ -16,7 +16,7 @@ Parameters: RepositoryRef: Type: String Description: The Git reference to be used to bootstrap Cloud9 - Default: "vscode-ide" + Default: "main" ResourcesPrecreated: Type: String Description: Whether lab infrastructure has been pre-provisioned From 3e5ab4a6586f0f686c0c72ba1dfdaee3c730740c Mon Sep 17 00:00:00 2001 From: natmhnty Date: Thu, 29 Aug 2024 19:53:16 -0400 Subject: [PATCH 05/32] New AIML Lab: Deploying Llama2 Chatbot using Ray Serve on Amazon EKS (#1053) Co-authored-by: Sai Vennam --- .spelling | 1 + .../modules/aiml/chatbot/.workshop/cleanup.sh | 43 ++++++ .../aiml/chatbot/.workshop/terraform/main.tf | 60 ++++++++ .../chatbot/.workshop/terraform/outputs.tf | 8 ++ .../aiml/chatbot/.workshop/terraform/vars.tf | 42 ++++++ .../aiml/chatbot/gradio/gradio-ui.yaml | 130 ++++++++++++++++++ .../aiml/chatbot/gradio/kustomization.yaml | 4 + .../k8s-neuron-device-plugin-rbac.yaml | 59 ++++++++ .../k8s-neuron-device-plugin.yaml | 95 +++++++++++++ .../neuron-device-plugin/kustomization.yaml | 5 + .../aiml/chatbot/nodepool/kustomization.yaml | 5 + .../aiml/chatbot/nodepool/nodepool-inf2.yaml | 57 ++++++++ .../aiml/chatbot/nodepool/nodepool-x86.yaml | 56 ++++++++ .../ray-service-llama2-chatbot/Dockerfile | 37 +++++ .../kustomization.yaml | 4 + .../ray-service-llama2.yaml | 130 ++++++++++++++++++ .../ray_serve_llama2.py | 103 ++++++++++++++ website/docs/aiml/chatbot/_category_.json | 3 + website/docs/aiml/chatbot/add-llama2.md | 86 ++++++++++++ website/docs/aiml/chatbot/expose.md | 57 ++++++++ website/docs/aiml/chatbot/gradio.md | 68 +++++++++ website/docs/aiml/chatbot/index.md | 35 +++++ website/docs/aiml/chatbot/intro.md | 25 ++++ website/docs/aiml/chatbot/nodepool.md | 70 ++++++++++ website/docs/aiml/chatbot/tests/hook-suite.sh | 11 ++ website/src/css/custom.scss | 4 + .../theme/DocSidebarItem/Category/index.js | 5 + .../img/sample-app-screens/chatbot.webp | Bin 0 -> 87314 bytes 28 files changed, 1203 insertions(+) create mode 100755 manifests/modules/aiml/chatbot/.workshop/cleanup.sh create mode 100644 manifests/modules/aiml/chatbot/.workshop/terraform/main.tf create mode 100644 manifests/modules/aiml/chatbot/.workshop/terraform/outputs.tf create mode 100644 manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf create mode 100644 manifests/modules/aiml/chatbot/gradio/gradio-ui.yaml create mode 100644 manifests/modules/aiml/chatbot/gradio/kustomization.yaml create mode 100644 manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml create mode 100644 manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml create mode 100644 manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml create mode 100644 manifests/modules/aiml/chatbot/nodepool/kustomization.yaml create mode 100644 manifests/modules/aiml/chatbot/nodepool/nodepool-inf2.yaml create mode 100644 manifests/modules/aiml/chatbot/nodepool/nodepool-x86.yaml create mode 100644 manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/Dockerfile create mode 100644 manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/kustomization.yaml create mode 100644 manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray-service-llama2.yaml create mode 100644 manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray_serve_llama2.py create mode 100644 website/docs/aiml/chatbot/_category_.json create mode 100644 website/docs/aiml/chatbot/add-llama2.md create mode 100644 website/docs/aiml/chatbot/expose.md create mode 100644 website/docs/aiml/chatbot/gradio.md create mode 100644 website/docs/aiml/chatbot/index.md create mode 100644 website/docs/aiml/chatbot/intro.md create mode 100644 website/docs/aiml/chatbot/nodepool.md create mode 100644 website/docs/aiml/chatbot/tests/hook-suite.sh create mode 100644 website/static/img/sample-app-screens/chatbot.webp diff --git a/.spelling b/.spelling index 60490db15..7c0e0539d 100644 --- a/.spelling +++ b/.spelling @@ -126,3 +126,4 @@ mehta sheetal joshi keda +AIML \ No newline at end of file diff --git a/manifests/modules/aiml/chatbot/.workshop/cleanup.sh b/manifests/modules/aiml/chatbot/.workshop/cleanup.sh new file mode 100755 index 000000000..5079728ba --- /dev/null +++ b/manifests/modules/aiml/chatbot/.workshop/cleanup.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +logmessage "Deleting AIML resources..." + +logmessage "Deleting Gradio-UI Components..." + +kubectl delete -k /eks-workshop/manifests/modules/aiml/chatbot/gradio --ignore-not-found=true + +logmessage "Deleting Llama2 pods..." + +kubectl delete -k /eks-workshop/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot --ignore-not-found=true + +logmessage "Deleting Neuron Device Plugin..." + +kubectl delete -k /eks-workshop/manifests/modules/aiml/chatbot/neuron-device-plugin --ignore-not-found=true + +logmessage "Un-installing kuberay operator..." + +helm uninstall kuberay-operator + +kubectl delete namespace llama2 --ignore-not-found + +kubectl delete namespace gradio-llama2-inf2 --ignore-not-found + +logmessage "Deleting Karpenter NodePool and EC2NodeClass..." + +delete-all-if-crd-exists nodepools.karpenter.sh +delete-all-if-crd-exists ec2nodeclasses.karpenter.k8s.aws + +logmessage "Waiting for Karpenter nodes to be removed..." + +EXIT_CODE=0 + +timeout --foreground -s TERM 30 bash -c \ + 'while [[ $(kubectl get nodes --selector=type=karpenter -o json | jq -r ".items | length") -gt 0 ]];\ + do sleep 5;\ + done' || EXIT_CODE=$? + +if [ $EXIT_CODE -ne 0 ]; then + logmessage "Warning: Karpenter nodes did not clean up" +fi diff --git a/manifests/modules/aiml/chatbot/.workshop/terraform/main.tf b/manifests/modules/aiml/chatbot/.workshop/terraform/main.tf new file mode 100644 index 000000000..7587514f3 --- /dev/null +++ b/manifests/modules/aiml/chatbot/.workshop/terraform/main.tf @@ -0,0 +1,60 @@ +terraform { + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.14" + } + } +} + +provider "aws" { + region = "us-east-1" + alias = "virginia" +} + +data "aws_ecrpublic_authorization_token" "token" { + provider = aws.virginia +} + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "1.16.3" + + enable_aws_load_balancer_controller = true + # turn off the mutating webhook for services because we are using + # retrieved from Data on EKS + aws_load_balancer_controller = { + set = [{ + name = "enableServiceMutatorWebhook" + value = "false" + }] + wait = true + } + + enable_karpenter = true + + karpenter_enable_spot_termination = true + karpenter_enable_instance_profile_creation = true + karpenter = { + chart_version = var.karpenter_version + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + } + + cluster_name = var.addon_context.eks_cluster_id + cluster_endpoint = var.addon_context.aws_eks_cluster_endpoint + cluster_version = var.eks_cluster_version + oidc_provider_arn = var.addon_context.eks_oidc_provider_arn +} + +data "aws_subnets" "private" { + tags = { + created-by = "eks-workshop-v2" + env = var.addon_context.eks_cluster_id + } + + filter { + name = "tag:Name" + values = ["*Private*"] + } +} diff --git a/manifests/modules/aiml/chatbot/.workshop/terraform/outputs.tf b/manifests/modules/aiml/chatbot/.workshop/terraform/outputs.tf new file mode 100644 index 000000000..7c1f1753f --- /dev/null +++ b/manifests/modules/aiml/chatbot/.workshop/terraform/outputs.tf @@ -0,0 +1,8 @@ +output "environment_variables" { + description = "Environment variables to be added to the IDE shell" + value = { + AIML_SUBNETS = "${data.aws_subnets.private.ids[0]},${data.aws_subnets.private.ids[1]},${data.aws_subnets.private.ids[2]}" + KARPENTER_NODE_ROLE = module.eks_blueprints_addons.karpenter.node_iam_role_name + KARPENTER_ARN = module.eks_blueprints_addons.karpenter.node_iam_role_arn + } +} diff --git a/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf b/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf new file mode 100644 index 000000000..e743600c5 --- /dev/null +++ b/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf @@ -0,0 +1,42 @@ +# tflint-ignore: terraform_unused_declarations +variable "eks_cluster_id" { + description = "EKS cluster name" + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "eks_cluster_version" { + description = "EKS cluster version" + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "cluster_security_group_id" { + description = "EKS cluster security group ID" + type = any +} + +# tflint-ignore: terraform_unused_declarations +variable "addon_context" { + description = "Addon context that can be passed directly to blueprints addon modules" + type = any +} + +# tflint-ignore: terraform_unused_declarations +variable "tags" { + description = "Tags to apply to AWS resources" + type = any +} + +# tflint-ignore: terraform_unused_declarations +variable "resources_precreated" { + description = "Have expensive resources been created already" + type = bool +} + +variable "karpenter_version" { + description = "The version of Karpenter chart to use" + type = string + # renovate: datasource=github-releases depName=aws/karpenter-provider-aws + default = "0.37.0" +} diff --git a/manifests/modules/aiml/chatbot/gradio/gradio-ui.yaml b/manifests/modules/aiml/chatbot/gradio/gradio-ui.yaml new file mode 100644 index 000000000..94efdf738 --- /dev/null +++ b/manifests/modules/aiml/chatbot/gradio/gradio-ui.yaml @@ -0,0 +1,130 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gradio-llama2-inf2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gradio-deployment + namespace: gradio-llama2-inf2 + labels: + app: gradio +spec: + replicas: 1 + selector: + matchLabels: + app: gradio + template: + metadata: + labels: + app: gradio + spec: + containers: + - name: gradio + image: public.ecr.aws/data-on-eks/gradio-web-app-base:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 7860 + resources: + requests: + cpu: "512m" + memory: "2048Mi" + limits: + cpu: "1" + memory: "4096Mi" + env: + - name: MODEL_ENDPOINT + value: "/infer" + - name: SERVICE_NAME + value: "http://llama2-serve-svc.llama2.svc.cluster.local:8000" + volumeMounts: + - name: gradio-app-script + mountPath: /app/gradio-app.py + subPath: gradio-app-llama2-inf2.py + volumes: + - name: gradio-app-script + configMap: + name: gradio-app-script +--- +apiVersion: v1 +kind: Service +metadata: + name: gradio-service + namespace: gradio-llama2-inf2 + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip +spec: + selector: + app: gradio + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 7860 + type: LoadBalancer +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gradio-app-script + namespace: gradio-llama2-inf2 +data: + gradio-app-llama2-inf2.py: | + import gradio as gr + import requests + import os + + # Constants for model endpoint and service name + model_endpoint = "/infer" + service_name = os.environ.get("SERVICE_NAME", "http://localhost:8000") + + # Function to generate text + def text_generation(message, history): + prompt = message + + # Create the URL for the inference + url = f"{service_name}{model_endpoint}" + + try: + # Send the request to the model service + response = requests.get(url, params={"sentence": prompt}, timeout=180) + response.raise_for_status() # Raise an exception for HTTP errors + + full_output = response.json()[0] + # Removing the original question from the output + answer_only = full_output.replace(prompt, "", 1).strip('["]?\n') + + # Safety filter to remove harmful or inappropriate content + answer_only = filter_harmful_content(answer_only) + return answer_only + except requests.exceptions.RequestException as e: + # Handle any request exceptions (e.g., connection errors) + return f"AI: Error: {str(e)}" + + # Define the safety filter function (you can implement this as needed) + def filter_harmful_content(text): + # TODO: Implement a safety filter to remove any harmful or inappropriate content from the text + + # For now, simply return the text as-is + return text + + # Define the Gradio ChatInterface + chat_interface = gr.ChatInterface( + text_generation, + chatbot=gr.Chatbot(line_breaks=True), + textbox=gr.Textbox(placeholder="Ask me a question", container=False, scale=7), + title="Llama2/3 AI Chat", + description="Ask me any question", + theme="soft", + examples=["How many languages are in India", "What is Generative AI?"], + cache_examples=False, + retry_btn=None, + undo_btn="Delete Previous", + clear_btn="Clear", + ) + + # Launch the ChatInterface + chat_interface.launch(server_name="0.0.0.0") diff --git a/manifests/modules/aiml/chatbot/gradio/kustomization.yaml b/manifests/modules/aiml/chatbot/gradio/kustomization.yaml new file mode 100644 index 000000000..1cca24122 --- /dev/null +++ b/manifests/modules/aiml/chatbot/gradio/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gradio-ui.yaml diff --git a/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml b/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml new file mode 100644 index 000000000..7bc6879ad --- /dev/null +++ b/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml @@ -0,0 +1,59 @@ +# rbac.yaml +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: neuron-device-plugin +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - update + - patch + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - update +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: neuron-device-plugin + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: neuron-device-plugin + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: neuron-device-plugin +subjects: + - kind: ServiceAccount + name: neuron-device-plugin + namespace: kube-system diff --git a/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml b/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml new file mode 100644 index 000000000..3a895a6eb --- /dev/null +++ b/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml @@ -0,0 +1,95 @@ +# https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: neuron-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: neuron-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + # Uncomment the annotation below if k8s version is 1.13 or lower + # annotations: + # scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + name: neuron-device-plugin-ds + spec: + serviceAccount: neuron-device-plugin + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: aws.amazon.com/neuron + operator: Exists + effect: NoSchedule + # Mark this pod as a critical add-on; when enabled, the critical add-on + # scheduler reserves resources for critical add-on pods so that they can + # be rescheduled after a failure. + # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + priorityClassName: "system-node-critical" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + # Uncomment following matchExpressions if using k8s 1.16 or lower + #- matchExpressions: + # - key: "beta.kubernetes.io/instance-type" + # operator: In + # values: + # - inf1.xlarge + # - inf1.2xlarge + # - inf1.6xlarge + # - inf1.24xlarge + # - inf2.xlarge + # - inf2.8xlarge + # - inf2.24xlarge + # - inf2.48xlarge + # - trn1.2xlarge + # - trn1.32xlarge + # - trn1n.32xlarge + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: In + values: + - inf1.xlarge + - inf1.2xlarge + - inf1.6xlarge + - inf1.24xlarge + - inf2.xlarge + - inf2.8xlarge + - inf2.24xlarge + - inf2.48xlarge + - trn1.2xlarge + - trn1.32xlarge + - trn1n.32xlarge + containers: + # Find all neuron-device-plugin images at https://gallery.ecr.aws/neuron/neuron-device-plugin + - image: public.ecr.aws/neuron/neuron-device-plugin:2.19.16.0 + imagePullPolicy: Always + name: neuron-device-plugin + env: + - name: KUBECONFIG + value: /etc/kubernetes/kubelet.conf + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: infa-map + mountPath: /run + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins + - name: infa-map + hostPath: + path: /run diff --git a/manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml b/manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml new file mode 100644 index 000000000..dd1b8b045 --- /dev/null +++ b/manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - k8s-neuron-device-plugin-rbac.yaml + - k8s-neuron-device-plugin.yaml diff --git a/manifests/modules/aiml/chatbot/nodepool/kustomization.yaml b/manifests/modules/aiml/chatbot/nodepool/kustomization.yaml new file mode 100644 index 000000000..b0f432bde --- /dev/null +++ b/manifests/modules/aiml/chatbot/nodepool/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - nodepool-inf2.yaml + - nodepool-x86.yaml diff --git a/manifests/modules/aiml/chatbot/nodepool/nodepool-inf2.yaml b/manifests/modules/aiml/chatbot/nodepool/nodepool-inf2.yaml new file mode 100644 index 000000000..5446c3fcb --- /dev/null +++ b/manifests/modules/aiml/chatbot/nodepool/nodepool-inf2.yaml @@ -0,0 +1,57 @@ +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: inferentia-inf2 +spec: + template: + metadata: + labels: + instanceType: inferentia-inf2 + provisionerType: Karpenter + spec: + requirements: + - key: "karpenter.k8s.aws/instance-family" + operator: In + values: ["inf2"] + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand", "spot"] + nodeClassRef: + name: inferentia-inf2 + taints: + - key: aws.amazon.com/neuron + value: "true" + effect: "NoSchedule" + limits: + cpu: "512" + disruption: + consolidateAfter: 300s + consolidationPolicy: WhenEmpty + expireAfter: 720h + +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: inferentia-inf2 +spec: + amiFamily: AL2 + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + deleteOnTermination: true + encrypted: true + volumeSize: 500Gi + volumeType: gp3 + role: ${KARPENTER_NODE_ROLE} + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ${EKS_CLUSTER_NAME} + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ${EKS_CLUSTER_NAME} + tags: + app.kubernetes.io/created-by: eks-workshop diff --git a/manifests/modules/aiml/chatbot/nodepool/nodepool-x86.yaml b/manifests/modules/aiml/chatbot/nodepool/nodepool-x86.yaml new file mode 100644 index 000000000..5ad10d70f --- /dev/null +++ b/manifests/modules/aiml/chatbot/nodepool/nodepool-x86.yaml @@ -0,0 +1,56 @@ +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: x86-cpu-karpenter +spec: + template: + metadata: + labels: + type: karpenter + instanceType: mixed-x86 + provisionerType: Karpenter + workload: rayhead + spec: + requirements: + - key: "karpenter.k8s.aws/instance-family" + operator: In + values: ["c5", "m5", "r5"] + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand", "spot"] + nodeClassRef: + name: x86-cpu-karpenter + limits: + cpu: "256" + disruption: + consolidateAfter: 300s + consolidationPolicy: WhenEmpty + expireAfter: 720h + +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: x86-cpu-karpenter +spec: + amiFamily: AL2 + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + deleteOnTermination: true + encrypted: true + volumeSize: 200Gi + volumeType: gp3 + detailedMonitoring: true + role: ${KARPENTER_NODE_ROLE} + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ${EKS_CLUSTER_NAME} + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ${EKS_CLUSTER_NAME} + tags: + app.kubernetes.io/created-by: eks-workshop diff --git a/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/Dockerfile b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/Dockerfile new file mode 100644 index 000000000..8208d88bf --- /dev/null +++ b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/Dockerfile @@ -0,0 +1,37 @@ +# docker buildx build --platform=linux/amd64 -t ray-serve-llama2:latest . +# https://hub.docker.com/layers/rayproject/ray-ml/2.7.1-py310-gpu/images/sha256-f84ecfc82d255ff9e23b8e40343a95655ec8e23a009633a183769edac6277186?context=explore +FROM rayproject/ray:2.22.0-py310 + +# Maintainer label +LABEL maintainer="DoEKS" + +# Set environment variables to non-interactive (this prevents some prompts) +ENV DEBIAN_FRONTEND=non-interactive + +# Switch to root to add Neuron repo and install necessary packages +USER root + +# Set up the Neuron repository and install Neuron packages +RUN . /etc/os-release && \ + sudo echo "deb https://apt.repos.neuron.amazonaws.com ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/neuron.list && \ + sudo wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | apt-key add - && \ + sudo apt-get update -y && \ + sudo apt-get install aws-neuronx-dkms aws-neuronx-collectives=2.* aws-neuronx-runtime-lib=2.* aws-neuronx-tools=2.* -y && \ + sudo apt-get clean + +# Switch back to a non-root user for the subsequent commands +USER $USER + +# Set pip repository pointing to the Neuron repository and install required Python packages +RUN pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com && \ + pip install wget awscli regex neuronx-cc==2.* torch-neuronx torchvision transformers-neuronx sentencepiece transformers + +# Add Neuron path to PATH +ENV PATH /opt/aws/neuron/bin:$PATH + +# Set LD_LIBRARY_PATH to include the directory with libpython3.10.so.1.0 +ENV LD_LIBRARY_PATH /home/ray/anaconda3/lib:$LD_LIBRARY_PATH + +WORKDIR /serve_app + +COPY ray_serve_llama2.py /serve_app/ray_serve_llama2.py \ No newline at end of file diff --git a/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/kustomization.yaml b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/kustomization.yaml new file mode 100644 index 000000000..623056e56 --- /dev/null +++ b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ray-service-llama2.yaml diff --git a/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray-service-llama2.yaml b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray-service-llama2.yaml new file mode 100644 index 000000000..7c3088722 --- /dev/null +++ b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray-service-llama2.yaml @@ -0,0 +1,130 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: llama2 + +--- +# target_num_ongoing_requests_per_replica will be deprecated soon +# and will be updated when Data on EKS updates +apiVersion: ray.io/v1 +kind: RayService +metadata: + name: llama2 + namespace: llama2 +spec: + serviceUnhealthySecondThreshold: 900 + deploymentUnhealthySecondThreshold: 300 + serveConfigV2: | + applications: + - name: llama2 + import_path: "ray_serve_llama2:entrypoint" + runtime_env: + env_vars: + MODEL_ID: "NousResearch/Llama-2-13b-chat-hf" + NEURON_CC_FLAGS: "-O1" + LD_LIBRARY_PATH: "/home/ray/anaconda3/lib:$LD_LIBRARY_PATH" + NEURON_CORES: "24" + deployments: + - name: Llama-2-13b-chat-hf + autoscaling_config: + metrics_interval_s: 0.2 + min_replicas: 1 + max_replicas: 1 + look_back_period_s: 2 + downscale_delay_s: 30 + upscale_delay_s: 2 + target_num_ongoing_requests_per_replica: 1 + graceful_shutdown_timeout_s: 5 + ray_actor_options: + num_cpus: 180 + resources: {"neuron_cores": 24} + runtime_env: + env_vars: + LD_LIBRARY_PATH: "/home/ray/anaconda3/lib:$LD_LIBRARY_PATH" + rayClusterConfig: + rayVersion: 2.22.0 + headGroupSpec: + headService: + metadata: + name: llama2 + namespace: llama2 + rayStartParams: + dashboard-host: "0.0.0.0" + template: + spec: + containers: + - name: head + image: public.ecr.aws/data-on-eks/ray2.22.0-py310-llama2-13b-neuron:latest # Image created using the Dockerfile attached in the folder + imagePullPolicy: Always # Ensure the image is always pulled when updated + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "ray stop"] + ports: + - containerPort: 6379 + name: gcs + - containerPort: 8265 + name: dashboard + - containerPort: 10001 + name: client + - containerPort: 8000 + name: serve + volumeMounts: + - mountPath: /tmp/ray + name: ray-logs + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 1 + memory: 2Gi + env: + - name: LD_LIBRARY_PATH + value: "/home/ray/anaconda3/lib:$LD_LIBRARY_PATH" + nodeSelector: + instanceType: mixed-x86 + provisionerType: Karpenter + workload: rayhead + volumes: + - name: ray-logs + emptyDir: {} + workerGroupSpecs: + - groupName: inf2 + replicas: 1 + minReplicas: 1 + maxReplicas: 1 + rayStartParams: {} + template: + spec: + containers: + - name: worker + image: public.ecr.aws/data-on-eks/ray2.22.0-py310-llama2-13b-neuron:latest + imagePullPolicy: Always # Ensure the image is always pulled when updated + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "ray stop"] + resources: + limits: + cpu: "180" + memory: "700G" + aws.amazon.com/neuron: "12" + requests: + cpu: "180" + memory: "700G" + aws.amazon.com/neuron: "12" + env: + - name: LD_LIBRARY_PATH + value: "/home/ray/anaconda3/lib:$LD_LIBRARY_PATH" + nodeSelector: + instanceType: inferentia-inf2 + provisionerType: Karpenter + tolerations: + - key: "aws.amazon.com/neuron" + operator: "Exists" + effect: "NoSchedule" + - key: "hub.jupyter.org/dedicated" + operator: "Equal" + value: "user" + effect: "NoSchedule" diff --git a/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray_serve_llama2.py b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray_serve_llama2.py new file mode 100644 index 000000000..673b69a3c --- /dev/null +++ b/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray_serve_llama2.py @@ -0,0 +1,103 @@ +import os +import logging +from fastapi import FastAPI +from ray import serve +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig +from transformers_neuronx.llama.model import LlamaForSampling +from transformers_neuronx.module import save_pretrained_split + +app = FastAPI() + +llm_model_split = "llama-2-13b-chat-hf-split" +neuron_cores = int(os.getenv('NEURON_CORES', 24)) # Read from environment variable, default to 24 + +# --- Logging Setup --- +logger = logging.getLogger("ray.serve") +logger.setLevel(logging.INFO) +logging.basicConfig(level=logging.INFO) + + +# Define the APIIngress class responsible for handling inference requests +@serve.deployment(num_replicas=1) +@serve.ingress(app) +class APIIngress: + def __init__(self, llama_model_handle): + self.handle = llama_model_handle + + @app.get("/infer") + async def infer(self, sentence: str): + # Asynchronously perform inference using the provided sentence + result = await self.handle.infer.remote(sentence) + return result + + +# Define the LlamaModel class responsible for managing the Llama language model +@serve.deployment( + name="Llama-2-13b-chat-hf", + autoscaling_config={"min_replicas": 1, "max_replicas": 2}, + ray_actor_options={ + "resources": {"neuron_cores": neuron_cores}, + "runtime_env": {"env_vars": {"NEURON_CC_FLAGS": "-O1"}}, + }, +) +class LlamaModel: + def __init__(self): + from transformers_neuronx.llama.model import LlamaForSampling + from transformers_neuronx.module import save_pretrained_split + + llm_model = os.getenv('MODEL_ID', 'NousResearch/Llama-2-13b-chat-hf') + logger.info(f"Using model ID: {llm_model}") + + # Check if the model split exists locally, and if not, download it + if not os.path.exists(llm_model_split): + logger.info(f"Saving model split for {llm_model} to local path {llm_model_split}") + try: + self.model = AutoModelForCausalLM.from_pretrained(llm_model) + # Set and validate generation config + generation_config = GenerationConfig( + do_sample=True, + temperature=0.9, + top_p=0.6, + top_k=50, + ) + generation_config.validate() + self.model.generation_config = generation_config + save_pretrained_split(self.model, llm_model_split) + except Exception as e: + logger.error(f"Error during model download or split saving: {e}") + raise e + else: + logger.info(f"Using existing model split {llm_model_split}") + + logger.info(f"Loading and compiling model {llm_model_split} for Neuron") + try: + self.neuron_model = LlamaForSampling.from_pretrained( + llm_model_split, batch_size=1, tp_degree=neuron_cores, amp='f16' + ) + self.neuron_model.to_neuron() + logger.info("Model loaded and compiled successfully") + except Exception as e: + logger.error(f"Error during model loading or compilation: {e}") + raise e + + self.tokenizer = AutoTokenizer.from_pretrained(llm_model) + + def infer(self, sentence: str): + input_ids = self.tokenizer.encode(sentence, return_tensors="pt") + with torch.inference_mode(): + try: + logger.info(f"Performing inference on input: {sentence}") + generated_sequences = self.neuron_model.sample( + input_ids, sequence_length=2048, top_k=50 + ) + decoded_sequences = [self.tokenizer.decode(seq, skip_special_tokens=True) for seq in generated_sequences] + logger.info(f"Inference result: {decoded_sequences}") + return decoded_sequences + except Exception as e: + logger.error(f"Error during inference: {e}") + return {"error": "Inference failed"} + + +# Create an entry point for the FastAPI application +entrypoint = APIIngress.bind(LlamaModel.bind()) \ No newline at end of file diff --git a/website/docs/aiml/chatbot/_category_.json b/website/docs/aiml/chatbot/_category_.json new file mode 100644 index 000000000..9e3a55f45 --- /dev/null +++ b/website/docs/aiml/chatbot/_category_.json @@ -0,0 +1,3 @@ +{ + "collapsed": true +} diff --git a/website/docs/aiml/chatbot/add-llama2.md b/website/docs/aiml/chatbot/add-llama2.md new file mode 100644 index 000000000..cd8efd82f --- /dev/null +++ b/website/docs/aiml/chatbot/add-llama2.md @@ -0,0 +1,86 @@ +--- +title: "Deploying the Llama-2-Chat Model on Ray Serve" +sidebar_position: 30 +--- + +With both node pools provisioned, we can now proceed to deploy the Llama2 chatbot infrastructure. + +Let's begin by deploying the `ray-service-llama2.yaml` file: + +```bash wait=5 +$ kubectl apply -k ~/environment/eks-workshop/modules/aiml/chatbot/ray-service-llama2-chatbot +namespace/llama2 created +rayservice.ray.io/llama2 created +``` + +### Creating the Ray Service Pods for Inference + +The `ray-service-llama2.yaml` file defines the Kubernetes configuration for deploying the Ray Serve service for the Llama2 chatbot: + +```file +manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray-service-llama2.yaml +``` + +This configuration accomplishes the following: + +1. Creates a Kubernetes namespace named `llama2` for resource isolation +2. Deploys a RayService named `llama-2-service` that utilizes a Python script to create the Ray Serve component +3. Provisions a Head Pod and Worker Pods to pull Docker images from Amazon Elastic Container Registry (ECR) + +After applying the configurations, we'll monitor the progress of the head and worker pods: + +```bash wait=5 +$ kubectl get pod -n llama2 +NAME READY STATUS RESTARTS AGE +pod/llama2-raycluster-fcmtr-head-bf58d 1/1 Running 0 67m +pod/llama2-raycluster-fcmtr-worker-inf2-lgnb2 1/1 Running 0 5m30s +``` + +:::caution +It may take up to 10 minutes for both pods to be ready. +::: + +We can wait for the pods to be ready using the following command: + +```bash timeout=600 +$ kubectl wait pod \ +--all \ +--for=condition=Ready \ +--namespace=llama2 \ +--timeout=15m +pod/llama2-raycluster-fcmtr-head-bf58d met +pod/llama2-raycluster-fcmtr-worker-inf2-lgnb2 met +``` + +Once the pods are fully deployed, we'll verify that everything is in place: + +```bash +$ kubectl get all -n llama2 +NAME READY STATUS RESTARTS AGE +pod/llama2-raycluster-fcmtr-head-bf58d 1/1 Running 0 67m +pod/llama2-raycluster-fcmtr-worker-inf2-lgnb2 1/1 Running 0 5m30s + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/llama2 ClusterIP 172.20.118.243 10001/TCP,8000/TCP,8080/TCP,6379/TCP,8265/TCP 67m +service/llama2-head-svc ClusterIP 172.20.168.94 8080/TCP,6379/TCP,8265/TCP,10001/TCP,8000/TCP 57m +service/llama2-serve-svc ClusterIP 172.20.61.167 8000/TCP 57m + +NAME DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +raycluster.ray.io/llama2-raycluster-fcmtr 1 1 184 704565270Ki 0 ready 67m + +NAME SERVICE STATUS NUM SERVE ENDPOINTS +rayservice.ray.io/llama2 Running 2 +``` + +:::caution +Configuring RayService may take up to 10 minutes. +::: + +We can wait for the RayService to be running with this command: + +```bash wait=5 timeout=600 +$ kubectl wait --for=jsonpath='{.status.serviceStatus}'=Running rayservice/llama2 -n llama2 --timeout=10m +rayservice.ray.io/llama2 condition met +``` + +With everything properly deployed, we can now proceed to create the web interface for the chatbot. diff --git a/website/docs/aiml/chatbot/expose.md b/website/docs/aiml/chatbot/expose.md new file mode 100644 index 000000000..473635fbb --- /dev/null +++ b/website/docs/aiml/chatbot/expose.md @@ -0,0 +1,57 @@ +--- +title: "Install KubeRay and Neuron Devices" +sidebar_position: 10 +--- + +Before deploying the node pools and Ray Serve Cluster on EKS, it's important to have the necessary tools in place for the workloads to be properly configured. + +### Applying KubeRay Operator for Ray Service Cluster + +Deploying Ray Cluster on Amazon EKS is supported via the [KubeRay Operator](https://ray-project.github.io/kuberay/), a Kubernetes-native method for managing Ray Clusters. As a module, KubeRay simplifies the deployment of Ray applications by providing three unique custom resource definitions: `RayService`, `RayCluster`, and `RayJob`. + +To install the KubeRay Operator, apply the following commands: + +```bash +$ helm repo add kuberay https://ray-project.github.io/kuberay-helm/ +"kuberay" has been added to your repositories +``` + +```bash wait=10 +$ helm install kuberay-operator kuberay/kuberay-operator --version 1.1.0 +NAME: kuberay-operator +LAST DEPLOYED: Wed Jul 24 14:46:13 2024 +NAMESPACE: default +STATUS: deployed +REVISION: 1 +TEST-SUITE: None +``` + +Once KubeRay has been properly installed, we can check that it exists under the default namespace: + +```bash +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +kuberay-operator-6fcbb94f64-mbfnr 1/1 Running 0 17s +``` + +### Install Neuron Devices + +The [Neuron device plugin Kubernetes manifest files](https://github.com/aws-neuron/aws-neuron-sdk/tree/master/src/k8) need to be installed into the EKS Cluster to allow Karpenter to properly provision Inferentia-2 instances. + +This allows pods to have the resource requirement of `aws.amazon.com/neuron`, enabling the Kubernetes Scheduler to provision a node demanding accelerated machine learning workloads. + +:::tip +You can learn more about Neuron Device Plugins in the [AIML Inference module](../../aiml/inferentia/index.md) provided in this workshop. +::: + +We can deploy the role using the following command: + +```bash +$ kubectl apply -k ~/environment/eks-workshop/modules/aiml/chatbot/neuron-device-plugin +serviceaccount/neuron-device-plugin created +clusterrole.rbac.authorization.k8s.io/neuron-device-plugin created +clusterrolebinding.rbac.authorization.k8s.io/neuron-device-plugin created +daemonset.apps/neuron-device-plugin-daemonset created +``` + +This properly exposes the Neuron cores and grants Karpenter the appropriate permissions to provision pods demanding accelerated workloads. diff --git a/website/docs/aiml/chatbot/gradio.md b/website/docs/aiml/chatbot/gradio.md new file mode 100644 index 000000000..3a02074c6 --- /dev/null +++ b/website/docs/aiml/chatbot/gradio.md @@ -0,0 +1,68 @@ +--- +title: "Configuring the Gradio Web User Interface for Access" +sidebar_position: 40 +--- + +After all the resources have been configured within the Ray Serve Cluster, it's now time to directly access the Llama2 chatbot. The web interface is powered by the Gradio UI. + +:::tip +You can learn more about Load Balancers in the [Load Balancer module](../../fundamentals/exposing/loadbalancer/index.md) provided in this workshop. +::: + +### Deploying Gradio Web User Interface + +Once the AWS Load Balancer Controller has been installed, we can deploy the Gradio UI components. + +```file +manifests/modules/aiml/chatbot/gradio/gradio-ui.yaml +``` + +The components consist of a `Deployment`, `Service`, and `ConfigMap` to launch the application. In particular, the `Service` component is named gradio-service and is deployed as a `LoadBalancer`. + +```bash +$ kubectl apply -k ~/environment/eks-workshop/modules/aiml/chatbot/gradio +namespace/gradio-llama2-inf2 created +configmap/gradio-app-script created +service/gradio-service created +deployment.apps/gradio-deployment created +``` + +To check the status of each component, run the following commands: + +```bash +$ kubectl get deployments -n gradio-llama2-inf2 +NAME READY UP-TO-DATE AVAILABLE AGE +gradio-deployment 1/1 1 1 95s +``` + +```bash +$ kubectl get configmaps -n gradio-llama2-inf2 +NAME DATA AGE +gradio-app-script 1 110s +kube-root-ca.crt 1 111s +``` + +### Accessing the Chatbot Website + +Once the load balancer has finished deploying, use the external IP address to directly access the website: + +```bash +$ kubectl get services -n gradio-llama2-inf2 +NAME TYPE ClUSTER-IP EXTERNAL-IP PORT(S) AGE +gradio-service LoadBalancer 172.20.84.26 k8s-gradioll-gradiose-a6d0b586ce-06885d584b38b400.elb.us-west-2.amazonaws.com 80:30802/TCP 8m42s +``` + +To wait until the Network Load Balancer has finished provisioning, run the following command: + +```bash wait=240 timeout=600 +$ curl --head -X GET --retry 30 --retry-all-errors --retry-delay 15 --connect-timeout 5 --max-time 10 \ +-k $(kubectl get service -n gradio-llama2-inf2 gradio-service -o jsonpath="{.status.loadBalancer.ingress[*].hostname}{'\n'}") +``` + +Now that our application is exposed to the outside world, let's access it by pasting the URL in your web browser. You will see the Llama2 chatbot and will be able to interact with it by asking questions. + + + + + +This concludes the current lab on deploying the Meta Llama-2-13b Chatbot Model within an EKS Cluster via Karpenter. diff --git a/website/docs/aiml/chatbot/index.md b/website/docs/aiml/chatbot/index.md new file mode 100644 index 000000000..adfd14ed9 --- /dev/null +++ b/website/docs/aiml/chatbot/index.md @@ -0,0 +1,35 @@ +--- +title: "Large Language Models with Ray Serve" +sidebar_position: 30 +chapter: true +sidebar_custom_props: { "beta": true } +description: "Use Inferentia to accelerate deep learning inference workloads on Amazon Elastic Kubernetes Service." +--- + +:::danger +This module is not supported at AWS events or in AWS-vended accounts through Workshop Studio. This module is only supported for clusters created through the "[In your AWS account](/docs/introduction/setup/your-account)" steps. +::: + +:::tip Before you start +Prepare your environment for this section: + +```bash timeout=300 wait=30 +$ prepare-environment aiml/chatbot +``` + +This will make the following changes to your lab environment: + +- Installs Karpenter in the Amazon EKS cluster +- Creates an IAM Role for the Pods to use + +You can view the Terraform that applies these changes [here](https://github.com/VAR::MANIFESTS_OWNER/VAR::MANIFESTS_REPOSITORY/tree/VAR::MANIFESTS_REF/manifests/modules/aiml/chatbot/.workshop/terraform). + +::: + +With pre-training on 2 trillion tokens of text and code, the [Meta Llama-2-13b](https://llama.meta.com/#inside-the-model) chat model is one of the largest and most powerful large language models (LLMs) available today. + +From its natural language processing and text generation capabilities to handling inference and training workloads, the creation of Llama2 represents some of the newest advances in GenAI Technology. + +This section will focus not only on harnessing the power of Llama-2 but also on gaining insights into the intricacies of deploying LLMs efficiently on EKS. + +For deploying and scaling LLMs, this lab will utilize AWS Inferentia instances within the [Inf2](https://aws.amazon.com/machine-learning/inferentia/) family, such as `Inf2.24xlarge` and `Inf2.48xlarge`. Additionally, the chatbot inference workloads will utilize the [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) module for building online inference APIs and streamlining the deployment of machine learning models, as well as the [Gradio UI](https://www.gradio.app/) for accessing the Llama2 chatbot. diff --git a/website/docs/aiml/chatbot/intro.md b/website/docs/aiml/chatbot/intro.md new file mode 100644 index 000000000..9794d4b52 --- /dev/null +++ b/website/docs/aiml/chatbot/intro.md @@ -0,0 +1,25 @@ +--- +title: "Understanding the Llama2 Chatbot Model" +sidebar_position: 20 +--- + +Llama2 is a training model that uses FastAPI, Ray Serve, and PyTorch-based Hugging Face Transformers to create a seamless API for text generation. + +For this lab, we'll be using Llama-2-13b, a medium-sized model with 13 billion parameters. It offers a good balance between performance and efficiency and can be used for a variety of tasks. Using `Inf2.24xlarge` or `Inf2.48xlarge` instances makes it easier to handle high-performance deep learning (DL) training and inference of generative AI models, including LLMs. + +Here's the code for compiling the model that we'll use: + +```file +manifests/modules/aiml/chatbot/ray-service-llama2-chatbot/ray_serve_llama2.py +``` + +This Python code performs the following tasks: + +1. Configures an APIIngress class responsible for handling inference requests +2. Defines a LlamaModel class responsible for managing the Llama language model +3. Loads and compiles the model based on existing parameters +4. Creates an entry point for the FastAPI application + +Through these steps, the Llama-2-13b chat model allows the endpoint to accept input sentences and generate text outputs. The high performance efficiency in processing tasks enables the model to handle a wide variety of natural language processing applications, such as chat bots and text generation tasks. + +In this lab, we'll see how the Llama2 Model is configured with Ray Service as a Kubernetes configuration, allowing users to understand how to incorporate fine-tuning and deploy their own natural language processing applications. diff --git a/website/docs/aiml/chatbot/nodepool.md b/website/docs/aiml/chatbot/nodepool.md new file mode 100644 index 000000000..3d3148b89 --- /dev/null +++ b/website/docs/aiml/chatbot/nodepool.md @@ -0,0 +1,70 @@ +--- +title: "Provisioning Node Pools for LLM Workloads" +sidebar_position: 20 +--- + +In this lab, we'll use Karpenter to provision the Inferentia-2 nodes necessary for handling the Llama2 chatbot workload. As an autoscaler, Karpenter creates the resources required to run machine learning workloads and distribute traffic efficiently. + +:::tip +To learn more about Karpenter, check out the [Karpenter module](../../autoscaling/compute/karpenter/index.md) in this workshop. +::: + +Karpenter has already been installed in our EKS Cluster and runs as a deployment: + +```bash +$ kubectl get deployment -n karpenter +NAME READY UP-TO-DATE AVAILABLE AGE +karpenter 2/2 2 2 11m +``` + +As we did in a previous lab, we need to update our EKS IAM mappings to allow Karpenter nodes to join the cluster: + +```bash +$ eksctl create iamidentitymapping --cluster $EKS_CLUSTER_NAME \ + --region $AWS_REGION --arn $KARPENTER_ARN \ + --group system:bootstrappers --group system:nodes \ + --username system:node:{{EC2PrivateDNSName}} +``` + +Since the Ray Cluster creates head and worker pods with different specifications for handling various EC2 families, we'll create two separate node pools to handle the workload demands. + +Here's the first Karpenter `NodePool` that will provision one `Head Pod` on `x86 CPU` instances: + +::yaml{file="manifests/modules/aiml/chatbot/nodepool/nodepool-x86.yaml" paths="spec.template.metadata.labels,spec.template.spec.requirements,spec.limits"} + +1. We're asking the `NodePool` to start all new nodes with a Kubernetes label `type: karpenter`, which will allow us to specifically target Karpenter nodes with pods for demonstration purposes. Since there are multiple nodes being autoscaled by Karpenter, there are additional labels added such as `instanceType: mixed-x86` to indicate that this Karpenter node should be assigned to `x86-cpu-karpenter` pool. +2. The [NodePool CRD](https://karpenter.sh/docs/concepts/nodepools/) supports defining node properties like instance type and zone. In this example, we're setting the `karpenter.sh/capacity-type` to initially limit Karpenter to provisioning On-Demand and Spot instances, as well as `karpenter.k8s.aws/instance-family` to limit to a subset of specific instance types. You can learn which other properties are [available here](https://karpenter.sh/docs/concepts/scheduling/#selecting-nodes). Compared to the previous lab, there are more specifications defining the unique constraints of the `Head Pod`, such as defining an instance family of `r5`, `m5`, and `c5` nodes. +3. A `NodePool` can define a limit on the amount of CPU and memory managed by it. Once this limit is reached Karpenter will not provision additional capacity associated with that particular `NodePool`, providing a cap on the total compute. + +This secondary `NodePool` will provision `Ray Workers` on `Inf2.48xlarge` instances: + +::yaml{file="manifests/modules/aiml/chatbot/nodepool/nodepool-inf2.yaml" paths="spec.template.metadata.labels,spec.template.spec.requirements,spec.template.spec.taints,spec.limits"} + +1. We're asking the `NodePool` to start all new nodes with a Kubernetes label `provisionerType: Karpenter`, which will allow us to specifically target Karpenter nodes with pods for demonstration purposes. Since there are multiple nodes being autoscaled by Karpenter, there are additional labels added such as `instanceType: inferentia-inf2` to indicate that this Karpenter node should be assigned to `inferentia-inf2` pool. +2. The [NodePool CRD](https://karpenter.sh/docs/concepts/nodepools/) supports defining node properties like instance type and zone. In this example, we're setting the `karpenter.sh/capacity-type` to initially limit Karpenter to provisioning On-Demand and Spot instances, as well as `karpenter.k8s.aws/instance-family` to limit to a subset of specific instance types. You can learn which other properties are [available here](https://karpenter.sh/docs/concepts/scheduling/#selecting-nodes). In this case, there are specifications matching the requirements of the `Ray Workers` that will run on instances from the `Inf2` family. +3. A `Taint` defines a specific set of properties that allow a node to repel a set of pods. This property works with its matching label, a `Toleration`. Both tolerations and taints work together to ensure that pods are properly scheduled onto the appropriate pods. You can learn more about the other properties in [this resource](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). +4. A `NodePool` can define a limit on the amount of CPU and memory managed by it. Once this limit is reached Karpenter will not provision additional capacity associated with that particular `NodePool`, providing a cap on the total compute. + +Both of these defined node pools will allow Karpenter to properly schedule nodes and handle the workload demands of the Ray Cluster. + +Apply the `NodePool` and `EC2NodeClass` manifests for both pools: + +```bash +$ kubectl kustomize ~/environment/eks-workshop/modules/aiml/chatbot/nodepool \ + | envsubst | kubectl apply -f- +ec2nodeclass.karpenter.k8s.aws/inferentia-inf2 created +ec2nodeclass.karpenter.k8s.aws/x86-cpu-karpenter created +nodepool.karpenter.sh/inferentia-inf2 created +nodepool.karpenter.sh/x86-cpu-karpenter created +``` + +Once properly deployed, check for the node pools: + +```bash +$ kubectl get nodepool +NAME NODECLASS +inferentia-inf2 inferentia-inf2 +x86-cpu-karpenter x86-cpu-karpenter +``` + +As seen from the above command, both node pools have been properly provisioned, allowing Karpenter to allocate new nodes into the newly created pools as needed. diff --git a/website/docs/aiml/chatbot/tests/hook-suite.sh b/website/docs/aiml/chatbot/tests/hook-suite.sh new file mode 100644 index 000000000..8b5a4baea --- /dev/null +++ b/website/docs/aiml/chatbot/tests/hook-suite.sh @@ -0,0 +1,11 @@ +set -e + +before() { + echo "noop" +} + +after() { + prepare-environment +} + +"$@" diff --git a/website/src/css/custom.scss b/website/src/css/custom.scss index a9b37df1c..61330a3b0 100644 --- a/website/src/css/custom.scss +++ b/website/src/css/custom.scss @@ -197,6 +197,10 @@ background-color: #6c757d; } +.beta { + background-color: #ff9900; +} + .category-wrapper { display: flex; width: 100%; diff --git a/website/src/theme/DocSidebarItem/Category/index.js b/website/src/theme/DocSidebarItem/Category/index.js index 92fe24e48..4e827c29f 100644 --- a/website/src/theme/DocSidebarItem/Category/index.js +++ b/website/src/theme/DocSidebarItem/Category/index.js @@ -175,6 +175,11 @@ export default function DocSidebarItemCategory({ ) : ( )} + {item.customProps?.beta ? ( + BETA + ) : ( + + )}
diff --git a/website/static/img/sample-app-screens/chatbot.webp b/website/static/img/sample-app-screens/chatbot.webp new file mode 100644 index 0000000000000000000000000000000000000000..63764b9510d6fae353ccaf17d66888465fc242de GIT binary patch literal 87314 zcmeFYWmH_t);5X+2^J(kaEBylkS2JLKnU*AxCUt&cPAuB2n2UXun=4uCs=R@PSZf+ z4vo9q&VEn!&iLMQ&K>vP{qgk}-D|O`YE{*oHLK=)o=Lc>vK&4RB@PM-3cma+8FdsC zOhyzGbY^T!(()6m%U@IGYOB9q>;R!lex*EM?nZVeX7jHf* z+;6^*rjGXN{%4FBqkC%iQ5wCvPLRq))E}RdpiB_-jN#U&s7lw+ znXxgb(BFFzMNk1Ynzx|sv>}AOpd#v2_mjPRAe<5~rbsur;>i2H!k4S^%frV#wAR?oKC;?ZeUGn!9MHHzD`RI#$;cdq5;x z_fy}`ClqF68f{tYVV3@REwrzkiIZ_|ru2MkrRX#``yy0=lhG<(ex`p$Y^D`PbL6tr zP6{xOsMP%=4KR~0=J&@9Grwq$L8%wlPG%tZs5PCmy82*{9G1>vBKLGZD7g4s)yuzJ zn@f&NA{}?KjhvU{o$L9yq6v+bUl5&>FpBrk(UQ||uZBqV1khNm>R`n%Y;(p( z^`NRS452U5lg}$yeCvGF@!MU zN^Id2d^rT=StyXE2s~0U2(MEeS{%hiZDAQ-P#(kj@tu15`$Zj!#{&MfBq27s8a*v@ zNI1_WN8M1D$s-C(^L=JLuoBl&T;G-DY?&h@!{`s4mR%blVk*weP-DpcIDKM+LPTL z(b!-)7M=-hd4a_f_{3Z|`N>+~GafRs&^!~0_gAZe8pNw%DRCEMd#8@ba%4Z#CZ2l?{TB0@qCPtPk#NFMU$|W zgn(q|d-J%GryMh}TiB$`gF5oKaA+q*XICeFXLDyu1+_mvtmL1Ccg zYkj#HC1J@1sV!DNRdsD%HD@&(?bm9=Pf6Le9`C9Pm%6^(Oc}MV6o$s`y!^;`)HBwr z&`Gmd4ll6KSW}x+-+FBQ`0Oz_J|ST+UM#^p9;44CVd^PDL%i%- z^&%%CKOzH1fko@(mr5DS_kHpSu&3a!n%_t#vQniuv$;M`s;X$vsY-ku)uhpaeiNM# z`j+tRSYs8VrrRr991}v>F4F69rJuMolFAxiL6pxT_I#R& zyl=gIn|Yg?y{AsZH^)vwj$MwEwgorfg5U9X@XYY?Ded{6oBtG)vU_`my%$zVge5!s zOa^}!&m7RjWo z+UVfgyejKLwO#wYq-(dR@qOcT^HB3tM?2GCPTw-CbR^NB?!a?|{65}(`m)#fQBG2$ zPS6x;S!)wblS`9_*V%^(+77xZ(11A|AKk}cvMes+25>6E3eF{=A@R|;2F?`V5|DZ4 zerK%+v;w#Y9$5xz1U_GI{O*%TZb!ClK~MIGSeI;?HkTrvuv&aWvRu#6$`Rxk5C-h| z5EJ+L?a%$6tDgm-vi4k!@8;;%hyd)x0tPIiT)uwiIoOTZPBKTjuJ+T@3Fe5G`m#7o zR9{{^6tT4&zj?F0@nl5KEbQ6HvxH|V<{9SJKQbe8x|AYdq-R{>^!_&2bNABj$xA7G z(KoX-Q(C&p{hmuIZ@4TQl_s#KUMX2gU;ivLJ~p{h%+kd7=lR5PqsE-Nf(BTzJd+31 z9vY5G;=)FomJ0~y+ccqM;CuqonITtxSdhvWGo)Ph8nIqkXIIDVdE2DbWYqE?0OL;Y zoyw1h51EQt0|_b3dj5K(dVXk$Cd~_4?r%=>cg}>=ghZS%UE-Y2)}E~9tqR;12o-9d z?EMi}ptSc88u|LAp+-}FapovD0XcYB?vQOoknBwx-31^#YK6SFLkiyB*Zx3*q<}U( z|C!SB5>Uh3@yoF+QVjAN0D4wi#t842>snLF1M6qjpp>Wq<*Yg0ZeQiz^u2lp7muNw zf&P>;p9+nGh$YBT+?rt0ql5uYuWN-K?wY)Bc}t^dxJ~5t`eED;IqZ%3y+;~e!uvce z$@??<*!x;J*0@^Ei;l|d0%#0kD{}n`uSTQ#!}{O$J8+x+;dRi(Jy%}HNiU$fRbt;lcl-~~`!Ytjs0!jsH_Zk;^9`7}rNU}|P)%(?T`XeZ^>vG9 z*HKSQU81?Am7(Z!(X;xBswc&TdCh|9abT8r^KTXV^`PDQ@rc@N8qxd~cat4lgsP1 zpZ>wXik?!m!tlua!u-ee@cRa z9)AoT-H{J?C`JgBz*IaG{LJCU%d6`WdXg*F+7xJa_xp##Dwhae&|C*ZZ-xX23ZhO! z@M&AKTlAlzjA)`PiJ=O6A3M?r7xfM$j7q&-4;)l89y=OsYkhLaxBkHCZWgWZ8~b_v z4f&^VUw_xyfL}32gr%;$m68(5Q{+81$~{y{6b$4YGRQy?N2U7vUKaH+3i@Bq(NIv{ z+o0V0R~=>K_17l``Tf=Af3E1U?@+Lie-R_Up4n)>YhyBIqyN4~XGWHxNNPyS%OkHE z=FXOu4lZxNt_kH4zeaN3@s*wn3JMv^uP>^+I`cmA{ZlrYx~{rPio)h#5SOV1*vyj4 z6Xf{oJt(4{!pK{YrK>5uC&=ExMc7k}@vj=f$opT9xf$vIs^V%V#;B{LN-qs|wxkDe z@o@1lisR7J(~CM=SP84k$o{K2@-H#QH?FRZ!ra^*9v)mCd|Y5>Ywl-4LPFd;yxhFJ zoX8rSE?y3RPlHc!$jC&>M)g!>s65BJ~iMm81w^;B5Z#?#VXPsRp>lo_%Q@#j4JqJP!@UnT$a z_(x0Ke_B2h5aju@=^sV^XHzW~OJ`{?2-&Br_&@skSK~hm|J6{G``6U}5XJw<`LCx) zL5t&va{sL~ahymxJtCwYscmFbG?7>2D*N?8jYa-E{-5iwdvE>ZZAEhwlou%SGLo8} zs5^5WCZt-Bvx5X$Vs8ZfE24WcT_3+beoVG=qgEeb7{mCAJmJeUl^J?hL^C@!V=(@w z`Lxmad1Cz;L%Zcp!skz<+k1t4*IQyOhT_Gt#D=fBZ!?;8IvY;5dEerr-Xna0@*jR4 z-wO)kgw+zsJVfJ2lKD@ykj+1Z&H1QfGkTKz=XbrxR=tPKh!dI+$ry-&hE4x(e**6l z%Iwqr`?9|pz4(l*@sN^}T=u_=CK!AEKTPPanLg%64GxpaS#AFhgZS0a3&J*$e>=i| z4Ly|g1)PAoe*Th2R3a|Zf9Xi_9^yX=3t8|m6-83!^N6$Qe^DZR z*aFRe)bI3bYpDQ{j zhKN24JEio?=RfB8u=qv8y9ew#C7-%8y(L4!@WR8x6S*=R;wUGIY*>lZ$_1Bg|ENyy zIE_OnLtSe)9P*D&srS(2dwY4;LR3~q*fKrZq`PodS98u}UqdS`-#%J$3=r}3|3iB? z%1ALRaMTn3Eru7U*txm6b>L0#TtxC6T_uii*B*YQ&b4<4KuCA{BTpbmE zjQ(u`axLk)u{{X+y%6pho2(BRR$BIJL+kC5>=b={n>0%`C9xn!+8!ojYn;5t71kEF z!CiaTaVnxl4hYYjcuo!B^9STBWYg-eMcjAd@u>tOFa)eXS4+#wxthJP>Mg!{Pt=+` zGmAmreTQZ-oVVxZ+&{M+I-5f-djkK!a|1|qwJSbtZSVC(u7_)qD6MS!1b>-!i+_LB2Xax`S11Y_y~I-Q^fcZpZX~-8JCj^Y~w5D;ScuVwF|1ndqh_)%mAKNs}nfy zTF3B8Ke#0>V6qf;7HA(Z@zqGnp?*_X2r{Ur{`7qtqE)*%mbMwIZvc8U``z^lcPQbm z@jzdw>_&iVw#OYB7%=%&TkG<4i?KvMlXyRQR=m9J7WKh{xU7H+u_INVkEnioabwCt z3@*mm#to;`^vkVxBlvYN#ZXge%$3ci;;E|rF8fvr5lQj$b6k9S?M6L9z3AS{M$*$# zqml>%_k++;<)rFw>h)jJXw=4ou@Nb?cHhnTgee7WA9JdRJ=@ON7x_JD;-FuS>7&YrB&lNF=Nq`6>Y%X5+n@{Onp!C^+XTWoz%LE5q6_ ziPmb}=e)g^Qpg3u&W%oYpmAP}b#>1n_u?iz%Pq^@H;)jP9;^{GnqOp`gw~6-o$vh4 zUiA{Qq+6?{5*@(CH%IfV1FY@(UmJSq`P8>wiKPs8bN{i=P~#wvU+K>SV!zMEppI7y zs+dH8K>PlKX_RJsV!4+u>%V;0W}`2#4BycOev_5kn{#r{oG5=acJo9S8<+f+<+@2Ia|-*^}pZJX%&fOz1Ti%f1j@EJSwIV|CT+u)ZDwcnRm z`sYyUC)nP6b)(kL@Q=_O1Fyg;CCB(R&u`y9(HAk(u!;(;8WuLZoQJ>McF~+g>{fZ> zl{FuH{NTBY@1c_Bx8DXUYgiV)vn?jmn-8#I^;6WVv5oTmoCWid%KS>R+7+GTB!hSX02E9Dl*3YE3 zn<`CusgT%riBFZ6OHesgtTs1+Qci*Hr2oDRvwJra@O69d>P&*Gsa$f)`(*1?7Y^wx zvF3>It8~9AN_2U7d9$t$I-BYWwfv)rB|sMKsU66veq%|Gv;57w=0UT4%=f#Q+8)an zL`-h=MP1G?PQh2Os$tjNgFRDLcUE<+pzB$+peLD!STX_~}3ho!4tt#q^y_1|w^n^tnsd`m;Tl0qjiZd31no zV|Ub@WMrHKyWRL`W~0=Fb8LM24$d23^LvFEhzt5U+2p+$0@ko{#&J6T!#vUF=T1|t z?hlI4N9vrUH)l2hHpL2&&|++G6##a1w8&=M)1BqFiWXsX=9ajeD(Lo}`f-@&Uhikh zM>4h&rH(h5=}jFIiiKKL4dbO+9h{#n&yFuex0~(r6=G?@H?}uc2&+o6H9)~u1BZ4( zUxL9+xD4kW=<{39=}@t8tJ>ryN3>pp<_yX3hZqZin)NpmhZdV(6+G&dqpxcX69~wM}tGAZLw44Z zVcShx+VMg|yH?4oB)OI+Dxx=EL&Y12v(V7ApPV~xhA zStzYP3v0fc;rKx3f89%Z4}O3MJmtaXP#(2RM+4PTj~z9*j$Y9P?4hBhH9M(Cny%PlyxYyD*4WctME=ZA^6gA$v;yrDe#de&x&2 z${2Ucr@a2#nS#By%}j1?WL-5nhEI2gM7l#&=jS*S%RL6HY;=1p5YZHt@V*Val%tv2 z#O<cNPpCn@#YY0UHbM9Pdpn119{RA#wjrT z9chI9(x<|*Z+6IHwD^I+8nH0oF&O>Z=@=$>R{~rWv|<`^*?*wl`WXDJ{BhNw^ntMw+j~p3UFbzx$yiXuw2s>*1D(>kAX`*vh{3+ASNzW3)%P9+A8` zO(Uw%vj#MsA8sFB0?iGmihpz@3mT6KV|O*>?;ed|wt{|quj(*Pz`q3f+Cc_uH|CCv zgd~$Vgp)yA)f5hTtZ4_PQ_Y&z60wadHWFOId>=HU-1V zp@ea)cZ?o$J^^biMcRcNWi98qxN*fw7ju28~TbyoWAWG?WKJszNM$JDYO%4GL&Z!xxiG1rm;?mwG0 z@lN5*v&fs|DQAhZ4}?-pL!cNBj+<3lIa|)CPHH9W2p8QF@cVvjj=s?Prs-UR(&UXh z@7A)fZw*ezX?(?0#d0ky+dGXzLsbk39U%t}d?Z(k}Lc=xd5Ev-5{OWXP$qGNYmAuD7 zjE%cBO(zRgdSx5j-3Z=?RcfAjAehN(Mda2746{Mh#r@WtZc~sMTyPZSULw8l&zc)d zBCFetxZ!>uL_n+;^r~TY)#~wzHoeHvTl>~~IE?oD@KPHE2B;!d8q}O2`ph3HVviUI z7ar>b_`DCe@@ts&Kl5|A;1+>zg^U*gfk|#pS^ICV6()->fYDb7N*$wlBVn5u^D39( z#P0WW0gHPznsQQ8;=Ee?w^COp-43;_Ct73%ZXB^eED~T0i55h zPA&77dv@aA>J;hg(b8RJQ;!u_c=Y&7aMvAs8>(*%4+Poc>hHkLYJJrjU4V@ZkFGM} z-_vL#vK69l9csg99J(M^$8}SjZ5AOHIrF~5oLH7wP?V;VsKMmtKi34qrfdBgk9Tfj ze5{+c{cjiFklTA9;vo`c5)U5A=YX2h|Efqf5BhqG?{ge z_;g-Bc|gJhzyiQ zB&S^sX?qiahlwc%ly4Z?$A()uz%56ghwP^pF&*frcwfw9R-FOiP^AA9RugoxIE*(I zv)KBj`Bdr12aET%ti0Kr9120ail2!%6-A24)KPK;M!4OBt4AY zS2+!z$D_V1>-w3wyA^&=VnOCgiy>G5z0v{TvkWCS`gA6KK0Y|`=G!jS#UWqmbC=2G z%3J@T3muEv-qURkB*Z?Gp-`Kgx z*pET^t#{8f8sK+<#S_8(?qxYt0`VhdV(U2i)}XALJ)~y8RLSk_%xa)#>+Qv!;>C3b zuv+#f_WDPP-uY$Iwn?Upm>x7jZ6Jbb)E-_BG6z6Bm^kQQkx#QKNKr$w1$0e6EQxav_BrJ1C0 z4!Ty5;niB&AdQp7?MLTHgY4}b1ipjmJ9qywng((`OzONjaaVxaJ|8@-U3Up(!)&}n zoe!*}0mF*}Ogmpu2waMJcq6?si=W?(ocrWCXW@Ex6{~@XcQsPA4p3H+;|2;!)#=kS z2|LSU7^3k@@9CLP2~fWEs_}JK#Om(w4Zto7f7D~F?NYD7AOVA|R*#BqqT$iW_5_qW zH|3ej$77#;B0-n-L`1z_q^rlc9*bTU%T-kAwqwC$mo9s6LzAYnb0v_qN8|?@TB8br zHY2ag$*YXqbl7{_aBj{X+lK(xzBrn4Tag$HEEonTN@owdg)_j4A0I7XT~RyMKW&wK z%NaRQ4>Z15h1kr4R^{*+?I$DQoIZr6h*>s7^k@q0gyDJINkf&@uuH!YbcYbe@uISQ z5!%6qTT%SrRNap$LTgksc4D3pz zUB!q5!u|$47g<%4CEt?n*I2qfK{HKrs!N)54r8a-DxdSp^`gF|x;D<=O34n#X&BuhzH?;wcQMk*F8D@?5t696Fb=MOQvufOp-r;07o>i{vAi zYgj@|vv^I~NRV9i&Euo4q~(>+B-uXUU36IS_ro0Fz`cpio=tCn&r1Hs41)@Lm#UI@ zExLTv1T5`Mi6RG_C)6F?a&=Vl%05Pn%38+1v}+_BxfvsS=Oo z^usid&;9#6BFsk%rI48(<$qvD6MA1l(Fu+RL2K1s zId1}3*Pk3TdlPAMFLou#^|~7>_td6`ghd<%mKFub7HOPq6XA+0i{3O1>@VOoCk?uO zjR6vVe))}V2QD2FrZEedk+|Jqr?~FXNZ}(do^K;fGF-ti2m0MBYFyY7#0$_OvuVB$ zBS0R-K4jd6Tz*)1pO6u~)jLA%PS5Ol#H$P8dYdL~!*Hi%a9{8E;-(gBheIqMFd{KG z`m&Zck}^m-JJ|nNq1;*c0PIlNYlBt5=v(WDoxdNsWhUYs^qdw$x8sNr!%$Cv+RN#JfR_5*XCP3cyTmioG`*yt$w z>TB0Qcipxg=tDXHD7Ep?XD_a>b-0HE#i52Sf;94hXKGEamD)7#OEPg?$u*DpM+_51 zTD^Eo*T`sZY*${s^zmtKZ>t5@s%6z9du^GxA96J1AOPRWaI1D76}Y*q-*h6jpR46N zX|Vg$*lo+no&&Gn59%{T|Bx_bNE=^t>rsUJXlKsBXNY1Ze@MMj=Kk4UW8}6(ZD+y+ zS%qcQujPbdDNY5jcO7=_4c*NR$a0w9y#9lgS%^VWKGeik4#B@6Dr#{wXh_)Q$yURv zD{*y5bnl04gqzS5SkP)0yDxI19~V7*cmodGzr)lkHhV0SFx^7JbO5#VU`(9U6qUtGuf z2G8&7u()X)hO?p1@zj^CfO-lVEF4Cw!Ibq|L307tVg!}#O-xc&!0cc|g`-{fM;eFT zqGCQpQ6K+5=@nHZS%|^X>-HP}MZK5)0tt!amfpdepd(j!*|cGOIGi4Jilj1;bd%J{u1D7!8kH8CSw4}Z+w0ov zpHm59rp5>v85KYn2-?#sqdBG(Z$tlHt5{LdmSn$oi>WQ^_7C#qQ!S{ z)Y5>3u`GG7S0mK*tcBuLtue^i=RpiHp<)2(Hb5H2NXgNzbBnl&atIUN!9Z)%R(pEe;>+R%P|sl*VxCI;u%ln*MtIci8bRxv;(Q3wo*bPsp&J_FZg-~xw#TrrBv|1eZsuvM7(Psdm$#1sq?b%W zup8f0ir*o*x#ftDWg{#F>NRj@m{|?2+%M1S&Y?khW@{irCgVgym515I!(yS437`>to|ZgkQj$z8MsZ4fNiU#?^DZ&&FNOBLO7nZ_ka@Cs{^1X!YJK zwON1CXzheZk{(a1^lz-5`Ue^YjZC%Jx}srfVU^}M+E2i%R->dJUhB=#)roB~Iyv7H z_YMG5@ulkZm1#i-b7yhKRM#T2azdCym!PvCzde_iWzE~35=Z*QGg8rNIq(Eim6nAW z-TGM4qP;>s0cH23O7Ltnm4y7fH$wV`^@jl*QetIO=n>ed%|tbDL4sj*i<^${y%LwH=t+ zptTNr!>ik~O+L4B_nb5>6Fw$;!ppifrMw}cnob;TwoOPI$ZqTs&t@jK;_9^N<^~BY z^>@l^?C9uRLox_h>C_tYuSB*b0N8gcRot8Qq0R7Dkky?r)p_yHr))YM(v!OtwwiXBgcpG!NegOpVO3vw`w17 zF@Hh)u&k^ATU#0&s&aqR&d2ytgswImzf_vUW<&Dw)oS9;VN?Q2JSS6LgQdPOI&D;wctx+hb%N};y;jx$AZ zPT5D5dsRX1X%nXX(olXB_DjF@5}H{KR3GDFqu|gdWU~gj{caPbT1?xU7ngoHljXX( zp9gm8ansW<^d|$fyE~=AY#aDBNlH|lbGiI1tDHE}_W`&+=C2mU2BhXEs(<`QNHWen zQ90%A0aa0Qr~(`8>qX|~VUa`NZ8Qv+DQXf1mBw8%1E2NaM9cKjR_3%~>TAk@jGtfX z#RI-5CO_8;2?JgFcfT>j^_vpN?QMX%?HJ&?tPKD+q#{3ja3x2B-f=056^34kZM#^A zWx@ZTWS;9HS4aryitlgu0wpN>{xk}N6KKvyZsr;z$^~%Gl3dId;7FoEK;xFSh;<2x zia@n_sW|3USE9GajubiQ%Va#deyC}Mrs(rtP>!&QBlTn9P1{}4hY(amhw0B5pVs2} zF#I*3L|0c={fjkKZgYz>h=-T}=S1_m|JCVH_me?uGw zzQD$whAhz0Ugi!>CNJVp8&8);%{px>IV8$p^JlUC;SmJBNJjd3Z{_6&1aZUx6vZo5)SZc>PGxQe0DII{H*!5~?VR-`-Ozxw>a-|qY00o30Vh%+e?#Y+f7 z|HJ0~F8kNB?`0^GZ;ge9r2lv8->rfSkTi-j%M!}pO8DPh?z6ujT-5n^^t;XfuJM9i z2dPVg@aO-zBh-7>$ULI+gk{nH(o>ICAU&fFBf|Gk0H zYa&fls%AaHq!I3qV7fR=L#&j-gnOQ_)%3XFa)#BNlXG+ zDQo<1WRYO7>5cZaS$(dbNc~0rb}lko&&!(-m(gKoT7C!&)1NW+EX5=`@Dh#=31;GP zo58zS$QfRl1Uket$c?*4y;4yj$7NF2P|wpWj3p)|B~Aa~r1S3;_1GD?@4YNlkgAe2 z_7zhWzgnWQ@pffO5%S4&y>8uJeY>1yEv{&5`>hyWu9nDor+_?IpCU8mJ_*Z>$G#?I zf0a#dc45qJG1q|aEK`!IUSW1IUitZ9$Cp_($soA#d>YIp-(N&}rEH3NT zj<=wBn8gPUvLDY2eya0rBhl}vTIE|ASs4o>Yh*s)Os2#tomi`V%BX|tTq*gRDm|3Q z6RJC8g5qRNy?Y6zdX?b%7Bk0s>affjDWUKWA7%?EV71_+)qb4~zOs>R@~LIpx+@O-CQW7W z1J9{U%gc))Ly9O-uy6{E)z>Q0j7=!g?ym*O zk8XUuqI^`r#e4gs%#N8uuKenm|;Ls-N_6b#|Q=gme~u>LJ|NwLN` zCL{{v6awqNGhitD;Mq2q#WCOA)dw6}=mMwFy0O5X8=ML-9XJZC#Cm{OB3bJIYl|QIE|`EZv=64bC(! zxLqu@*ZCwlD?m9;SZ$~UDdZ5+kmQb^>AOEU5s3niS~TUnvu zs@wRyK2f3p?0CC7>an2e5$$Z**F$R<(JK`U1J{pa9Ob=JCNt4}q^{Kwh4kjw<&YVC zh=AjIKF@4~A2A6@$zq4Fm*!Ouu84bKMyQ%t!z>FYe{QAuuQ29^?|DkHwX179Gc$9~ zWC^TVLZNq;2JHG-b}eKd_cwC;cizQcJmTA2B(VF+9XMiJ?I-Qu$qs%sHRRDV5=GU= zmgUa^_mhy{t6wHNgA9s#x{U)XwXFqbUAhNLV=C}p!)}q_X+3z@M}g1KRT7BLxz-oS z-O8nOQdLw|Cu7a8ISnqPOB4Iqct( z4Z$gSlAG}P4KfvWWd;fNxUbhH(g^S024iA2y7Js1)1llMy4*9I_PeIw^%S@_K5i~k zgWsCu&_lUfXxZfiz9LFfO5vw;Wy8;H4!Io`?!IE4tRVASEq^kpynceOy|?2Tr#T6} zg^9I{LeV8|c(sfuZr|QE105Q##4q3iN&Q{q8S7T!(*r^8--{OpDMh$8ROO@*Az{2K z%I4cOA6}1JsrrMtgnazb8a=PYC%){Lj?P3EKvU|*1fh`cR z&#ZusHmkzvfrs1dxbI6R0=v=DIf1o(SabmlMN`*I(zIJlxUQ@+cUW`@NS=HNEqlCL z6Q2a|I%7tFMVk8(N#54)F8~Zvd#%)`6|rzhIXEt)VJFiuSe;gmq6f#3!Z3>r*G<>7 zB8mzH>-Y~;5+j58p04{Sg#~<1kee^i>$FQ}wK`5Eo}H`lCk7$e(g)vV8o`j9F+&;_ z@sFttM#hIE|855eQvJnubs4csfofMH8s;Rv4+hpD^HdzZLlfn_mo^vGjgzn|Q&0e~ zh1IF~hgYZmWqTQ}+EQ9)gSW}ay1-{WoER38o%QSKPgq=`pLO`F%c1B~hJGo)h9^&L z;g%6Q2b@R@?TSFqVAqOyT3z;Z@Ftr7Ud0Wf`68AJT~p7#5 zAZ|yZui8Ao6Y>r8=CRx-liH(?h_==RS#t)OYQ9H66OYe{FXJOU!HDD z84~7oL|&glpJe3QQvI|1$E-b|{YzJWCtBpMY^y`9PtGck`NLlsue#R4)9Uho$SxpP zUwhIo#ytbE_qG4UZibi{>U(k-yfJ#SdakyKDn!VyEQr+)+64_P6&#(@-#*KAi!}Cb z7G&(qMeBd$Sl^D;2Aaf5!2DxZ?D#zZ?+=B0{Dg zbSgEKd4uF8hmtVyw_Yo{R%4<)IlRy@pCgt)cjbY#bhSE;5PNB0C4tt0IH?|z)3Ft7 zo80MHr>`YqR3Z^gEiiltB8*DD;BXL^}E?NRkE9 zf|9_XAn3c=;np^|-wfZXxQv7P`D;7yv^{B2-m{AWxHAExYi-9){ZU@oJ8o`?ySk!J z$dnzcfa5)h=}-wVSBh`Q1jN`fH?MqgF%i308XY2v;uGFX@H?KD4I8{bZS#;nV75Q) zQ%VPUF1ucFZE!-sqp(;xGTCtmi+A;_DhM?2HiEQh)vV;RNk1WU0EMdV&aUezIlnII zvT=d32fP8Yn(ok0Q#C25%XHVchhwPyH5vG{CsAGaYGCZFzSP4Hmod|Ck#sY!3Y&6c z02fyL=os9pGNn%tvH~0KuA{g=f53D{vjd?wD(d(-ZQYR6s~aqE)5p;-ww82Oo2j59|Y0 z8qvtb5OC!!S$9LWRZn?=A?I6%v-u|(jb{lTX-ea?IaPW&e0MXhuITs-7aR{>YDa#F zN|Q9pf>vtR;*B?X;jV5E&%3n%sutFb13VO%y!=cT71tg7@`}l@AeI&RH_!SxeJnmp zE$2{rLYyRap(Qb{LL~*JZW6cSsvTBFxHDbA~_yOs?ztpaP(^=GBPxndF# zFE|AeI)A4wD>BzI7Kz{rnImM~XG=WXk}yxU`sQXbE8t4;2P6UUS0(=M-!aXfUL=`Y z2Jb=gQa=7<`TYN4gTeEMEDA}DETQ)Dthglh;Y{EvJt8Dqmry+JxJXMA2tnN-rqtB@ zDd~Up?gj^+d2e$D=I1`7SCZQS$}q$a9=gOo*ELM-NxVq;QiHvRgdwS?no@wVJ&m-X}O880h< zc4E_zQ@NId(xI@8)H3?^QbC<0gRNXt$t&F92}hHBJSyJrp>%qYny{OaAMGw3#&L#? z9(ENQR0eZzo4Z=qdtyw7pDPp*nGRbh7~^|xs8L)x8kbnA4RZ<#$XT-<2fRt9E_&Ho zq8&nt9UzNNordqUtCeK4p*}=ojZpzg45hJtna>&a)v%}`jmR{#i_uS+~tXH0m zn%&q};3Ivz&&Z#dKB{cH-hqwVjGe%)bE*lX zaiL9?kwYlcd$!CE*~t(%I>YH=ICN1ttmIi(Dn!Hw1Z!vm6yjrw=g=R>=w0J{~-UlMx?>i zS2d-*pHgFb(xg-?#`U%}Tu=CA>{!Q>QU$(k2l~feAHw)EILl|mT7jwZ=om=p@V&6_ zWdS2cQLCs{5MV7c?a!|0r?Y-#nSTkrDQ((el>eDR4;939v>M7}%MnPfa33E_xu=b5 z9yFQp>b3IZm!I;bx-}UPHyA+2VrIzZ3u;iPB;cqMxn^4WfBgY0M^C$ zRRyg(EeFKVh`4X5zwGXNpucj?&G`@%BsuXdVU7c>D8)p9N$=UVc`6`rt()JihH*z< z8@O1--8w-*q-3bJSzPtdzCIiL!@sPp%S(L|u8#%CZgGVgAGQ!wo^AL?%UGvO@Vwln zZJxv6WD7;UKp5db}Hw@5Xc$F?=<=lT5cN0 zMV>%ku(fdYA;o2--5r#h$)#ios%*g2rfYAwW?Qd2N%( z9zx*5wr;8M$Ox(UdmZUa?J`I1tDnMzO4VF4MTu;7b*i!eTc3dQ18Y4bs|ytW=DMlQ z)$?3B>0{+lNjW5x2T5RNPt9?`{2q;pL(%oG{!yqcDfI~=2cXvoicMH-jNDlFy+_!Pv*Twb! z^d8Ug9M8cg1~}*3d#}Cr+H3t*nHak!ca;LUH~~YVV~jvT2oMmLf`>HS_3!u?nJgmZ zqRyQDdN6LY_-4VpVGEaLr%B)F_4eJ}?Q``^z|xcjZnPhVtVyb-=zhnfhTQ@;2qR)+ zr}bgFnj}n5Ras%VI@`{b4wT5Wmss({b_}&L>Z42XG(i=IBqa`&J-+CZR-SNYdwWgf z+GbTJouf>|qc$>_*P^~*aj}_F3UDV zGl=@#`NMOUwGVCq9oN{Rl;~)^mFt6O7G~7}S+b!>5#{YMNAN7<8PQG2{oqT)=MVcd z36s_5k(8{17N!#{Wcfp`@zt|TV*^X7rh5NREstVwMR35;R9~~t9h)K=NWuoYTng~6 zATXe{ga9|vrLcu{wD_HZ1~&+1+r8pP9LLz^`SADYN!X$NXc-fdn2SsBYQ=qXQHx6e^ZLdzX%z zAyj+%1m^+XA2Ig((^pM&@){{=X`9a(A|f9@p5==^-nv~W^SZ;YMe7B_@o5g7lY}4L zMqBrinh(hL;FU5@BP``n>kOEPnpCReZGGeX=s#&H9Eq2-%oAv)k4cV!MGwH&Pec8u z?Xo99i0%W1kdQPv<~w3U#52}c0|+F&OL5QPL*Ykf zYMyS%^`(czuxm@}Orvk`fB-M$a<&Izp|;$ZG>!4=4Oe4s<~?tF_m*@>=-TG}e#+Y8 zf8zL_!GJ8pIbdhCCIsd-TTW+7cI_5oxJW{GyVy1h56s&|B*XS5_?qNv*^pj(Vp~6$ z?tKBWuHPYIf8lpa9Drm5=v!L$cwUHx%m%zIDxKl>Y65kHP?UzFmBEB)gq64@(xJQVirQCe!gUGdsTd z+WAj{12-MuB>;y`C?=6hgPtW-q#E+DKLzDUdgEje){*d|lfuG>AkYsGsAE-a%>Q~f z&%l4r?lXv}1J2Tx#elfgR`tAbOtIe^hvh>|yc6P`B%(#G(5YfDXSiBGorst=8P@t} z&q?I@#$*hQYH^a3kM5(W{AVind8IC0In>84r4ngGe>&`Y4*v!-0%wc=j@45^fNWcy zpW{(i_Hm)CHy%m028Roj4Fw~D0)q`HYrDvxt7Bfrr;i6ra$dwRuka*ETFaFy-*}d( z>?5B8roGN$3;~hT)W}ZI7wE5sCX#)Y3?%+$H|TL14B_GuW3@%RejTh_Ra`>UJL)c{ zcjUllYeW~a-N~_z5p=AI+!N5(-wWtPHLh&W>WhLx98}+)=X@Ck40LTx!V_LcVAkF| z{H8gR5#6$(B?sUBXuR@(Z%c2lEH;b*A2rO&#PUy$Gq}Jq)B~okSQQ7DX=23Ut zB24>U5b^>Elt2jJvtqUIXh*p_X;Zfq;RjBsM=7UVLhKry_wUyE5~F_zSy`7R?^DYK zr+$fCgh4uF-9p84wjy>tuhUo+M z9{FDBky}ATeu=v5**Tnemo??2SPC8;xqA83gc{89E(2!d0j6=!ksNqI#3Yxf|2o>q zKHeIIiGz}``cM}1KPz^KW*Id=;r~C zhGJ)DZC_NgidUn!U+hV=L}d|AWCp_UX-^LL2C%@-dZ480YNF5`HDnez!&5wf=$ZNZ zhl2(qX?v274>trq?@-}7O9T04W|3SrB~K3F{^2atbNb4|2FdNi+j>=P?lr`)BZE4N z^G?YDP>G}^j#Od+NRinywi5zH}dFB!A?+Wm|A>a$=Hi=Z8-(?v7%iAq(BOZR)*Vs~59|`r+EY z?#TQU-~0txA^|@zN*&I`L4CnMJbsrrmpO>Le$(us4_lfoEat|G5^iZ$);!dJ1&F1Y z>U3%~AWOARue{SXDJV z5pD@ZxJ!%aDF~)}X>R9w3XNUR~P01&~PM7QvV%1PGMIZ#zx9i?d38@&?)1 zy85C*ubn{}oNuOhDgMRw{TD@?1Nd!2neS{sT0=33-aDFZv9tAl~c!zpu-b9j9uBo6(c|%tSVvnUZNpO|!RX zqeRob?aQVWc2%+RLVW3NO7$q78S!5OTW&KkEE{OM45%M zM%G9eNG`JCxY@fPtgPpDXseQt03GGVCK9-u59ExVy+Gz2GY zn*%5(vFWF#r3oB*H-UW?YK*f?`us=S`+${R*>M?FrQKC*5i%H>JAz&Y$ojySGkdLD z&HWQ9do(hgW=i~DL&fdYIS{$E-&VN;J(Ke;-G;p~S7u0lDA~Ers z?MMnHDcSIjAj+`Nory-iym_VuqhXupn%FdWeCw^*9KmjTM#EJNc0Iu?*-e!twP z@(Z$H48Pok=aZ7qpxf9~hD-1eq+Xx-=$@Yc5x-L^`40!Ko7SI%rm?{Ig_d?E_|VMS z!t3QHFAOM;vh5h&nml{b$;(&Awp0jfK+8}FNhx47EqFl&9D(|`Db0;#dPwUlP`N%% zHQ{He=CAptMh~YW*T`5`L%0`4O#N-6;<;Epa2Mo4dj>w_URp4sd}B35S204mpeZkE zpq_L^#E15O(j|WhLQhc=AgEFIYs2OiCfHvqm9(E01q7i!^dXoXdyQBd>$P=Cs>;gS zta~La(F`jcxvF@}`FnzZc|h{GK5TgFq2i@_eHP`qB?6v^&`=e;dc1o;!39*kzLK)% zY5?&bdq@9KsiTqI{AAP>63~-lyNIKEPW}`bDA}Q7#Z#?NDihyNEDEnlSoo`Ea^xMd zipgLUi9jUZF(TRvQRwcCpgjRnopzfTsm&bB@1!@KN} zHNcOB`=<#r`9OIf4qHGJ9NQLkHsz{-ADDeXv`G4Q{aWpBr4F!Mt%(3Z2mJ3$v)F(3 zje-%lxuEJ1S}G;n3iF0$2p!eNjYLB(LesHwE@@N9WIYx#;H5Yckn@Q&!kw2kNX#oy zgWNANhDyG5ReUVXSc5o!U<39c8JDf2lwA|g0n<&;KTEX+zLOwW^~ zp;(2Cn@?x%N(}h>h&|JD?#Jv^2}x8I?aJbY&(_>-+&u-BpY)aVJOSqlXpytTuN!QB zhHPQr&H^M}8UEsZux;Pov}2^gI6cL(f_5y)xRG^Ji^g+6=S-VmR^ga#IlN;YEcleX z@|eUrv$yW_j)es=((dK7moLW6n_%-7v`Fx?Ea%rOo2I^5Hdmo-iE$;G^sQ>@ z(6sd8hUG`fXj)uUnJqG1!c7EPi5QTJD#m8DN>}0a{@r$7LC5MXn_iRR-A~^+5B8*3~Jbl&z2cFI> zrdXMOoxHqRMF#fNhf$;!Z~-U&S+O#@i8_JN=PS}re(>tG!6dEJ1&s%oW57M!J#4AE zG!3>{-*3+~$K)$)P-;H8VHb>=;-fMhRv{8F!gY(-_AxW2W9n z*>l}xh2?<~^|Nl2n9A>v-RZ2$zH8ECULp@No?s2CQX1LQ{ErAf&+vh4 z084i&nQ5s!6LlQ{i3AYK+aj>tcC+rAM0ljDA>X(#5rISU^$YXw?Rv|v?%Mbh45uas zWRQasy-$jDLw(7MIn;+<f4n=OwYP^d%VN9J7!TcWQ@@m|jKFYe_MW zz#XD1f~Y8sOv6W%=_+H0%b+?9L=-Idcd-u&D=QeWL!Ek*&q43VbcOv;B9CBgEFj}FpBwb ztdo5ppF*Y$>m0+O;UR#B}%d5a~SBEEh-$6u(*rQc22D0t8&0dezpW0;JdMD_itC6*+(XFA55i+$?z3N zqt?3dlB7&^L*;KNiJF+}rZu@OlGjuIbGiUT`4k`GSL|Kkg%b7~0o}KuDx8g%8*g6Ys2rkIw`23^9szw$NQQmK zjKTMGn0B(!`{y(vuE3=Z#N>GI=wgEFLJ7T5^VeXyy1I{yjX!k0>b|Y*P@a^m3ISgx zREJU1aB_-iCvYYyyA}d|62@n&t@1FJSNi8*J>mc?nXtI`>{C%N1lsSMiBrUr3aK~I zcXlpMT3khhhs)qU= z7Q8Za|5jZQas)hBpCjvi36ppyQR;Q7bTA`B?FA~3WY_9#Fl+LUe% z_}7AqK5@W<`#VOtaR>->SO8SN$B4k0?o3?z=cWM^#stzvaL^jervPzUy>vh1-)b;= zZUUwkPK_Y`E-@H}S%EFOe}W078F#uz`B$~&*Wvd>173&za(GP@z`l7X8HxU_EF%aG zJUBg+S_g-EAztnY<4natpgw#-Dd&ky2uK(>{AgDz+yEPAKjj0#|u!l&dNq&dT5nc0HF4iZS@RNrUg*k3#J8K zNLmb>3rUNPc_C@hFfP;sv9%}uqK{~hjZ0>U5=6yx=riZJ5SlIb<*(f`cn=L}CVeGu+43a z{vcpTG@UXM5Y(8L+!x3FZ-hE~bFC0M&CJx35=}aosqELy1JTCm=`!Ab&fgOPfnnZg zFY*9A>l?WR+rM+j3t4pZs%c&^+BS zL>p_O%xfMYhGT1|w|x-}4K{XvWtg8cykIS0PcK;O7Ykjm*4c5pkXpZ_(}mOm4Ub3I$>K2rr5Y%Y)>35nfjmj}kpzVqH59kVn5Rf{7^)0x>=2h$n`; zp`%OMi(T6}h_5IQl5=PC^(*FLmxzlK2b)k$3&aG_osmsHNBoX1(j9y~B#00@G+bQ1`O z(|ywX>f8`NA4OoyLUrO1KQIhdyy9hXUj(-4CXDwhSnq!V_%_cX_AH(iOMvQJAUZ%a zykeNfk(3Os)3fTb$jS+KS%xFnbe*|?A3^mPyvyfp09x@ zPR7-E>93quS8=fV&J8hu=-2N&uA zfPP${2mjBg2cDJVYOSrUj&|o`bZDNSGu$ipIdqcF3xRNW8Qjq&%|IUl3BbuG9OgM6 z{JL^)sfmn?EGe(2T1TXo8UU@_DRj!NjNc_L@ptp%zrc7c+@8+qH2GOlVpjYBlDO!%$Xzdrlko}?h%l$_ zwLTXads6GCldlzZ%kS&)&ujP2cr1QTc;RiRUvnqTxp}z3_!!c>Bw^BWW|}X4UQ|7i z=3Vs&7?cM0F|GC?!PngL;Mjy)FU!Pqo+oDbL3|L}gbx!tFu^^GsgDYK{fzP{pgRgR zL9h%Xx^s#6ojA9=AhNG+1u-XN85nKLYy(uAW-ME zl+VwADq0BboF4siN$?*ELV?DGsTudgY54L}nUhD{1@p0n_u2rUB0YlbBy z8vY-}KL1}x7!|tFaXM+~5W`z$B!y1-)=>$!<9*3GmB9gMgra2DbX?^me6F>``xsvFzA9 zG``~G-}|eHR|_+`14HqgkaKJbjYmwTnV_bPfoWRCQ-|>-(zqy5#|8f-%_@eZ;+nZ5 zOXTOmsP)L5KZoC*Id`s~!%k+fzkjh4e`0BRUekHOXL2aUJe(BL{TSdmMbYp1Sy;Us z>{T1Y0Tiv$^RL|W{zxC3RAC*RuzsHYK)0gL?3`72(i^GE*>zQWST+8*en1``4naBD zGSrGa+l>d!FdZBm$hQ_>4jTHs9vY(D?eY(!yh;;~Z-5itwuYtv zUKL4+;w~-6Fg)%uIXKEP-x>wn52A?OCA8h&Gu&r>xnN_rvy**oik0;!d ze0fSIp&8$sH@r=`>{wU_ycEyt-MZZW>{tSQ_s0d>B4h)?C$`Ut@ROrvqTRhh?kB+6 z*IIrPT=JRywwYv2H!CZvCH)?a1#icnYAnNVJvy#GQ-A&qUXekHwAY9{X-?P0D6Q&JY%82{=s(pNj$`W4rb)?Yuf&&!3s6av<(UqjX_Q3Xh0-Pw$cY!0$lh4g} z(RJNf5x3Zw%w=FYO?A`1h3*UbmkSPa*WY!Bz)WJDw}F~SN8R;-p)(1Y*3&P4?SNYv`_nEj#EFLQD zIkINM9nsN^^SA5iR9p?4!2UX&j{x%07Vl@xVu2@#E%5PooJ9vc1Jb|kDSkzOhacC9 zdMjiMwdBvSAl=Pn8%M>F_)~*LQU-aq>s<1S55)QQ^(PMTw^I2w-c>cP(>GJO?GtE} z?c}UtGD)14ROp1|oV5aDS7jdT=d#?bOYG^S49JVm~y&MaP2rAexvBhrlgxYLFGmj3p z4{$$nFN%-Ht(PmUyKHJLjU@W>Ve&CCvEH(5uvz6YCVcR%KP*{xLnAaIVk})uty()- z&20@oe`>#KuH7JRG~I3$W2<3=IzYh~pQ1ZQVa0Sq+@g|89PYeh@gpfir6C+jTBC#xVUX#%6y9YOS~8Yk3S7x5v@f;1+)jpl zQylV);ugj#x=nAwl9-`H=8K(!tN)clA~?ea6GU!bXFc8dnK!D0q=7yWi|NOKSMZV? za$>FLjZzmMEGoaqvo1PXJ~a=eC5)rf#2)sIDa5#3y#G;aoX9}LG=6F)ZDU-oT zX6uHhHW{_$c{CS`J<$bsJ=oi6ouWv~%(Na$PfN3-&roF33Y-YeDCAVkx{-GUA6d!8 z!y~Ktd}9=uAxp7fLf%JekYL6-_h0QjKM^?N<4++KqDa`FR~-m`PQmCRKxSfV5p9&nI4p_Y3~JQ6@Hs zyt7+D>$-BWtMAd*EloqTJjrVK$;f(TwsMAs63|xa)M@Kx)+=?I2-gi!2ZE9;NPQ>M z8h@MM>8e_1=l9iuE>+{T=;vK{2$b4z6rX3c!5 zJA(@>fZzo&c(MUyI@=q42DFxupNf4U&~$;J&NXwVcq5*nHeUPLCI!B|OBG@eQw$Q= z_|6bYZ@m)Bx_zhS2)1oHoPt+llXjJyR~hd3UEJJ|wn9q~Xouyr>l>lhF>2SFWkUL3 z86l?I-vVB5LKOpjr5DuQjMr*TUFv+n0_6_xh=pt_uP#mv0TN}X=2+|m^#!jt*@p3E z(E=5x_AuzdL8GjRk(=jOb3s=PEA7=-vC4gH$al2eHU%P8-y%+ZKdqIcm1bSvG1`LoF)4eoJ)Po)eN4 zpv_Rs8btf3{8niGQQmU!8zgsY4V8-=!z!OllgVfAy`(%})`G%>Z&R{Ivei~654O4! zgW9PY{9Eg=r7d&v@CJCMqM$%;lJpA0waKguioG;i-QnQ6^&44cvQg4{$!=%(L=Y*|{L?!PT;@@xYwY=~PxUvb)Z*50*E@#`l{wTt4vB@&KDcj*bUi zZHmTmw^D3l*sPfh%z(n?f28dU_`Wso4$IoLh6QW3{B0^$Q-Bke(=1_9ljfU z)CDVQwptY7RO}L9%gx1pY&!8->U~>_fM5GIhqV2Ij|3HbQ*jMJ!HWM7%`@|Q zF80PMIyZTGBm`!~1P z$;mW_^*4(g>jnmaotg#p>z`}UkGVeHpaU8WhlxwQZ^hDl@cgad%u{?4MGLNbVG37< z@efzbE2ZB)*t)$mmKssfQV@C&r+ILxYJY_}jElc?JBRSmX?4}a{p#bwL+vRmoeA-; z_$5b@`tJ9JCQM5X%wk=p6-cift+q*xi)%>*tzm(8AMAT++@4GrAgZZ8eC$x?)SI?Y zruE@)x%;_4HMe9Z@0zaDhWAbDrFTlLkyX3l0ulrfPShOTanccZ9QTzNWNvT_O&v)n zp=-;IIq|HnGF1XIoWN(>$-=8)7TWLOnEe=lYArK}VSYnKrA+UK@6KgaU!9$=o9@R9 zH2;g(Q2U`EFnMWbHdn^He5XsLEKdZ!6Ug#`Lw_x0hWeg--ko6ggKg^8=55lJs0DR8 ziHvVlw0><5kE?v|MV!=Fl*NCnyb}1H;Rq;ym9P0V?Cj|35)+%d$VB&-#Y-#4e4wrO zlJP>Uy2n}i;J{wgcDe|nd~Ie?QwM%7w?0!Tk4*Lg)&4*;Z)?UG355=QqAkaJ)}z;a zBQ$|hrBDaK(NqW7vV&rwijUgkkv+U$o)boWpEix+GRTho7E6ED)Shpn9A)7ifucT& zP{deRa$1YGuKzA!qR_Os1|KxfaXX>_EJH&=bU< z%(JG5VE2}Zi{7kDf|;SCZ{jFvKq{VtnBZ+v)^MB7)3+JWT*l3u{gI-rn^lXvb4kSG zx*ofQr{1To{R)qcwiY5x6ksAYAgi|o1%)rCwGYNgyWUM5A1+w$(_ojQ)yB5vS|7~u zO732dfOknl+zw>nOp0rpMLDr+m2K@Tv(?yCp{IhVgmK%60rPQ96l($bKB?D03kJ@pc zlzNdbK9Ai|fI+!V0ZqbVTLh8n-bszn3^U!yltT?oyHaMTKb4aj`{8JWo_p5o+^Mk} zpM|RCOV*e8@t%F~M%=BT3)6I$I)%Sea~Zg9{S_TmriEr&nv68B5}S&TgJ3@&CwN6o=l>DddqM1EWT7?5?%~;t`q)fvepx%APP{jr-!YycECSEu$PwDO8 zk3b=xEkFeLo~)qAJ2u6x+cWKRyGVEzq3+eOX>c+&n9*(5QK@yafh9Y%VxzQGA#2p> zd6{A~d@nT4(AS(=Nz!BisFDAPhax40aqM8U+>U&<+{As^??>fxmVk}ZW?Jh83;1;B ziv2JS{0uh01fs0Kme{(}>$uST+~;F6!K+E>X-4h74A;SC@n5*Xh*;qjFN8Q5%vIIA zYFZ^KV*q_`@%Cdz`0jl`LcQEZ?>bo2aY#HdIcywWta`|e4K|2AbgD8sr7qhoSDn?z zC+>|Kj^jQSb6O2JT|Od?X#eVYoU8}Rv+U767~vW!LmDl_xP6#>Dz(<>cPAhD9k;4Z zfBIp285t`gHysO^X=y=O{7EBjOFdqU^~IcrF9ylWXtyTPqH@Uf++p82pW|ICPcQsa z+xZBNP|cGW>!s0To9#Zcz@nLN>{-LP!Hwxq_;$sV6o;GKdgzA6qMv9UerV7oYWe4+HDTzA)Y~B3reU>7J>NT zScgs8@!TZ#T-1U_sKlw&p5j=5p1)-`aNjte1;05wn_-;oPln1ZE@A#3ZpK5owM{a(Qh`iUs z%xt=NtpWi*YCUZ=rDU#s{m^81i#BFVdw(+?lZc946O4!l41%kCF=7uP`WTblPj(?K zcO5w%nX7qd|%cIGG(e9Hsd3_d4wv+@c)?QBNRXC9%}*w{@`Ge-V^YJaQ#b|{Roy$sgC?R$xoPZPG!v{F@n+n}t4-7ceP5{9hydEn zDxNsDTO?O>`cJcQkHmK=&OdYv^B&(6B1XsLqAdzdt;@ zoR*dG?(%WD{jqb|OlnPNPvXOJV|?T)5`ZUW&Kg~G!52P(5n&$ceJ4i|Lq=E)Pxihx zZEC5?Dw-*r+Q;hJNF)Igk;37%GJQj0kPsW;xLXAQ?T4D{NP$~A+jeeZT~0v{wm?}& z0+Rx)Iy9d(i-?Aq<$xn!jZGc8A;NCxj9Y7C(Lv?%iap4$H3v%tmq-2m7 zR!GjpCkm0nuHAe$6R~A?WKRzsv(@bu_ z%~G$>s6L$Qv%8TZkX&A_qTxJ;Piau1)+u<5m;bPR2pE05pgL(qzJ;g8CCVjz)qmmYk52+>zMkr6D&jr@M!RM)wPlHu2-(GP!snAn=9&S?kFxjIZyi)> zTzhY~d~182&5|@s=;)**se(W%oXSrb1@tCYXLLMS=ixAnrLBigS{vPj4R5S@Z7*w1 zK{l4*?weJCg+w|@71O8D@8`yjz;S}Rx0VvbxCacc`=}!(T~5V-a`REq$)14-Up z-Uu7Jw+KC6!lpBgf5fB9_Ap+e;n;RmDupmO@#4S z5N3)2hF-iIm)bt>KLr^CPmSG){rOXs`yik*`+bVZZ z_j$?|xbmn+Li}ZW%W4F_c?ASkJ4CrSeyhi;X03ABv$T0PVRI@p7fvi2?0M<1Y`e&< z>CO8BS(KOUqB65Z)p0Dc@5}rt6sW%nMDPAYx_v7EpTqQeTzBDH2=t!1h!_axUF~Ij z=*dT(nl+=G^@(EsEQ@!Sx#Q|?Q0^|U;*%c!SYKjBUwt@aUDb{1i`fITRA#$+zDVWwi>>wdh?kha+*UB?$u{C)m3w$8|sVcU{-|V zVHD{sp~Lt$X6DOF(O?p~)w1T>PAf4?=37SO1w$GJYx%}kk*|3*{q2p*Do^J5;vzH4 z=S)^NNMNpW^M@)hR`MU0>Zpvc>~1`scy4vjUakD@)OS+NKc4~$q0bKa4Y zSHAUa?)ey2sgfXM?bsk{NJqAXHb@}(I`sOwvdhjYjT`EReZxcb?SgjO`=27_ zVo|3w8Hzb(lW*6CFk-dDu-&z1*5|!v$brrrXO~&#hBg5R9aFxRFQ;SFF^R($+fw;(rC@oY~ZSbrt|C0XAfZ#IfmMKofu?-Axu|t;nDux&z|5zIyT=LCY>ucM~qExq4X2N?~ zdF5)x6XgQ?EOS8#=at3*ef>bU>TI<#l`A_%lTRfTjkH8wy)iRI?Gz6^i!0vtd&Nnc zL6!F^^s$*cCr=t_3w<{A^7686<_uLfSv}kWmEHR|Tzf(yO#|kzmjAhj%j*;XA6GGp z3810uUkbZeHtuH7BOJ^PHLJG8D!%K-)%*vYjKo^C0RW8WBTnyPwg;A-k9v%!|5s)ab5l2-7*dTBMF=SGx$Wh$6YyNuVq5TgO<#F}^{;)4;@#hnj0%-vLiQ-+Wr`eJN z*pUceN7kiB1eee;?Ik~QEj$?;{ra}EnR8;kc(>uK34~(vawA(GUty8j!wqRJ{{=|5#acay!+$XJQM1Zlel=H|93y6hJ)k1@Ex?6rf3p&lmvkmC1j|jHE5>=!M z!0f2e;o*;AY21r>+-0~`q5M1Vi;g=ze^wtC-o+aZM$TQo?)>oq8MLtY$E4rd6D%;W ztg&1h*qJl@D+U*8iid*crz1i))x(!?fK0T1aG;zf3V@S0WR2kx%+UOTg|U-QIMC6_ z`z*TvxAZKsbFX>b6%vxoWcDID%3RbZ0E%$`hx*zKU)$C*c0>yX(`f`B2sOUNz}h&v zUBz4vZ`zEx<3emz`jZ;L3K(`O_*&ay!%h>$orDF;FCRqO16e+XO6!a3Uj$uE^YElm zlcG4NzW)ACK*9TK*Jih8Rc$LG-v9KF3tz=Dh5L_=lBK0*TP;|rvfs1$vBAetRbnH$ z5yZnErXa5D_)Jdi3rPBotE^Sa&tXAk=;Y|h`RQ^S2P5MkMO-bybh`1f_?T~Ni!|>H zsUKun5CUww`?T|wTTm3MxzQ#T-ea_*kk+)`okxM6RPM@F!l(&Ijhk$3|B2N=-ecAo)e^0FRiV zkxI}zZjmBEN7Dt1G=KSG0-Dl=2zpAYZY`ijHYWAw31=Fgy#c|<%G9YG_aLe$L&tMwG0OhLj~~ewHP*E z6)t2BVTQ7=IP-9ez1Ca8pPL3ky!VJ{7#JKV!HU1t)jVvyIiKvMRPc9hz=!AQ%!u1A`Pj2t_l)-j4|o-B8;H9|5LoD z`V;4AZwF5!%@Rzoe(wjo4z>mVNhhsk6Xw^sTKsFa-Nj~-mEl` z;qs=$yuHRn=s%$He+7#07;a&uVPTDndu8C0jg3i@s+@{m<_)#3YLy!yCj-@cCrK84 z@TNbmy>2)LTi14RG`4vfA~$ZtDPd1#!vs|nDxjAE*gVC%rsUKS4%3V3@+l|l>bmA> zk;A=L+3%GoZ&LnVs0M~=tn1jJXdk?qK75n?c2!PP;dSBJ0A1)iF9Myn_ z_4}HO=G*a#;lek`?iO{I26n4K1>oP0lLDyOCilF$?&wrs%^o~%Y3PSdJ=>Ixd1M;n z%I{_hX7s^ucqCjf^2AMb4Y}5IR&*ZjHpw7SIR`D6n}7Ze8yRAUP7Dv|K)!_=jZwKEKVz5 z)$rQCFMEUr$V8+(zEPgol1DOEdg_;Lb`R%&*K|e*!z^JpH}6dKF*>`QEOqZ88W0@4 z!XzSsLrx`>77Gj4{F`r0J9J7E$b)pyawMn~thHu!?LKAvk%1EA??|No`vM^zwd{DA z8U-5b?jI$&&Bx~yUZ4Z%lypQSqKF?$Vc~UR)_UEw{7g)eh~sE1&tjlOgCX9)4f8%P ze;>FUJrRjS?r&Y;&$~xxa%~To%a&3YUKB9!z4cEN{+KMBD)WHTe1kvgUfR=g2ieCO zRC{|WWZ(BD0F`*ytzp=rj+xpzOtJnmpg>GrMf>|(gY_^1BeH{4&B6H4(v1*3_4er` zT)Hr*m_OazY?xHxW2n;V1}n}!VA4;g>R&C~^WgCp$SEmnus9*8#}*wss?NcZ-l$ zVOgtMI*oS!0tNHmnZb%!IoB|4+}T8KFncb=bo@{-yP3~aS+4Vb70h|}%NrK-=84My zTbp^1MM#%Jpi16|Qi|Kn{|wB^1Nvhi)FYZtJoxOdk-}czd=7HKLpuWgy4l5Pui7?H zMniicm0it72>R%^PcoiM=66grGSiFj@BFI zj2uD6h#U5N{{Q%XCU6aD4@^+3>U;Lt8K!tMJ_Fa0*8{y!a86luO6a|SQ6Kzs)YY?Z z4b4&H0taQ2`@iw#JvYpPAkc>~9r3?oDSmPo=s9rYBJEwj5F#Pac?%Ej`>tLHZs9lW z47kvl-ad+d^rQO@Xio@>spf+g;DK?JnawId#cyPRSwyhL1muH9e-|*?XT82x)`QmS zh+&PzD`Ts*0HF4o8^!Yk6msnL1^kZ*IRFzXt+_PgbOea=)xBbVV=qK;f5(;?HW+5g zoD<&G#>~XR@~+;?iPG^++Qz7~&@@;-`?AffEO2Xc#`>M6N2o4QP48mCi zg2QgLD`i_<2kTKv(MLTgh9_1M{hjCtfB_M<%u+2h{?}R;tG1F0@6k*`aP3q+=Zbja zR79QD2P()Zm6*`uFGLTB5D1n&((g)p=dv+I*B)VF1S4lBfvpzEdPX~#W1D5P#BGe; zqIP|nNT@pBQd)&aC)@BI0IQL0fiig(g$CK zU}TXnu`#{pHOp8G8QCKNDXXsC=d-_MG#QQ`rMo7@V6zi>1<7W1%KiGCS;MY+LbD!0 z@x78~G3w++PsRNBUli%bJc)ZSbnH4F8PfU5WI<-#v7>iX-%b1B;QQqccu~>L zA)&*hVzaTEsrk;R1b6uRqJr$~$)Z$mDzVrB&HTS#4j^sMQf%-p#f*W>t&j$S0T*)d z2l-lJiIe_D{WM6MBr$&q(kEYH)fr31%O~;k3R%=gCPjP%OkT)^2H#2Co^bn^)@PiY zknrsAN9VghcGf-=X7FT^98baPK}ogZ%aLG7ts<;lLzu=9gZD3r@&Xk}{cp?lc9MQ5?gyJ2;g zP;He5fhFmPTw$G)dU|a!ZhUIqr+!zL_i%e67-|yHMH9!~PJAgL-`h=<-Jn`eIh;vm z`9b}-Vmj`d`3xa7ukhs`D{Ivy++!O;(c5_@K?9eEP0WNYVgE3y&f8e)A1yaMV2;}R z4o&L;)z3f6byIP?qu+1IrpdaaQ9htlAtGegB$3~Wbw!f&gY9yEkCXCkPW^tIG{8{X zOYO84gT9$^>6+ik5+GT(9UKax5QJUwqIgl{ob@D*YY0n_WPaGhd2gA$!SAXiO>O$X zO8l_ zt(D_%O|J&&Jr_^rQ{2w?(CViW+cCm*DgQWW*5w7oV>eZ1&q&IJuZ$#SxTtHmslScHD!ep*G#kul!M58H%w^*^d>oO}qUic09YYC6{ux*;v(8I5uR` zta!EhO08-NlJ6?+{ROF-N~psv$z#^{m1ft& z;O%;%q+oBHs*72T2FR+`2cwDRd zE#!veo_UPk%TvL1{R(_Gz3E0lURke+@e=ZbNo}4PQdX4|p2eXR>7(rRlv9-~XygNv z*dHr>&vz+#+FXpbl`;TRJH|+5S7{pKExQW6TsF_H(4fa>49~hthixaQwqu>M=}mb^@_(?e2t}@-uq(YxTFpXD@%7*wbGY+S=iK zG&USxs9ciQT2uk45oRz%vQ_3D2!9!P>)!IQI(u0+is1V_sY|bMc_kjz2nwj(?{O7& z_5M?3<3FE-5tuVnuphl>(<`Lmk~Q$Mt>uh?t$pI_`TEk{8MkuDQLVaf@o-J6agbs& zTkYkgkePV@_PHG%TU~$ow-D5tDkL>pEq#yj!|Yoj3)^X5QkZG~l|X4ES#n~hkd^{2 z30YwqY`t$o-er3D0}5s(xRknWaH=@g`6=op3&si9lM5s*;2yOHjp2N*)S8FGf^8{hYw z=bT61=lAdT-@x2+?^v_0#c*hsVRPPox(I*?Fvj79B?sNNj( z!l&bk=z5cfGn(_T#%-y^D6=Zdw_H!?l4uDvD2&cMO(DPyr6qp0>-gU~r8t^jgOsm0+18!jh@ zb?F&h;gw^H;M~>qa3IpIx~2=ABCS{-f)QGWVem2fog{8Z?_q*Oe8<*D3N>5TwXEb$x+Znd85~AobW!Z6?J^WI3$`qOl%u0nG zijRf|k?LEN0vrOxbyMn{9eu4xiCa$f$jR|VPf;$w&DC*@DP%(!=EJ_coR3?CP*s-# zj9UViapbaIw_E#?PAC)vnEV2F+2{#8;pa7qF=-Hg1k z^;){cuXnrZGQxK(8^q3TMhhJKr7mI;GK6Piv@jdL2^JUR7k=ShxQ{8YqJaV!&0b=< zR^3^;vZ*+sP2trMGrehj?DNxW^IJwIk_?w2JbVZHif1t6N0%z;9&?KWsKpt?zhJ*m zYMRh^*!57*JQh#txQ*R$vk`-Iw8T-Jsx}WuuKg`8J(=AYym-|X-OENr#%;8je|N`3 z3PNkTTOS43i+CgM8HpNQDbAJ2U8 zCmKbrKG}B&5bq}o7VRjR_2j^K3MZ3^HUKOZT!v9{Tpy0wPo)T5@~!Nf=)Y1h{|-*E zA|z)6v9?36P~3^j;#wSHCtb~2%$jyu?e*?>`V+%HnLVpB;tQ1-#^~DepW0jJSXH|G zU?VQXSw3R<*!TW+o!R1#62GRQ-7#DSZM)(9_ZZXyOqx2>!dp8P$MbO^5&dCsbA zfoDlvi-$Vs`mQWKR&AB1tK3j*tJ3qAq(Yp0l^ zl$L7Io9Hqa9Qg^oNtzTasF|OkYZEVZ$jwYP0d&sQV0ac9U>Dq^#e)` z<)a^6hGpa*x|Q3aZDYnBZqjU4RXC|k%2&-69LOB$+WvN0pwMhdV=hJ~%1q>C37WiY zw%moI12jF7@`_Nkc904h{9)=UH??hFoBJ;M#V{!B;(P{zu!GwfNor=3SXBHL{`?P_ z0Lxhm>#8dLXX{fXPh?JB{c98Lm=MSUwkDbMvahU?3R2Y-wVJ;psNNQI@Q$hFZGRMKkiN2`s1HxoToBf< z`R0twa4O0L(*Gkw)8$Cnn$A?P*8x8a<<@2iI$T(@FI`NtKV^U{n>zZAF3gO(^gaid z?!P-9h?!lPQlRm8pSa01phV5w7a~+H0B0KAB*m;YDG)A2x^b~594sTpA*a}aOk>;U z6;a+5{sw4&c$>QgS&X0ZT>2cf`>i_-!FHdFOOF|`F|2X-9O)q?8;GA`P69Nj>U=dr z1G)H-&uI_NzY#~}7l@E6x?$_m~n`YYtG>I(9Ml;iRQEpH% zDvO!pnKinlIEDUIf2d|14i@N2r)BgNlG=n0w=(|62k(I+?@MzRBQBUt$`$Kca6q5Hn$MPvbq2p(r6G zkij?^!;Y(dkYNt@f4$C7N+07<#n8t5wmSb7E_S|KPi{v2F2G2*L8*9`xzPbSVffwJ>>#78cdN80BgOuM9j*=;moZxt2pQWf46F#o#dQ;d zTmq+l-sS=;Znd=aKdZx!h-8)Ke|ESHiRQWQhF;#aUEQG#=3*jrZ&@}5vW6A@>w=K`;jkdpNSw&eSA5> zk_HRQb1%0oaYa0j&6;ELIckQE;9O&fJ>&JqY{CgCzXuPwvw8zX09809~u z`S(KbuaeBdJI(|Q91l0fm2R?B>V_~E@pQ}+G-DQ6OolF86m(Mdr&`!o0`sa4m)V6I zsBUhqOp-*ey@JJur?X`&)_Y(r#>|W%>ziT;>KH@``2IHS7}fbhr8KouQvTT8H({q3q*H*(nV{4*u)bgr-UOA~+0%earDT zFg*o$PuzB~$@*IBu$2Tv=3!0Qr>jsp221LB{7@Cb5&WL%`Utds2$%!}V|y$&+riY} zE2TN73o!a9j+eUm3DgEZ-_i%-;f~8VIw8IZo``IUVhF4Zi#8nqN-DxWIh38T?8rB( zN;({I=zfztKO9}Dsq)T{MUG~q;#7{dkrm*VD&tbUwywOMHLy3vD8P6mkllhS$H7$g z%&ZTSZ%);tp$s6ytGC*EtLT#=?Z;ktNbudUWBD89CMicA6-+!SVfUTfj2{?6$XuoQ z{_1mE5?ScAb+dLf$M|ybSYNQHAT}yo%Wk%jOyKz&0I!zwTfuWqrqbc|HY%#j{PKKd z)QD8*JIbYj!lfT;X`gaP0$_IcLmeFVTbp^^ocw$H2PY)oCE>c+D>947^HK5Ge|<3G z@WfrhV<5qOgNcfv|NQOJHONkKL@UjA6OY_n_l4P&W3;P;ajyUC3myV^_dfmI)TomT z)o}+`e_c71^TTFj%GGTCi%*5ZMj^ik4sA@o0?a~=j07#uL~}rN-6P=rAY$01+;-a$ zqH!Ll!2UefI(D)BaJ2)Pv>}!h#4!S-E1nQy*by48#$gcN>RsUeW?pYaWuRQ8$u`?m*7N7L?)ZfJ-_X-R*?nUEsWAWc}vPH0uQZ8eRiT#PrC4X zb$DzZI%oo$wjZt(@(I_5dJCfEyjy)i4*8stt^`W8!k-gasz>L8&=fiasBMs9v zZ}v`3?akN%(-19*$_EVg5TtO!YJ5E^eg>+aN(f)`C=Dy*Je``LGpp5>{Y5b$Iw-{- zd|tOY{1UePm`GpF^rHE)e278@x`P&d`f2e`~aK|;N*t>O#^Vy5bQU;-|o~5;CW~mrB0z2`8%WvC7uFi_1$~D=k z?GLxm`-ZKJM|i~_&`AMNJa$S-NXJ@~*H)f#ig?!K$#>PazJppu?m-xF3bX~WI>tpF z^x&(*$T5-H-Xe0&Vpkc_qEV#+2MW;_>S_{2nhGBjd$Ml0l0E11Mog;{Ts8?rR8&%IVsem&J{0#u{`+qwr0WC4 z1JERjY8|S|p#zyzf_64y(-RnI5N`|WAx8ND?6GQT^&c$j-bD?7#bH^t%0l2n}R*r8Z%(qrrqie}TPJItD6GRcdq?U1sXnX}sUzAFO-K z=m#H(iQxY#T$Q}!nB$j{{pev2dQ2uI3u3Jx_#LGn#QKF!D*7U+6VVxYjl<~%6Z9Z< z;Q|u8XgGw$67$bNl%gMOrEcB3`>St3Vg;VcgmDvd@M=s2!jbA``#&#G|L}{jhCWJF z>H>XgYGn(s8WBgw00-6L#l+BM;yF)0|8o><&#ZyVnJ$h{X!)2}XutKt*Uznf|EjpJYQldSm%vz8*NdE?zWA&9`ac`*8O^WF z&Bn%Fi}&>!y~-XtcMy%1m`LbVn%|d_h2&Qplwv0)f_3FO#Fx><@{@wE00dI9i6HKu z)drlh06?)d&3~00{{<(?>VK`S);9J%(b6k@8{o6`COv+>3A8O1^Vl>Rtmw%e1(p12 z(h~FqXc_ikBR%=YOyP&tYBa&Cdbds8;Aw0QDG@S;_b+bKU$3Pj)C$EiH6j64AFaU15YZQsZcnBJ7^uc+f@VG9?qciQsN#W%=UtLH9{V zzE;7?JP47b?tebUYu16)1dEHkAMp|0GO_GExWP^n=l$?-7q=JXL62SCG9J*TaR&zn z5jt#A!yM4KU)4JNXYDQ{TgC(tr9!sq+d)hnDvb)IKjv~uH2_Vu(yMd%rmNMlzsQ(f zRwgxBsK;1gIY?X(($lLHbzN#cjZ=5A7i2%xDi%G*WOWjYy|T)$YSkmH)*zez&qv)BmC^8ask48wN{W3`ZFp}65Nq5yE4n2lU z6J5d{%G*s2EW!6xDaD!GUvFh=-4qCE37-$!3 zzabZW`wuby`$qw>zZeyj@2vk#82f#5jBan54(*}wntFKtkMI7~Kf>4OmSQ~~3`+mi z`0o=yM}>x_%QMJ@F#dV=q|w|bqS5FKy?-9Ic_7;J3LGe+c;(T*C*sdXvGUL@DT;(x{>2UYqgUuNbI>%L;fTj~|M1>__3HlzrO9}h zTjUTQ-pAal8t%3ci)BG&^G}4C;mKb-2+jed0cR<|C~JKDxzd|47M)%wG;VbC@l)2G zvlVWc^Mw1jTdNB(I!fzm*g{7G@v+tM|2QiS=v5lM1nRY%gkU%?2Hd~reu9Ry=a`Tc z4xDv$44s|WWhHR^Yd3|um+8jnVztMER4zLbkmUKM1eck-PTsez9@oeQD+$njb zKpNlmA#9=5M6=CFt!}oNE;GShPH-s|O01scNeYtJz_Tsr&Sv(+{ivv=w#ddtO^sJ3 z+^=Wr>B^;+rPnX+@4LHJy{_%5z$f5dg5|js&6W&#Tk4%)RQs zH4+zfT8>^9=x_b(MXJ%Y_50PkyJM3|1XlK-6Ur?wxT<3^EM%PKl=~l42F*kI0NzKv zve)0ZPh@g#7Piyk-`sGP?zc}R@Vrljd!i>_3y;@UGp+f%q~oMuZz%0#uU!djgHTaP z=xoJIuf9;q7*r%bS?u66K}lvjWfzJ!=q@ATFgwKxZHdKsS=eTlfuB+7;gFTMC5nHs zlfZp-@nm(Pmw#_PQ!4mAqXMH>}k&hpJ`X9sAnUY8i%3oFT^nE*Dm+b zNFE1$7P2paC+kg#lHelfAKU=l4q78k5gCuFptuZbLf>SOV~`{5;GdkvNda?TZ`LK) zv$L7#bJFw~g6}VdNR+b$x(;zANd>yDT1gln0H-Voc_&5nt<4haJH!+s0_v028~Zga zT0OIMCxa8X)wH;`3mMwhGSOR2q|IW$mCx4@{Dm_QO!OA<(IKLn{o8An%xYMr&m4N^ z9-kp|WazGNKKxQkOZ9MkOzUu`VxqXOjg8)3+K%c&ADbNhr&#VI@#17m+FU0G4mRbg zg|-MWg|%LroTS=tLTC{B2!g+B>c9S!e28|2nZqhPHd*Ad8}F4~YbH_(^y1G+2SLAR zFp;p3DOL?h9c1F)KUsOB#TCi=wAXp!fdaJ_Yd5XVigX}4VHg*!Z}ZGvx0v*KPgaap z9Znd%jLT}|m`b*`3@$E%_<63e{1)Cd)UUb+FYma6B%W1`M=OjOZ$QWZlttqCh8wJse*fWPrUx-@WEW~S<&_I^&&1vmEjt4{itozW$M*3Z~V9e?3$Y52ei$&H;V2QT(iSpLAGRL};RL9{ep3BmD_^Kox zap9rq`LS@_6#*o>VMzU;3E>gTOHJI%M#fQDs{Eip|Ap7J>M&!Y)QpN zZ@T&3go4j67T66TL&62J0-J;y#)JpEj_!$}G{tW1yit$hSQt?~c4_HLsh+m8POq)L zykc0LS+h#Z`{%!pFmRbjse!@ZpW0JsA?M=m=)6zP>y= zaY+ZjUJi|GWR$z85FQjcdL&}W9V6vVo^+%aj7aW`eWv z4j1iul(Ms*aTtmlGY9;#e<&whZkuQw9m29 zrnfsk^WzPY5-nT-6oBWOwndNfhzr0kt}4xRgiI4mN{ovVwCzshLh;GgvXM5oWmQ%4 zRiR=!MG}tj+&&o{s=B+BUH^4_)Ze0!fPbC!wlw?_UGE>3c5$Mr%qDt$gR@jZH$v;RaakNCK~F}s)o_5) zJ{NPgwDC$A>13Q72MZMysUlms%z*9rloQjZ&*ZTrZ`~n$f9oFIt=oV5NEeJnC!0NN z4v5U)Fg9FjJ`-5jt+uO%*mYf(H20wv(@C|nDd*09_8!0yl zrBje4{A}YJj1`;`pYuVQeKV#_5|1K&)nkDZ&d)~>y5PsjO3F}NUtWhQ{`9P@r|I1p z=NgpzaIx^Ye6Y=o|4kP_^5!LiKVx5&UG-~hO^_&~o8R!)kY=MN>)-!jjOdZ)s@x-F z*c!EeVrsHV`gLV|@N?3tS?nuu?YPBeRmgL0!VX~xE44Cm9<$X?8h%)-+N`V^J(>LC zi|+j&ZHlMjh1s8;NFP1!DL0&?7O~XzSLWb&9oApvLqv>838w|pvqmQDm1H_867%TS zsCfoBO9*;KK}2e%YR0AyC317QBgjY_Ec!JC2Nyi2N+-niPPErXN|*8Rnvz2{iMYDY zbHw=_KCj8$bB<#P*@Nf3k<}xj3bv2O$`tP%%P0%>dJWId5$7<_UOMC`>9oB_uzB|y zi}lH=w!{-Pjo(H3A3={lxr0UIZd1KsmxRT-Al=w8rDe;Sf2d<37X-e(DxdvL0mn!L&bccK=ba0y?V384LwwrdnZvJD5X?hb? zlDFn~XW6jTKmWK&^%i|*HTIZaACbmXPB3nSkt=TUc$c)uQ~QVa|4-fg-?q(2B*Eap z;H>{kd;inB@msfpz?AP5N&erDbuhmtB!b{7(Enxz|9R0b9m+*t*+ayCD~|b}r}o!l zVDyL>1jwVM|NiUWtW4uBTJ>yiF{u8tcK-SWA>@{{tUcG`;6H8jzh3_p18uJ3w`4#6 z*2r(ep%XyQ#+2a;oWC3Uf4%6}M@c+FNpS9M@bgz`t)KbN_iotyKRBZQ-A8Oa{8cI{ zItry2wUk|!$T-|OvCo+767-1hcmKgej&5k>bBypDBtD+F#@juEot48)POcN6+7Yjt z(=EGBe=8syTk*Z3y1Hu$1qGLgNWXZFj*rw6tCM+F4vun9r2zIy6nLKfA(rIVNJ;(A zX?1SNNhzC@0t0R~HZ~S4XZQXXgW_wnc$9>B-17{09~X16Ist+Axft6c8TC@Gt@Q6k zGP>K)Y_i|Wmyv|}zj7{PXJvhUJ7f4kmR|8I4;grokvlhvUK&k&!(bxgBVB>)vTapXrL54}m zh;WB>mn`{dVRBxjakRWirGC`4A6V3!JhF7_suLB#&homVBHRu9uK({Y-9T= z*y`T)o!{*)J^`(TE<-A*8%YnM`*l|8yQ*>DV8}*>x|_{xFx|6Fl`O-?^%h)t?y%I9 z%~qe`;l4f{4QNvT{2b5$%4X305-9oLPJmUAXt;|DWda`q1|z=IZ(7&d#qyr8>|p)t z5Ry*9q}Lp`=^)lT#odE)^Tj>eFRhezuZ)RsNEkE98RA|y)UD!;3=09i2n~++y9cagZ!3;Kl(7h zG>wfU;pa^CX!9B;42X~C2TarU8%3yo)?$@qt!~9Hsb@{| zrJi8}5rZ!S@dUZ4Jc6_uAo0y5x@p_`Q+x^me>fT@Dzq`1L*j(T(`AFf69qaKsUy4| z)i!gssYJzwDcfXR`a8nz_Ksl=e3L@Z0Ny*ikJZz7=bCO7yZ65Esne5rQGCw6s_bhc znY-C10o?ea+?uL)^lT2-yzV|N-_@Ts2I^`mLq-Vwr=B6S(2L2tTjXhs?K9R`1oB(Zh zVJ{>=ACM$lMK$U@GL_7A(Qbpcw(^|+l{E>h5or4;kH49_`Mir$uY6#)0tNsiPk)c05t@7T1wG9t_jJ>_)iuGo)?JY9bnkfw2l+-B{F$1^ycGBRuPrJ;$Y7mW z3CG$l4>>_D&ou5;?4IG{T@>Rir|0atG4PsaS&n1>Fen}Yci{^$ahtB%8&8OK7Xe7r z&X~cK`63riN-*akG@DCCel;yBRr=#5qL!IZ7Of(UdqKE`WX&n z9!dqyU>ASGzE!1l46@y8Jli{m>^V8iRJ!=r@RJN88W8?&v_8jq4L@hcnl?__!Re{d zoF-+XqF`6|~UH&Xgc?b1=!-(@mRh?WWE9X(ozk|r3<9j)q#2HPGXYzu7uf?2H|-zGY(HB6FwBETnVqs&wey-H#4LCj>@GUV zQta7Ze|zS@Nkuk+g}_f_Sc5PvDz5V!R|Oth(v zs81Et$OjBd%+d4i^|fCszNp>l+n(E7YjWD*3B9=f=xF8OKY92DJ|cJGE-@H*FWvwk zNT2ye7Oc_o9TF;eWI1S9VJWn_#x~krPN+7r+jY@g77Vr-v#J+SZJf@8uk9`24sjR} z915RasMa(c)qh3I8xDo535pFR?yo^3(Vd zLAAFYmxY#5wTEx@ri1I}Wit5CW%|qGq*mwUg<>~|EKokz;pE9AA7vrm)dO(_lt^uh zpY}NP=lR#7pMKt5%F1T07dsIpO{l=gOJe5_p#B6kR6Mz~y!>}+0#xRXs}uAmxVhVV z0dqaezb|Vg@WywG#_v*(M7n$y4`#H8u75Bno&nMN>==WN#5}J|xKW5fOd8;*D4l+t zpD*ZQ-DayYWwMK<7`|pSM|$HK09+hzM5XI;bTekoHtzqi%EX7F7vIAHxzueX2OJ_C zS02EnEMMf%VTcq(di7q@H#->MP0L-6!^FJ#9S^m`PQ4SzPnMz)Uvb;>N z)^xd*C*M{Tx0QKe`EopG3%tf3E#@#s-N; z=bvtExYj^QCNGcC)okfA?)%V19_Dkk>qPrm4x{$zo1D52AEdWo-vI@jpwsk#+m2Pi zv?HJRi<8)OyiKhk^GzI${k)F3b1wayh99f)=9YE^9G0s$<_EhT_G-mBij=Ut{Ir8J z7%NihZ;~Sv_v#pRkkH7Z<4W&G2|8LEjC$8$%X)WH7hWfBH~Cn{zUvw}$-Vp7*u0zc z&Nr2u==sw;LZOyxWJ7|pUf0rB&fOp~$6Y2J5k4#EQ@nz)cM*d|Fs{`8;8mglQQQpP zu{}ZOLrnfnA9nQ>1$!pi}3}->CnIXu9se=){$q?uPVAsZsIrRj)P5Mj0oi#4tsZLFkZ$laEb=1ub+zx{dmrT1j*L$meT1gJ9{;@OL+ zwoLch*n3UnHZ%*|p@?(!1Gea*&z}8bs6asULNWBM~j(OuCu4u z0C&FtD&E=_3I6`c<0)s7m$JG`qZ}+T(zBbsOm}^BW5;H^5$#~9%S4hGXOZ1!9Sa6g z*0YECXLIufSLXmD%4p@TOCIFjc_1fFW;!mQ$&k%9fj|5M?5y~65 zEzH&wc}C3h)A}9;Qysxyc>eSQop~;ySJLx!lkVqhdl1hm;1_e0&C-HzeRBA`5OI9- z^%|z<&)Xn~SF+`9qT9~}gUpkx2Y>)YqzH*1@ZnLH;O{$XMXp~_En_9D=Lz@dvS(*c(`&{P>c#`g&BK~M%F>5+=r;vp$(^OEh<{l@>ABR zzHLM;Y{46fxTRx6L9V0w2ro&3y0Hbc;89|kZB_dc*OY_Fj4iZYQ` zs(N}iHYk!l5TlkPT4a>!D--Xm&$ie+T_JT9uiLbr8Mnq`;FJ{E^_nDV6}C`!LE|+$ z=e~V1f9uhsXobt?qiuR(>(LiSL;zT74a#@aWJFrl((+@%?Hzy1+&8&|yIUrfzDc>x zyGhC3GJd9}alsh%UBy-RV3U{>Q1e6b zRk$x}mFgF71RKUxC}~7ilk2$AQqb_r52i<^D9{4IaQ}3%RI@98Wal$fmjMEjJ3Iw@ zb5k?z&EA5~gmH;jhR(5lR-nP3?_wu}x&GL0J%(585)o zKhSq*mf~g0fj%&WW*lk*V4746I^A4EoQJJJe5WtZ27uHU2NN| zrlb=tF+O?2#`op=B7h$ps1*~(ZLq22i3^bh#g;x80 zuPtVlcDHXyfxDuNW&sU58u$?DGtt9SKIG#Pr<>YqHve?08(=c|V^V!3XlNQwFf`ut z<%^c!jz$rOLgEarru`k4CCfTU;T6NHpAy{7TV0>tQdV-dH&1@;VvbvV*ROMsyNVyz znqO6DrMayTa)LV~J}5zMs|H0bt4kozjFPRWoqWb0hz&*p_lBL?R1& z&Y^2=3XMK6OYiaz1cFvRX=G;3KH$A(U+>Bym|bnAz0Lw5&vdRDUL#bZw3a@%eOJPI zaO1a+H)TSs8|r)21e6h7-fLV%AVoRcb&NbKKm#)gpB{-KOzEW^bjNzfVwD2>VDR@g&p-SPn%qFt%S+;3N{jY4m5|CZiq9wkMyuMMuQ}9}CioYV zO_%8kSH&we7PtKNEFz$OkrYLtURfhv@G6ewFO8JZ6{XB2O=rc9Yuw3S*8Od*E>hO7 z(+L8c_fm2`NqMR}CMGQ+5}++E{`^^&->U;944Gs_FQ@qCpZOsvSmZo5KU;MyI5y59 z@F#tvYs|}Lt_o5c$&J0E8m1ovDz6y44YAuTpB!=xo{gi|3)vT3IBSjEVL2BjW+=Xf zOBmKa5!9;1rx-HdM>_%FwFT-2gxi~Ove(LRgM^}ggX?akS^a|V0^|Ho*7g>UZHy-$ z2@kTu$j}*>h9Ym|zS8TQ&)x07WW&|x%$-%quZ*QTWcdh<-R3C(f_%n(STAPHKB>HT zadw(DNLODmR=m(+=-UM3v$y*&QpWtf^0~~33;fmfM{*YYo3Vhs53Xtv?6N%-QXv8q z-cMwE-pDwguBMiy00k7m8eR$Tz4IE-QIZw9iV?l6w>KRiL1g9I_XOScZ$Y*>5W)LD zv($lsH*)Lgi^!RGIypRjpM1)T!i+rUe^}_>_@_S`355wAmIiWSwx-Y4ksB>~BRBRs zxpoylDkwKoP_7;j4~b>+vpA9gb*w`xJgiEdA!52!XU~ToB$bHX)CwvpgHLmUx#C}Z zojcgIS2(pNNdvT8_>mCRpMm-+B-6Om#g8({jjw^r{NF^j1BD`<$bNBIlQaHVyLA|G z8GSHuzhKT^HJ0LFS<&%U@&<{~5;hUTKtam)z*}eVs!a8<$A%=1LiX?R+kjT+2e+1Uq?-U_Fs}Hwpj& zgYJhoRq?xwm+WOIqB#vo z@8}`E3>7FUOZ~cA47Fru93+O^kT~1dAttsQ_+s&)WAgf{#;cbYH_E+W-F=Xh{KE6Y zd;zWhd2qv8xXP6zKli-04x0=)I4e z1~fdwklVUtn=ulrhO{t43||VkrnzQ3>VxQ$YnR?SKk^-a@`c3 z=Wd8<8Gi6Fe48sEyKML6WU&#&R+Wg}y2bdYnr$$_*%(-1HS$YRsIArpJ||j%BQ$8&yMfOFfYUjbh(%#@On2V4%*$iPwHQGMan7LV<)xL z)fOCKG{5EJxT7;?FzCd({JHTqrGC6VHqi{Ve@3USqk~AeR^rM@9822C!7@~ATN4#W zalXgyPGg@~XOmslnl3NgDz__%JvRxTlK{cdzW3*` z*jfa0o{-&-dmDJohqYcuy9AANnRRbQX4&t-T&pWC&$qcy=;gg*ro{}KC?^eACW6{| z^OAL{9TbwAe~790eq?4E-jTdPtPZxenP((^ZhomfQs0O?H{iG&64=XLjyc@k_9c1G zgzsP@9Tk3Fc&{}l%{>lv2wo-1$ZlPFT7r%3=n)sZ^LYoJ;1vhv;im|~3byi|SlewP z5O-Ej;W5b^+gvx?0@Q8lbH_im|CsgzglL?WP+2!}`_{11R`K%X<0Ua;CRxSU_*+jA zA+2^wrs2oNA{UV)O{a9<55x5V^D>`ZlZZZYoKu6UZ=BmF$og_C$iwp4WLck23Ry}@ zjL9km^N;bRCyb3pBBNBIeqLK5VW@WhLui!fz43(&hPa~-D#ac;CcrWRD))_7EOn`a z28)s;fRo1Do(L%nzhJHU{aL})~gaV z8tcUBNJR>qn04zHV|Lp!haQd?zrFT5h5B(KpP6dQrI8$d7;Ef-$S!uy0}+fjhU{NH ziu$sVFfheh+v8+B;r>>j+N%2wK^sijfR4j1FV<2~5{J;-M#_$7tP4JZv!Iu(j)9G9 zZF0T|wJvBqt7|d3Am^qedq{Tb!bMp+C7I7+lf9MlQMpcPJHsDo6FM#JH|U-e6yl0D}7urcQXx`xtW8{-qRkF!vGHV;a?{W?Qj zts=EbVW!vsDBDx&e~8`vsm#&Z#sGcm6<{GS#<}L^$WeU#y+?nVH}Z*}&N-Ke10u?z zPmeyU#pOwmN+n^O1w*@agxF0j#V)D5QkQ>HnFm4!e-@VNKa@vyCp(UuA!I6gW%yZ&F3K2w$ zo$rtCbN3}>40H5_{S;$}>@Fy2*>2{ONjop{I94#tN%P;5s!7^y72nx-(})?^0ogaa z;dO~rC-mts#q=h?+to*BE{K?)pRjcjJ<|avgx>guVoUK|@gM4I7$Q~eJu4oumIOBH zf5_afWH&T^P%kh4&iRdEL0;EYs44Av=~H3`V?O4ZIX9y`!Kk*1Mw_Li_#5RFtnfBk zJzIOY9w2kFn&=h#3vw0ki@~)EZCq%kdE1YA%>f^Z*PhkXC!u;%slQBlYH01`Qx-n^ z$$4ztX@1N6j;E6f*n8e%mtE;VwYtF=s;cq)2KK2t`coG%1Cw~;TUa#Zk%6otFMxPn zbEqa(^e{88kSUNb<4n7m^hr+hVL7gE^qpYZ1R}jOshK$c@dK+`{}jTEFFZHCyPTeL z6F2MiC8VF3WG~?ljxOLA8?9F;F6we=o~P!c8JN54O_mR5wr@iufb7=znu-2WZJjMI zA0?T#uC*1FL$}yw7|@ zqsWe5EJ|DBMdiM4QkIliVPpTVqoKJU+0j&zbt(z5()g@ArKMCwV2ec@aXLoJZgdaA zrJ-%MJ?WZ}9?E+|2J-a=;7&+F}jNx1}1Y=w(^Qd&*c+m+>mC^m7wi8Kp7UDc&c6wH7}K z>w%fW?q_vJx`y6t-lFZ>=(M6m3Q=>_>`UVu(Jz-i7{bXRo_C;q85IPkwfEfXB>(ct zb|1z+QY0Mb15S7<2fbN~OtgE=)v~Z7Q}T#f$O~+8;DjF|+lcX1pa zB>s2~4N7aWUq2qJ<}>Q`fHv*pq^B?4IU0}U(UdpXwV&j2p)h$O>>yztE68yRJ zwy+XRdSXIDc39a{JqO!*L)Vtu!V6#{cpsUIlb-1$B=u?lCFtNss;yuaI~o6=pyhMf zD-meBbWmH+)$+NWa{D`*!A3jjq%|WYNYbVD71rL|fJeRp73Di@^ov zy>y8ENfn!P&=Nu=7B}eeoI1}AX^!!g&)B^R)D0@7VJ}~$iC(WQv?H_b;}>1}#zb7Y zkA(Dbsu&lpP~lG0n{9avy)adO@k4cYdGK%}dt)>~x4GR*8b9#Fh!mA0K*qp6WyfmFfAWvgIKJxo9Hf`i|wRIE$N z(O6J0Gg)8aoRK}`8AgPnFI9odc7GT6&il&nsBC;5@)@$8=F{5ZKaU1eaVbvI2-?Yq zi0J9%@5D);x|i*%Rq3I5B?(*(GQR`t=fsj=U6T(dr{^=o42cPn7$y`s7t;W3iY<%d z3sj)G*SoP*W{#fy%l=%Ry?i@Ifn{AQSc|`jXY{IUEd3gziogXeZV3}3Y$DlYM<(%J z7i!JR5>Sr*a{!C{x)x2D{W`)SR^ zi`jK+v;v;K;rLM+nXi0@lkw;=e8TrL^_$33T&Tz#)gI~RJO=tDd6j*(7>w0Q4SSTG z1Ip0=K3!;Zilf3i;3C#D#Ti%?gh}zn|J))G8agu9#htOi7?HxwSS5`#be_Jy7fS2r z0=qd|_!C1g-#lmUV8Y48#X>a!k9GMszO3`YuF~+sp zgqj^(uh6K?8&elYUnrKL$`sEU6w&sbH3lqdlW3Qn6 z9Q|A4a|C3bwQ$&&SW$az%{wr+QJM|ChDknE2s*@j>^ZRL2dir99pK6G3kY`Jv|Cbk zoBkx-p~oy6SrPkY=1EHxHVbd5RfST8&*@^ z=xSP@NzYdUuV-CUzr4-~!deaXn(#PbAOA?4p~8U$O_IK?B&&G{T=jV*Y%ZCXBgG)y zL9<4k>BBdc19cNL^bA(FHZtE{6fj_aryAR^T}{<^nn7rc65r2Vb$T%c;ddUN^{8-POI*#POIO!o<+S zD2*TsUP#DEYJ)vZeyMHw9jZcK5-bUh?YU@)Y1ua`Zac0jD^oEiTK;vzEp`&s;^^?> zSlq~gA>I#D>1*ELhhH(O4b|_2n8{=`<|KA>C3A zCDPr^z<2YW&vnjuegA{+b@*v62KIjT+N++m?t49CRoQ;O?_X6$9r>OAB+HV!Ghg9T zJq_cb(6j}8{}C{hRImfhEzTLdq}Gu3CUlEuWiu-Qie3rjM~WYnmB$aDnrsS3E8 z&qo=(3kuq?C`7ny2M--Yd%DyX9U^0q42C@+B^LxHin6?sWQyn&YfBR{VGQ2`bum#$ zuE^4>)r521vm`+S30Fg$7Dixik3H+44?QINvZ+YHB53#Tr}7GgFRbKmTM7B1kU zw&gF*EBdobm2H?fJ~*`urKDe!$Q_)wr1E)$YHGYP4Yin{wDPWGN}gUD_#uK6>aNkx zhN67=iD|M{PC&_jaWs75leM?p0Y6=(M{SOo%L4(e%o2I5N_zV1%U4X9N*sdorg&A! z$H$CAbgsdn(-WA^6^!Q!lqe^;*Ec^JU>Uw+9)d%+E3!On4R)HTTt;{6O}R~5Z+3ZR z(HolYe?7fCzGm*~RbJFY{MF{r&=|A)^yZuo>!ZxUXEYAahpKC=@Sq6}50XI;(*PYq zGZWin%IwlwC}rs4gx_e(l_FC!a@DC$Eqx+%wRcuhe_yqKqMJ?&a!P&TjfUpUKkFJ5(Xg{kGUu2^QP9#j zdTEI~@uscGNE zMxM%_a#SWQ_o{MX<(9Rjee4NTsQ0(g%imFMiAVqGlQv)rrnnf~&iA*4F&U}F?KfgY zwG!f+$Ttbydz@QsP@IW*#)B$nd8WX|WDob_q{)|463<vk((foev4H^JW5cA1H=mJFpKZtGE((0{FIF<>ZIJ(Xy;=qG@00-tCKMY z82;*Nle^QC0*iQbtFIzm^K!vN!VD}}nZmu_^Ylj=)UBcTYunGw*Qns2)* zW7U!;%mV#D4M=at{fnqlW9tFp(Dh_f@L_y$5QN{hAyH;n#g? z>3g0dVU@QxEmU`L#_QzjnW&ER-7{`Hl0(JyDu*UZnG(+@K~eqvg%n)?C}Q8D2+46k z{mL}@lEcCPJ5GKxnOC>(!DqctG}}vE>`v3-;}flH1YBYIkcer|nfU6}USa){_3K=O zU$JaP$z4>;S_vNbp1iBYSd5k}1=^N^ckX+FJSUqi#vag7s*i~#m*bdwRty}rqplhJ zE{vZ&vPZ`5MK>j};pJC#Il>1ti@Yr$9xpdDj@6o;O(6lpcvgohJI;ki42cW;hF{+X zqP)EQ6dn|lhD)iXrfi8m12cp;Y;T&-4%iXSys08#($=y8FchgGo{efj^WV`krNYtA z_P1JpFSF)B(7&N_(iN0+E!#TfN27aie|fX0Af2@0ee}&=lr1eOamb64#`58#TeH&<3um4vS4wY&BFyoqsTFJFe6 z)0lvI{l0pyytklT!p&4d!D<_(u6)lf{ApJ8HB;K`_lEkgb;mE*u1|+ES;=wb4@f~XETi>ZFqzI-bd{xIjy1ur} zii*TF-=)Y?f5n3P@%&b{Ya?;P8yBWLT79{h#xTtOd_^m0h}!o+xXyX=nc4C;Xoo+Q z*aV|Z+AoG~-<1o%1Y^bRwkW!p2Kgdw z1|Lq$Za1PwbLu;Ic#B>w9(WAHwKOclalzysxL8qNU z9k^2Ohz?&+V5Q|!u&?KeNn}>O%@#A{#jfn~B8Uz#tf7lGA1U<+&y$_RU8&wj}essZLcUQ*rP* zmEBUgLg+i@V>q*S@}f&AYlYP;w!PyMRj67 zB*hFy%zaUyLyypwl5<4_Ck%DCj7iKHGBjW}-20(tLIdy5 z@Xk*SJOKu>FoAPn0t+`Sf4%Y8T9o{E5^H7bV$vraJihBJc#J&8^^wVuYkul_U~OyZ z5qUL3Ukr2U>0-^6cH9JH94~@xZ_Q$9)IVL3I5i0hwKV-|`Ba|hbyhlIzJghXC0ych zBE^WiE&MjiBX)i1S?C5zqH~vJGYTr|z!&~@pIPDH-RpMhS2kIK`C!rSwNGj)xe2mZXi%}sgimI%^ zL4W)8A4DR0BtV(g(vf_g{QbrJUAN^G0Kda5RDq8Ar!xNcOw$YO1n3*y`4|FtW>vUqn(vJ1D}N9fG*J zfUJ>b6W{)4TL#V?!Tyn?XaxlI5|Ne|4EXa9I9lnKpKQbaSP1oZ6l&QoR|q^J$Ws=; zU!QxOki0~8gPLhawC^B8t>@3H0iy8TOIi|)&%WdT+qwR9CCHC*5q zkh*TnxqD%bg7AbJ`rnfXaa^2Y*53Q($uas2gJytC=^zczy&Srh*W>Y%2OizFir&$+G&atqlGZ@$JN$b$5j% ze=wzNq3gR6z$A5DU>M1)5WtLujQ3^$bZBlKocTcBmtr^KqYB6U!z?R35apJui{menz?^6AAQ#@1|~BedR~z@Wrr z`uA_#v=$cUtdS>nb{Iw;-C_O?+Rv>cRt6W01e`m~XG^4nXZsu34eJ+1OdER~Cn_|B z!lrj!?#dJ$M6Jj4e@!}yLhIgCkKQ5b5`LQF3Ai`Md$0JODtJ}z)pY0>T5{Hif(;tlY|d{I zYx&)@+>EH=Y9!m2W#qHl&LCgs_ElaBia_va-d{)qOYbPl59aK5DIwEN?tKI9fX%@k z%`adxJ5lO&0Zy@eThc4y@Vd0XCdf-E~LvCR5H5osVyjyW{|OTRRTL z7FAV^$KJaK8P>lKC1P>ofjK2;n1tes>(AG&ZjN{B?RK?Lfe4 z2mSK2-*hqNQ>}3vgva;7i1X&N>+cCL%sNx2Y_ZfRIW6a~olJBvp7~uxZvON62jg06 z72`D3eaH7Ich}O;l&a_Uqap%MclZKs>QtPRDp~wPu{Z9m!mpQ&TdPfI;|}J(xD;gY zsNmx7^B)mmF?J@DbGHfbme#9U>|D=v*RdO6i-|7*G-$LYq3xG`QFd@`}t$K^D0hU zh4jw8hCL9xeeufa}h@4%ovBKmMU%`fHTOTor<) zV78!lbEcK-d6oyUE#gS|BxXM`z{OoN3D~4-n01_xGZdCMsQfM}cH6ye#5#7~ zD;$$@_70~D^{uC0@mjv7I!%ovZsy}W-caguZ~vAa32D{Czp)*hiokaH)!ssSs5}<1 za5dmG4-l)&FDEAEL8~ zZwXIZDv8h2z_TwQ?dLpf()~{@ep0cvB8AOZcvVOrkTrSJj`e1=iWwc}+kt=0Vi`}A z<7`|f9B}*YS!QFKf82~B@^?1NU}#3v|HL4>cQ9$_bVzF1Ag~#+S4??I02t%9URw3z zTi=3nmKn}Q77vEROD|fL87<++h3FTLs$vdQb0TCyEdaBvz#bRNoh{P3yxB1!>ZnX~ zDc^kiTVRaj(cO}zi+s&IOk4=k_7D7vK~U{ge*zl_Y#c`8+%P z-%P;B36W={-p;7|il)GnhB`cBT1hFiksouCqIQ?QXJSEC*d|U(NbHV`-hy>89s!&J zQuXNFZ%ohW6PQGjZZd85?d94$W`^MAxytgq@pzAAe|lY=-HHbd4eOsOtM|J^r*XWS zKRZ}~HrN~bL>}6!x8)^_Djs6q)+Jo5xt130F=^TtdUCKW&G2D`^!0 zB!_ngwVdr2d6hxSy6lDyzefen^%?*~rB2ZRizyw=?;o7)4eIJ`#+z}ODJ>@1=yZK| z2N!eh!m|slbogBgW@T%}xd~bl14E{cr&*ga&nVKcs~vtuy;><9rZd+~b9T2G`*cW; zeFm**L7|A3Ho@+ zr&Kl_CF}7C8XblEn5H#*SOUFWWCOPQoKzSJ zHgfy<=+2IE`mWx}vm5ezt}?2=d)e&nLBY29&2@h9erU_Z1pPSm4G+uw0}R&UQEv>j zN)0}$X!SZSKtaJ(=0BY5gk9Xy2Z-%oU`b=IhZ8&H`B|19PysBH#{15`?$44hw#W8< zo-X2_MyMm_vY_CkrhUV8ZRBfGTA;cH9VFVAHdM!*guRcI${%jB$=dT+a*=PkpPeBB zxTW&IK_Nzg*m=u^21KYY&*vKz0A3!V_v(We&kn1W&O^K5C$2k#1+z&#Ma#n<3yMZMxqQ z^{B6utO*q?u;5Z8&r9q&Uo2eTH@)&ZudrL~1$P{$g@?8)g=lUS6KpIdT1y7s?|!L9 zK_%HH*&sSfbRQG4+;@I7P)n4W^}$+yuRSCMoxpF`vl?a!p*2%u68gH$a$?!!O=}Xjf2? zsE?pwP4CJ7AFMi z^XL5_K%qy46{w3>54!Qt+IzOf z%GdPCegVLhn&l-mhdAdPZT`Ac z&zD8ylzxAoMWh>)bq4}{cN&-!8@4z5YD1?X`;Mp6OCcgb;DZQ#5Vw?M2+5WwF?1|i z1vfoRA!4r2RWcoWp}w5|N-!=D11$jGI^LR?hJe#Uab zo!}WQ!sBBQhlPT+v-!CR05(TTJgq-gC@;Xc?1;;jx|@`Ky*Z(KG1DhbuP8??d-imdg6zI z1hW_5apfGp>tGbISY`D2Nyd>>2K%KABXnmfaj>u38ad9#Mhgyzy4L$q_Vjp0tT;FC~+>etf-@$`NGIoQEr@bwwy(4?+>j#c?iavad<+wmvp2)xs9cZ7)7DO*1!0oSWiMSxX?ortmvpk;&Iq9TRBY;k zH_aq6AUI`SsSas`broB?Fl~Z1IV)tKrEZJ&0ulUC%`BB9*b0Bis!w7`okcDBFf9Ho zFSVY3LLB}>D-o@vwD5Aa2bfhk9;#DZ92%V}c)r^&b$OgG+xT!(&fn1hbXesr>$HhK z15;$u&3r!#ti54uyWxC#j-^u>xv!q~EjM3;xOtWdPct>6A-so_B)sSovPR*!@&<|q z_n~Ln!wke}WH1!m$v|Ork@w(+kx}m&utwO(m=mz*air$bNA6xcZ`RVei!9mBgK`I2 zkwbJrd1f}jxKzmfD&T5>%lX?(+-}zo zp^sriVPZjmKzw50{leX|=+-OH7zzmE;Ww}glIC?WoWC0=TbXsin?=R(o5i8U-`LKD zd9X5?_}yJMh#m$@bHxf7C7Vi93gK4f-8rT`~2JR3am$9g7ty5~3kY%R>NuyKn*rOhnN&@XB zmbzj<-Lw?NA9BG?C4-QZ+1QZnVntIHXP)DkH#HSKhl(nXDh+gRJyVlV)cBv6_2t3* z?jCYMSs(LmVDVv1u*;>Nhg&*VX;dTXrAM5xqylau?_7t4%{0-3fAk*0JuJqQqV|2H z9HgOMExTvJk`P3QKcFjUis?{1VdX?6F^5DOOMXBv#U<$lpGR!HUh%~*3h3%7amoh8 z>%;`Wc9`pokDoEnlKc0IT8nV-d_`JK7haB;X7DJ^p_n1{bT^HpM!hUZkVNI9pIC3f zq4V<(-T<5{z)oM>j%wgst!kyJ(-ciAZ$_oW`5x+Ye*5rN22Yg7y}N7C(z0ckLd_FY zE7x+y5p3MK(bCveCx4YIjS%wMVM5I6b84dR*01&@E-F9Ko0bh-;avh!CIEo;;)FNh zGEl_GGI>_@JxTLAPPJnW`^(B)c7POqvJKDy-fc%qsb!$*_|EgABKO{6xv;<7Lx^EF+mtINjkl7IJetc%%h(a(&MAD9l}jms9hfz&qNFwZuv*oi5h{R1y{s zTF{vZQj=#TvT=Rl=!@-OrBsSu9D!zek%AV(hV%@&$i_dBkt`P5P_iqt6UYfSDG_Lr{h&F)UPK%vFrna7%;8voVKbx>{1%6_9bq)AtRSCNzXSit9AETc=RLA zVCo4xsv*5aM;AN9iQw2$y}{~EYmOrHz0h4kTbo=UdXdlay_s-#+9wV7IFhec!^<;W z=S-1xG1a&@_^OOXXKMRG3m4`X%*VIv-yWsmySu<&vC6*zS^2 zzHcM(X{gUEOrxwk2$xA^m5@Ew;g4SlnLLJm#6AWn3oGO}E6sxg)GkFGB+HwhatmB8 z7rz$STL~d)St_T!&K1GNbu;&oPRD+(NQra1mr;eCR}Zp9f7sSVE1F)MSs~AnGP6gE zElg1C4j^54((iC+`_9&xye@(E&dJE;wzO+NE&|m znrr3tZ6|Ed8!NKbHQBb}*+8kEErSv{n4Ik6*HN~SWgL@uA60;-=&f;Et*Ed>e;VPn z&=Y~;q@wY4XzT44CFhoC6r5U>Z~a_>T_Nqy4;#BsmZ2fSX%iNVv$o-yMg~Kw)fubA zkBM9|gE~R5AFVqc0X+hw=7L2yr@v=Pnk;f59N007!yYT)4;!Cxq;h*v=--UTyRS-v zRlcZN!AsLR(!XMyA@1sXlOOxm_pz)^U6qI2|BE#~(3)_Rk;`>ZC);_&ybjHZvyS9%YX9~gqK>{71XG!d?noKUzII+!>h3D6as-FQtT3N8xK$cwofetQf7L`& z>(`0tZ8U2(U(ILi#MB?Xd?I(4X)|Aa7nJNzLoOq(21RJuu%zecBrm5_VQTFWAuN;E zDNSXq;nHL5&B--Wm=8H4Z#gQHDTWJV5uHc!utqFRcyR!~gBrtm9+99M>ap!?Y6itanCSM5z1alyyocva85%Ie9AqsCb!jw@N#P zW^#|Cr9w)aleO{Mf_i-a?P#-Zzw9Jc3jR~tG}r7q5uftQt_1AH!R#U`HnV12T~^&t z=&aYedw!LfL3bz5&8LyfUix@~kyuwH`GoB@E6)e={xYi?6#S20l`qGptoH{cUtX)q z!L&sl{B#9jDrlX&Ln0o~rkIP{bk811T|v8a*j=}k8+3_pFCbaT**Tx&u}LEyg@gQX zDWM&%lVZO*t->HdYeH2v7<%&Rh4T9(8)l*<_p|9yS-`TmErU%3mIileN-Tj4qH_unzp+0PI9vaxX& z)W(!RH8`b;Vs4-W*`Gc+A>O1~=dyD>H!BI1hEAcE9b=?7%jSzK@qEMNFQ`U`vf#u= z$k)692C-MDK_TF^_CcxY;S>J;@p5(16DS~WVl&~*w0UsA_w)Fa*IpCm-`A*C+i08J z-Z%^I%gwiC-cTG9j9B&1b+wAstBOZ(Y1@pH7IHkC4;c>xMHWg*2a}(M&7HnVTXK(Z z}feVLr+ z9k&T8jt(~MBD!_KrhDJpczHjf$lz9C9_4*vlM6xN2c~bV1X(7`@?~r?Fpx7L$}cY`d;$p1)hG#p0zfVOe3qXoZlno|S$qUF+Y~Wc+81@hSTY;m zYUL%4lZXgrX{}#4c^4oCrlR29WUJqjyQw21iJ(PxK1^ICU{vEKo>W;p>f@`Y_0ZqN z$=ii^5V0Bk?v5)NLvz??sSLkTisFR{mXw?8>&Vr zEWIm|iMF1}F-o(inW9hc4=#AsofL|0sM$J5FmlsR5En4r6Ap}f^{)n%9IVy&Q0@W6 zz;#mIIzHU$RJ%uf70~&K6E8fR#7>Ob(Rr^Lo9z8IEQ9T`I-mbN+U{{XxP0H0Vg>Ed z(;)yQVS2bo7C?s_;HUY<8C{LPyO%PGYb{v!Y~lKTIgZzvvA$y!dV~m4Un4Wwfe}JU zYB%)2A(HbI`)u?Ybh8yoavilIS4bu3Q>e3bZ>Cue_Y;8riPUs|KW98jBtcX_W!5iT z8g3qL6)xKinW)#7UgZ;&yul?WTK&|lSi#hLu`x5;5lZA5d%gf@7qIpHQqF3MWej%f zc2@;?Vs+g0Z<c*jMc1{X00*>=9to4}oEp=^KrWf-Ki3A)s9A^8WT31<7Q6f+1 zlZs97&}g*Hn1?-|nTox6ei7iJ-|~5LW{u;-EIpYlU!tDGgZ*x#m*qrDLl4Y1Xo$nD zxAo2=SBVlsDUIXwbh!)Hb4Z*+A|L2X_*blDAjRMs4zWs@hvsk^ zVxg9Oxg)xPHT7($+e`nJHN=A?TS&VXM2k2v7;G_mFEEMdsN=iZ)`XtzOteK`q9Wgb zZ&`Fhq@P0=6QyCsuit^XJ6x0H-)gvegl_E2KrF_;JP7&qlK(K7xpL_grE}D) zE3&e=T+*V+S?gz2_|u^YZ+s{B#SGd_miOJRA#g&&Qwp+pHbWKZe!@2Ce)3+vHo@q` zWXpUBbEg&n7psls-&A574m#6f;Z%Q|%d=7G$`WCJQ8%kMa@(K%(i?6BvMIA9+N?UJ zVu;lxnR^dWZhV9q>%LvK3V@uXgiuj=@iNv|)!_~Byqy-6=RUN0VTB)i*Q@|GD z{?-0jJur`j&{D9X_{x4bA!CMmWEN{@r}4g+vr{aJZeI=;CVP{<5sE-!Q4oDo`hsB5 z6wwG$QvK$R)EAht7@w3R-|>D4#aLZqLiKe06eIi`B-g~KxEF8y5g}|5S?SznaoGUF zfupNSl}q|~k_v!KA5F)lN4k$&Q?p42%(~mhdt;&t+02m+!BhZ)8E@v-VDOE zzaP9I(?`nSp{V^jOJZ8S8(iNl7VN)W1Vx$gYUE7xm7dYQg7)(et`wbvX>ZJC5|5va znXi%jc%Je>yBtFCHYGj(J(EPoU8wf;&`dci^VDLJd6rJ$?$qOaYITypnq^WQo-^Su zHM?D#-G{eF$48q{J3TLmcJ*lYwJ)3B&R2`-&N!9Z3nD(+DkYHgdRYo$K5oFMuln8= z63prpm;2wHCI=>HEK9$zS1kZOFv`A``=KX7W>J%S*=TLhF zVtg~d(amGk0BEEGZG7{G40gtsT1|%cJ98Lm_GR4Ne&wgtaU~O1eXyUdml*9mExo3R z;t#jRW)U1Y-Y4hrH6Ja`rqK+bo;Z+b#ZT=3H2+;)+3`vq+IDP*5E%M(M0IX#PKQKxf zu>`%kFtQ)pg5;>|T{%!BrRi%n*<7W-G9|k=n!Z`Fzkb9)ivzdRd&G=%A*C97mI z5kc#il>Ajhh$bop@`&PDn^e)DXfMXOmQJYJe@pmL*a@Ez6jjb~?EM#1d-?n2z$Tlo zgw>&72r)bU)jJKM(0Qx?op6?0%97QG8Pj#eGf3xvS6}eyN)Ax`Xpj4AFDea36XXu) z3%xWojYTU2ztTiIE_^pk42c4R3N2vf`eb=18fzDoa$=LtlWAG@?dqj%+dReOMaPkC>2 z+*AdJb8pn%_8MVg|7MjS5`!QMjg%cmavY4jjZ_0F1OidZZMCif>tQbjM@b`3 z$GVGdRG$cNY_?%Lj$;L=t<@gNJ=^blL3OAn)92379CAzK7kBvhhX2#v{J@C)Ly#(1 zMdsW*D_WHBp7OBPiA>bSlr6>WOR_MfA{T`y<^5hIsxj56?^>>trUWzR+j2vf{qO4m z<%Y3zf2xc6E&{Ej#9Oil_N$uFu?Rmg|17<*t1SxxKdc6-N|U-O95B&gEh4Tc(XMZ+ z1p&ou(-y8@&erMs$8jy#kEbI1CGysEDr62hJL6S zv)b%-9}3`D9)&)aiN>SRuyBkWkFU86#gz49s}$ENwv#p<_{Z@P#rp7tHC@eAGff(G zfSJ><(Qg^K!Q9*Xj*XBT@c8%o#7v;qu0c{)2Fp2umuS5>#$tk$yU?SituX=!o^cu= dH*pxl(F3@{GFUQnRvr%cQIJuU21~vU`hSsZ1(g5* literal 0 HcmV?d00001 From 26dba289ea0246e4cb9065992a1b2336c61ee5a0 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 29 Aug 2024 21:23:50 -0600 Subject: [PATCH 06/32] Fix MNG scale command missing $ --- .../basics/adding-nodes/index.md | 30 +++++++------------ .../adding-nodes/tests/hook-wait-node.sh | 1 + 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/index.md b/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/index.md index f4740f1f2..20ce910fb 100644 --- a/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/index.md +++ b/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/index.md @@ -14,7 +14,7 @@ $ eksctl get nodegroup --name $EKS_DEFAULT_MNG_NAME --cluster $EKS_CLUSTER_NAME We'll scale the nodegroup in `eks-workshop` by changing the node count from `3` to `4` for **desired capacity** using below command: ```bash -aws eks update-nodegroup-config --cluster-name $EKS_CLUSTER_NAME \ +$ aws eks update-nodegroup-config --cluster-name $EKS_CLUSTER_NAME \ --nodegroup-name $EKS_DEFAULT_MNG_NAME --scaling-config minSize=4,maxSize=6,desiredSize=4 ``` @@ -24,29 +24,21 @@ After making changes to the node group it may take up to **2-3 minutes** for nod $ eksctl get nodegroup --name $EKS_DEFAULT_MNG_NAME --cluster $EKS_CLUSTER_NAME ``` -To wait until the node group update operation is complete you can run this command: - -```bash hook=wait-node -$ aws eks wait nodegroup-active --cluster-name $EKS_CLUSTER_NAME --nodegroup-name $EKS_DEFAULT_MNG_NAME -``` - -Once the command above completes we can review the changed worker node count with following command, which lists all nodes in our managed node group by using the label as a filter: +Monitor the nodes in the cluster using the following command with the `--watch` argument until there are 4 nodes: :::tip It can take a minute or so for the node to appear in the output below, if the list still shows 3 nodes be patient. ::: -```bash -$ kubectl get nodes -l eks.amazonaws.com/nodegroup=$EKS_DEFAULT_MNG_NAME -NAME STATUS ROLES AGE VERSION -ip-10-42-104-151.us-west-2.compute.internal Ready 2d23h vVAR::KUBERNETES_NODE_VERSION -ip-10-42-144-11.us-west-2.compute.internal Ready 2d23h vVAR::KUBERNETES_NODE_VERSION -ip-10-42-146-166.us-west-2.compute.internal NotReady 18s vVAR::KUBERNETES_NODE_VERSION -ip-10-42-182-134.us-west-2.compute.internal Ready 2d23h vVAR::KUBERNETES_NODE_VERSION +```bash hook=wait-node +$ kubectl get nodes --watch +NAME STATUS ROLES AGE VERSION +ip-10-42-104-151.us-west-2.compute.internal Ready 3h vVAR::KUBERNETES_NODE_VERSION +ip-10-42-144-11.us-west-2.compute.internal Ready 3h vVAR::KUBERNETES_NODE_VERSION +ip-10-42-146-166.us-west-2.compute.internal NotReady 18s vVAR::KUBERNETES_NODE_VERSION +ip-10-42-182-134.us-west-2.compute.internal Ready 3h vVAR::KUBERNETES_NODE_VERSION ``` -Notice that the node shows a status of `NotReady`, which happens when the new node is still in the process of joining the cluster. We can also use `kubectl wait` to watch until all the nodes report `Ready`: +Once 4 nodes are visible you can exit the watch using `Ctrl+C`. -```bash hook=add-node -$ kubectl wait --for=condition=Ready nodes --all --timeout=300s -``` +You may see a node shows a status of `NotReady`, which happens when the new node is still in the process of joining the cluster. diff --git a/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/tests/hook-wait-node.sh b/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/tests/hook-wait-node.sh index eff00005a..e684dff5d 100644 --- a/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/tests/hook-wait-node.sh +++ b/website/docs/fundamentals/managed-node-groups/basics/adding-nodes/tests/hook-wait-node.sh @@ -14,6 +14,7 @@ after() { if [ $EXIT_CODE -ne 0 ]; then >&2 echo "Node count did not increase to 4 as expected" + kubectl get node exit 1 fi } From 4386fc84fffec177b39c4913d4c35b298184f179 Mon Sep 17 00:00:00 2001 From: shivam-dubey-1 <64331200+shivam-dubey-1@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:16:28 -0500 Subject: [PATCH 07/32] Update Neuron Device Plugin Deployment to Use Direct YAML URLs (#1060) --- .../modules/aiml/chatbot/.workshop/cleanup.sh | 3 +- .../k8s-neuron-device-plugin-rbac.yaml | 59 ------------ .../k8s-neuron-device-plugin.yaml | 95 ------------------- .../neuron-device-plugin/kustomization.yaml | 5 - website/docs/aiml/chatbot/expose.md | 4 +- website/docusaurus.config.js | 2 +- 6 files changed, 6 insertions(+), 162 deletions(-) delete mode 100644 manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml delete mode 100644 manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml delete mode 100644 manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml diff --git a/manifests/modules/aiml/chatbot/.workshop/cleanup.sh b/manifests/modules/aiml/chatbot/.workshop/cleanup.sh index 5079728ba..d82c3a314 100755 --- a/manifests/modules/aiml/chatbot/.workshop/cleanup.sh +++ b/manifests/modules/aiml/chatbot/.workshop/cleanup.sh @@ -10,7 +10,8 @@ kubectl delete -k /eks-workshop/manifests/modules/aiml/chatbot/gradio --ignore-n logmessage "Deleting Llama2 pods..." -kubectl delete -k /eks-workshop/manifests/modules/aiml/chatbot/ray-service-llama2-chatbot --ignore-not-found=true +kubectl delete -f https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/v2.19.1/src/k8/k8s-neuron-device-plugin-rbac.yml --ignore-not-found +kubectl delete -f https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/v2.19.1/src/k8/k8s-neuron-device-plugin.yml --ignore-not-found logmessage "Deleting Neuron Device Plugin..." diff --git a/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml b/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml deleted file mode 100644 index 7bc6879ad..000000000 --- a/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin-rbac.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# rbac.yaml ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: neuron-device-plugin -rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "" - resources: - - pods - verbs: - - update - - patch - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - update ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: neuron-device-plugin - namespace: kube-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: neuron-device-plugin - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: neuron-device-plugin -subjects: - - kind: ServiceAccount - name: neuron-device-plugin - namespace: kube-system diff --git a/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml b/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml deleted file mode 100644 index 3a895a6eb..000000000 --- a/manifests/modules/aiml/chatbot/neuron-device-plugin/k8s-neuron-device-plugin.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: neuron-device-plugin-daemonset - namespace: kube-system -spec: - selector: - matchLabels: - name: neuron-device-plugin-ds - updateStrategy: - type: RollingUpdate - template: - metadata: - # Uncomment the annotation below if k8s version is 1.13 or lower - # annotations: - # scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - name: neuron-device-plugin-ds - spec: - serviceAccount: neuron-device-plugin - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: aws.amazon.com/neuron - operator: Exists - effect: NoSchedule - # Mark this pod as a critical add-on; when enabled, the critical add-on - # scheduler reserves resources for critical add-on pods so that they can - # be rescheduled after a failure. - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - priorityClassName: "system-node-critical" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - # Uncomment following matchExpressions if using k8s 1.16 or lower - #- matchExpressions: - # - key: "beta.kubernetes.io/instance-type" - # operator: In - # values: - # - inf1.xlarge - # - inf1.2xlarge - # - inf1.6xlarge - # - inf1.24xlarge - # - inf2.xlarge - # - inf2.8xlarge - # - inf2.24xlarge - # - inf2.48xlarge - # - trn1.2xlarge - # - trn1.32xlarge - # - trn1n.32xlarge - - matchExpressions: - - key: "node.kubernetes.io/instance-type" - operator: In - values: - - inf1.xlarge - - inf1.2xlarge - - inf1.6xlarge - - inf1.24xlarge - - inf2.xlarge - - inf2.8xlarge - - inf2.24xlarge - - inf2.48xlarge - - trn1.2xlarge - - trn1.32xlarge - - trn1n.32xlarge - containers: - # Find all neuron-device-plugin images at https://gallery.ecr.aws/neuron/neuron-device-plugin - - image: public.ecr.aws/neuron/neuron-device-plugin:2.19.16.0 - imagePullPolicy: Always - name: neuron-device-plugin - env: - - name: KUBECONFIG - value: /etc/kubernetes/kubelet.conf - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - - name: infa-map - mountPath: /run - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - - name: infa-map - hostPath: - path: /run diff --git a/manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml b/manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml deleted file mode 100644 index dd1b8b045..000000000 --- a/manifests/modules/aiml/chatbot/neuron-device-plugin/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - k8s-neuron-device-plugin-rbac.yaml - - k8s-neuron-device-plugin.yaml diff --git a/website/docs/aiml/chatbot/expose.md b/website/docs/aiml/chatbot/expose.md index 473635fbb..27611c09b 100644 --- a/website/docs/aiml/chatbot/expose.md +++ b/website/docs/aiml/chatbot/expose.md @@ -47,7 +47,9 @@ You can learn more about Neuron Device Plugins in the [AIML Inference module](.. We can deploy the role using the following command: ```bash -$ kubectl apply -k ~/environment/eks-workshop/modules/aiml/chatbot/neuron-device-plugin +$ kubectl apply -f https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/v2.19.1/src/k8/k8s-neuron-device-plugin-rbac.yml +$ kubectl apply -f https://raw.githubusercontent.com/aws-neuron/aws-neuron-sdk/v2.19.1/src/k8/k8s-neuron-device-plugin.yml + serviceaccount/neuron-device-plugin created clusterrole.rbac.authorization.k8s.io/neuron-device-plugin created clusterrolebinding.rbac.authorization.k8s.io/neuron-device-plugin created diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 6bdcbff47..c9ae30ca3 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -162,7 +162,7 @@ const config = { type: "doc", docId: "aiml/index", position: "left", - label: "AIML", + label: "AI/ML", }, { href: "https://github.com/aws-samples/eks-workshop-v2", From 9dc3ef96c983542c13bee93aabd7dc74a7b23b99 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 30 Aug 2024 15:24:52 -0600 Subject: [PATCH 08/32] Remove 1.30 upgrade header --- website/docusaurus.config.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index c9ae30ca3..0c7b7cd97 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -91,13 +91,6 @@ const config = { themeConfig: /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({ - announcementBar: { - id: "upgrade-1.30", - content: - '🚩 EKS Workshop upgraded to EKS 1.30 on 26th July. If you have an existing lab environment please see the major upgrade instructions. 🚩', - backgroundColor: "#0972d3", - textColor: "#fff", - }, colorMode: { defaultMode: "light", disableSwitch: false, From 4dd3c31d9c55ce0aaba0889f69041e77159048c9 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 30 Aug 2024 15:25:24 -0600 Subject: [PATCH 09/32] Switch out kubectl commands in navigating page --- website/docs/introduction/navigating-labs.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/website/docs/introduction/navigating-labs.md b/website/docs/introduction/navigating-labs.md index 56d497727..861f1faf4 100644 --- a/website/docs/introduction/navigating-labs.md +++ b/website/docs/introduction/navigating-labs.md @@ -60,11 +60,8 @@ Hover your mouse over `echo "This is an example command"` and click to copy that You will also come across commands with sample output like this: ```bash test=false -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ip-10-42-10-104.us-west-2.compute.internal Ready 6h vVAR::KUBERNETES_NODE_VERSION -ip-10-42-10-210.us-west-2.compute.internal Ready 6h vVAR::KUBERNETES_NODE_VERSION -ip-10-42-11-198.us-west-2.compute.internal Ready 6h vVAR::KUBERNETES_NODE_VERSION +$ date +Fri Aug 30 12:25:58 MDT 2024 ``` Using the 'click to copy' function will only copy the command and ignore the sample output. @@ -72,13 +69,10 @@ Using the 'click to copy' function will only copy the command and ignore the sam Another pattern used in the content is presenting several commands in a single terminal: ```bash test=false -$ kubectl get pods -No resources found in default namespace. -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ip-10-42-10-104.us-west-2.compute.internal Ready 6h2m vVAR::KUBERNETES_NODE_VERSION -ip-10-42-10-210.us-west-2.compute.internal Ready 22h vVAR::KUBERNETES_NODE_VERSION -ip-10-42-11-198.us-west-2.compute.internal Ready 6h19m vVAR::KUBERNETES_NODE_VERSION +$ echo "This is an example command" +This is an example command +$ date +Fri Aug 30 12:26:58 MDT 2024 ``` In this case you can either copy each command individually or copy all of the commands using the clipboard icon in the top right of the terminal window. Give it a shot! From 872da6d62bad60ed5d69dc01a93bf9f00b9c8bc0 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 30 Aug 2024 15:26:09 -0600 Subject: [PATCH 10/32] Fix lattice cleanup for incomplete sessions --- .../modules/networking/vpc-lattice/.workshop/cleanup.sh | 8 +++++--- website/docs/networking/vpc-lattice/index.md | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/manifests/modules/networking/vpc-lattice/.workshop/cleanup.sh b/manifests/modules/networking/vpc-lattice/.workshop/cleanup.sh index cd489f6d2..f7357490d 100644 --- a/manifests/modules/networking/vpc-lattice/.workshop/cleanup.sh +++ b/manifests/modules/networking/vpc-lattice/.workshop/cleanup.sh @@ -9,9 +9,11 @@ logmessage "Deleting VPC Lattice routes and gateway..." kubectl delete namespace checkoutv2 --ignore-not-found kubectl delete namespace checkout --ignore-not-found -kubectl delete -f ~/environment/eks-workshop/modules/networking/vpc-lattice/routes --ignore-not-found -cat ~/environment/eks-workshop/modules/networking/vpc-lattice/controller/eks-workshop-gw.yaml | envsubst | kubectl delete --ignore-not-found -f - -kubectl delete -f ~/environment/eks-workshop/modules/networking/vpc-lattice/controller/gatewayclass.yaml --ignore-not-found +delete-all-if-crd-exists httproutes.gateway.networking.k8s.io + +delete-all-if-crd-exists gateways.gateway.networking.k8s.io + +delete-all-if-crd-exists gatewayclasses.gateway.networking.k8s.io delete-all-if-crd-exists targetgrouppolicies.application-networking.k8s.aws diff --git a/website/docs/networking/vpc-lattice/index.md b/website/docs/networking/vpc-lattice/index.md index 3725b134e..c873152ce 100644 --- a/website/docs/networking/vpc-lattice/index.md +++ b/website/docs/networking/vpc-lattice/index.md @@ -37,7 +37,7 @@ Amazon VPC Lattice takes care of common networking tasks such as discovering com The primary benefits of using Lattice are: - **Increased developer productivity**: Lattice boosts developer productivity by letting them focus on building features that matters to their business, while it handles networking, security and observability challenges in a uniform way across all compute platforms -- **Improved Security Posture**: Lattice enables developers to easily authenticate and secure communication across applications, without the operational overhead of current mechanisms (e.g. certificate management). With Lattice access policies, developers and cloud admins can enforce granular access control. Lattice can also enforce encryption for traffic in-transit, further increasing security posture +- **Improved security posture**: Lattice enables developers to easily authenticate and secure communication across applications, without the operational overhead of current mechanisms (e.g. certificate management). With Lattice access policies, developers and cloud admins can enforce granular access control. Lattice can also enforce encryption for traffic in-transit, further increasing security posture - **Improved application scalability and resilience**: Lattice makes it easy to create a network of deployed applications with rich routing, authentication, authorization, monitoring, and more. Lattice provides all of these benefits with no resource overhead on workloads and can support large scale deployments and many requests per second without adding significant latency. - **Deployment flexibility with heterogeneous infrastructure**: Lattice provides consistent features across all compute services – EC2, ECS, EKS, Lambda, and can include services living on-premises, allowing organizations the flexibility to choose the optimal compute infrastructure for their use-case. From 7ce7ca96171f1490269e41e6507ee77d35fe28af Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 30 Aug 2024 15:39:11 -0600 Subject: [PATCH 11/32] Changes to account for VSCode option --- website/docs/automation/gitops/argocd/codecommit.md | 2 +- website/docs/automation/gitops/flux/codecommit.md | 2 +- website/docs/introduction/getting-started/first.md | 2 +- website/docs/introduction/getting-started/index.md | 4 ++-- website/docs/introduction/index.md | 2 +- website/docs/introduction/navigating-labs.md | 4 ++++ .../introduction/setup/your-account/using-eksctl.md | 12 ++++-------- .../setup/your-account/using-terraform.md | 12 ++++-------- .../docs/networking/vpc-lattice/testing-routing.md | 4 ++-- .../sealed-secrets/installing-sealed-secrets.md | 2 +- 10 files changed, 21 insertions(+), 25 deletions(-) diff --git a/website/docs/automation/gitops/argocd/codecommit.md b/website/docs/automation/gitops/argocd/codecommit.md index 8cc90240a..4f543f4fb 100644 --- a/website/docs/automation/gitops/argocd/codecommit.md +++ b/website/docs/automation/gitops/argocd/codecommit.md @@ -3,7 +3,7 @@ title: "Accessing AWS CodeCommit" sidebar_position: 5 --- -As AWS CodeCommit repository has been created in our lab environment, but we'll need to complete some steps before Cloud9 can connect to it. +As AWS CodeCommit repository has been created in our lab environment, but we'll need to complete some steps before our IDE can connect to it. We can add the SSH keys for CodeCommit to the known hosts file to prevent warnings later on: diff --git a/website/docs/automation/gitops/flux/codecommit.md b/website/docs/automation/gitops/flux/codecommit.md index 8250ee511..b1fb2dde2 100644 --- a/website/docs/automation/gitops/flux/codecommit.md +++ b/website/docs/automation/gitops/flux/codecommit.md @@ -3,7 +3,7 @@ title: "Accessing AWS CodeCommit" sidebar_position: 5 --- -As AWS CodeCommit repository has been created in our lab environment, but we'll need to complete some steps before Cloud9 can connect to it. +As AWS CodeCommit repository has been created in our lab environment, but we'll need to complete some steps before our IDE can connect to it. We can add the SSH keys for CodeCommit to the known hosts file to prevent warnings later on: diff --git a/website/docs/introduction/getting-started/first.md b/website/docs/introduction/getting-started/first.md index fd87e0917..1f0adc5fc 100644 --- a/website/docs/introduction/getting-started/first.md +++ b/website/docs/introduction/getting-started/first.md @@ -5,7 +5,7 @@ sidebar_position: 40 The sample application is composed of a set of Kubernetes manifests organized in a way that can be easily applied with Kustomize. Kustomize is an open-source tool also provided as a native feature of the `kubectl` CLI. This workshop uses Kustomize to apply changes to Kubernetes manifests, making it easier to understand changes to manifest files without needing to manually edit YAML. As we work through the various modules of this workshop, we'll incrementally apply overlays and patches with Kustomize. -The easiest way to browse the YAML manifests for the sample application and the modules in this workshop is using the file browser in Cloud9: +The easiest way to browse the YAML manifests for the sample application and the modules in this workshop is using the file browser in the IDE: ![Cloud9 files](./assets/cloud9-files-initial.webp) diff --git a/website/docs/introduction/getting-started/index.md b/website/docs/introduction/getting-started/index.md index a674ebe76..bd4d533cb 100644 --- a/website/docs/introduction/getting-started/index.md +++ b/website/docs/introduction/getting-started/index.md @@ -11,13 +11,13 @@ Welcome to the first hands-on lab in the EKS workshop. The goal of this exercise Let's deploy your first workload to the EKS cluster in your lab environment and explore! -Before we begin we need to run the following command to prepare our Cloud9 environment and EKS cluster: +Before we begin we need to run the following command to prepare our IDE environment and EKS cluster: ```bash $ prepare-environment introduction/getting-started ``` -What is this command doing? For this lab it is cloning the EKS Workshop Git repository on to the Cloud9 environment so the Kubernetes manifest files we need are present on the file system. +What is this command doing? For this lab it is cloning the EKS Workshop Git repository in to the IDE environment so the Kubernetes manifest files we need are present on the file system. You'll notice in subsequent labs we'll also run this command, where it will perform two important additional functions: diff --git a/website/docs/introduction/index.md b/website/docs/introduction/index.md index 40a0d9fcb..317be8b17 100644 --- a/website/docs/introduction/index.md +++ b/website/docs/introduction/index.md @@ -16,7 +16,7 @@ This workshop guides you through a set of hands-on lab exercises to learn and ex - **Networking** - Dive deep in to how EKS networking integrates to Amazon Virtual Private Cloud and more - **Automation** - Apply principles such as GitOps and provisioning infrastructure through EKS -These labs are designed in a way that you can pick any combination of modules that reflect your experience and skills to learn efficiently. The workshop environment provides pre-configured AWS infrastructure including an EKS cluster and AWS Cloud9 Integrated Development Environment (IDE) with all the tools you need to complete the lab exercises. +These labs are designed in a way that you can pick any combination of modules that reflect your experience and skills to learn efficiently. The workshop environment provides pre-configured AWS infrastructure including an EKS cluster an Integrated Development Environment (IDE) with all the tools you need to complete the lab exercises. To learn about the basics of Amazon EKS, watch this short ~10 minute video: diff --git a/website/docs/introduction/navigating-labs.md b/website/docs/introduction/navigating-labs.md index 861f1faf4..dc0a36945 100644 --- a/website/docs/introduction/navigating-labs.md +++ b/website/docs/introduction/navigating-labs.md @@ -80,3 +80,7 @@ In this case you can either copy each command individually or copy all of the co ## Resetting your EKS cluster In the event that you accidentally configure your cluster in a way that is not functioning you have been provided with a mechanism to reset your EKS cluster as best we can which can be run at any time. Simply run the command `prepare-environment` and wait until it completes. This may take several minutes depending on the state of your cluster when it is run. + +## Next Steps + +Now that you're familiar with the format of this workshop, head to the [Getting started](/docs/introduction/getting-started) lab or skip ahead to any module in the workshop with the top navigation bar. diff --git a/website/docs/introduction/setup/your-account/using-eksctl.md b/website/docs/introduction/setup/your-account/using-eksctl.md index 4cead763a..64c69c25e 100644 --- a/website/docs/introduction/setup/your-account/using-eksctl.md +++ b/website/docs/introduction/setup/your-account/using-eksctl.md @@ -27,23 +27,19 @@ $ curl -fsSL https://raw.githubusercontent.com/VAR::MANIFESTS_OWNER/VAR::MANIFES envsubst | eksctl create cluster -f - ``` -This generally takes 20 minutes. Once the cluster is created run this command to use the cluster for the lab exercises: - -```bash -$ use-cluster $EKS_CLUSTER_NAME -``` +This process will take around 20 minutes. ## Next Steps -Now that the cluster is ready, head to the [Getting Started](/docs/introduction/getting-started) module or skip ahead to any module in the workshop with the top navigation bar. Once you're completed with the workshop, follow the steps below to clean-up your environment. +Now that the cluster is ready, head to the [Navigating the labs](/docs/introduction/navigating-labs) section or skip ahead to any module in the workshop with the top navigation bar. Once you're completed with the workshop, follow the steps below to clean-up your environment. ## Cleaning Up (steps once you are done with the Workshop) :::tip -The following demonstrates how you will later clean up resources once you are done using the EKS Cluster you created in previous steps to complete the modules. +The following demonstrates how you will later clean up resources once you are done using the EKS cluster you created in previous steps to complete the modules. ::: -Before deleting the Cloud9 environment we need to clean up the cluster that we set up in previous steps. +Before deleting the Cloud9/VSCode IDE environment we need to clean up the cluster that we set up in previous steps. First use `delete-environment` to ensure that the sample application and any left-over lab infrastructure is removed: diff --git a/website/docs/introduction/setup/your-account/using-terraform.md b/website/docs/introduction/setup/your-account/using-terraform.md index bc92330a5..13e216921 100644 --- a/website/docs/introduction/setup/your-account/using-terraform.md +++ b/website/docs/introduction/setup/your-account/using-terraform.md @@ -9,7 +9,7 @@ Creating the workshop cluster with Terraform is currently in preview. Please rai This section outlines how to build a cluster for the lab exercises using the [Hashicorp Terraform](https://developer.hashicorp.com/terraform). This is intended to be for learners that are used to working with Terraform infrastructure-as-code. -The `terraform` CLI has been pre-installed in your Amazon Cloud9 Environment, so we can immediately create the cluster. Let's take a look at the main Terraform configuration files that will be used to build the cluster and its supporting infrastructure. +The `terraform` CLI has been pre-installed in your IDE so we can immediately create the cluster. Let's take a look at the main Terraform configuration files that will be used to build the cluster and its supporting infrastructure. ## Understanding Terraform config files @@ -62,15 +62,11 @@ $ terraform init $ terraform apply -var="cluster_name=$EKS_CLUSTER_NAME" -auto-approve ``` -This generally takes 20-25 minutes to complete. Once the cluster is created run this command to use the cluster for the lab exercises: - -```bash -$ use-cluster $EKS_CLUSTER_NAME -``` +This generally takes 20-25 minutes to complete. ## Next Steps -Now that the cluster is ready, head to the [Getting Started](/docs/introduction/getting-started) module or skip ahead to any module in the workshop with the top navigation bar. Once you're completed with the workshop, follow the steps below to clean-up your environment. +Now that the cluster is ready, head to the [Navigating the labs](/docs/introduction/navigating-labs) section or skip ahead to any module in the workshop with the top navigation bar. Once you're completed with the workshop, follow the steps below to clean-up your environment. ## Cleaning Up @@ -78,7 +74,7 @@ Now that the cluster is ready, head to the [Getting Started](/docs/introduction/ The following demonstrates how you will later clean up resources once you have completed your desired lab exercises. These steps will delete all provisioned infrastructure. ::: -Before deleting the Cloud9 environment we need to clean up the cluster that we set up above. +Before deleting the Cloud9/VSCode IDE environment we need to clean up the cluster that we set up above. First use `delete-environment` to ensure that the sample application and any left-over lab infrastructure is removed: diff --git a/website/docs/networking/vpc-lattice/testing-routing.md b/website/docs/networking/vpc-lattice/testing-routing.md index 0701827aa..e162514c1 100644 --- a/website/docs/networking/vpc-lattice/testing-routing.md +++ b/website/docs/networking/vpc-lattice/testing-routing.md @@ -29,14 +29,14 @@ $ kubectl kustomize ~/environment/eks-workshop/modules/networking/vpc-lattice/ui | envsubst | kubectl apply -f - ``` -Let's ensure that the UI pods are restarted and then port-forward to the preview of your application with Cloud9. +Now restart the UI component pods: ```bash $ kubectl rollout restart deployment/ui -n ui $ kubectl rollout status deployment/ui -n ui ``` -Let us try to access our application using the browser. A `LoadBalancer` type service named `ui-nlb` is provisioned in the `ui` namespace from which the application's UI can be accessed. +Lets try to access our application using the browser. A `LoadBalancer` type service named `ui-nlb` is provisioned in the `ui` namespace from which the application's UI can be accessed. ```bash $ kubectl get service -n ui ui-nlb -o jsonpath='{.status.loadBalancer.ingress[*].hostname}{"\n"}' diff --git a/website/docs/security/secrets-management/sealed-secrets/installing-sealed-secrets.md b/website/docs/security/secrets-management/sealed-secrets/installing-sealed-secrets.md index b88a5a784..f9ffd7cc1 100644 --- a/website/docs/security/secrets-management/sealed-secrets/installing-sealed-secrets.md +++ b/website/docs/security/secrets-management/sealed-secrets/installing-sealed-secrets.md @@ -3,7 +3,7 @@ title: "Installing Sealed Secrets" sidebar_position: 432 --- -The `kubeseal` CLI is used to interact with the sealed secrets controller, and has already been installed in Cloud9. +The `kubeseal` CLI is used to interact with the sealed secrets controller, and has already been installed in your IDE. The first thing we'll do is install the sealed secrets controller in the EKS cluster: From e6f3e6caccdc4fd2048818991cd85a19d6838c05 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 30 Aug 2024 21:33:35 -0600 Subject: [PATCH 12/32] Set EKS cluster name in CFN --- lab/cfn/eks-workshop-ide-cfn.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lab/cfn/eks-workshop-ide-cfn.yaml b/lab/cfn/eks-workshop-ide-cfn.yaml index 532b46a6a..d0e22e7b2 100644 --- a/lab/cfn/eks-workshop-ide-cfn.yaml +++ b/lab/cfn/eks-workshop-ide-cfn.yaml @@ -360,6 +360,8 @@ Resources: - !Sub | set -e + export EKS_CLUSTER_NAME="eks-workshop" + export AWS_REGION="${AWS::Region}" export REPOSITORY_OWNER="${RepositoryOwner}" export REPOSITORY_NAME="${RepositoryName}" From 5c4887438262c2807a1ac532606765c0f12c7ed8 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Tue, 3 Sep 2024 17:17:38 -0600 Subject: [PATCH 13/32] Fix terminal copy on firefox --- website/src/components/Terminal/index.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/website/src/components/Terminal/index.tsx b/website/src/components/Terminal/index.tsx index 6e88346f9..4f62c762a 100644 --- a/website/src/components/Terminal/index.tsx +++ b/website/src/components/Terminal/index.tsx @@ -150,13 +150,7 @@ class TerminalSection { } function triggerCopy(text: string) { - navigator.permissions - .query({ name: "clipboard-write" as PermissionName }) - .then((e) => { - if (e.state === "granted") { - navigator.clipboard.writeText(text); - } - }); + navigator.clipboard.writeText(text); window.parent.postMessage(`eks-workshop-terminal:${text}`, "*"); } From d356278ef9951cdb83ac6aa7564706bbc9560b74 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 17:14:45 -0700 Subject: [PATCH 14/32] chore: Auto-approve renovatebot PRs --- .mergify.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 000000000..4208d2b00 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,7 @@ +pull_request_rules: + - name: Automatically approve Renovate PRs + conditions: + - author = renovate[bot] + actions: + review: + type: APPROVE From 76abf90e0beff988383e56383c33ef4ade5afd55 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 20:14:13 -0700 Subject: [PATCH 15/32] Replacing codeowners with mergify --- CODEOWNERS | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS deleted file mode 100644 index 2caf8529d..000000000 --- a/CODEOWNERS +++ /dev/null @@ -1,2 +0,0 @@ -# Require review from core team on all PRs -* @aws-samples/eks-workshop-admin From 15495132aed91ebc30e0937b93e9c932f83f3a77 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 20:18:26 -0700 Subject: [PATCH 16/32] Automerge regex managers --- renovate.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/renovate.json b/renovate.json index caf36b5af..6404773c2 100644 --- a/renovate.json +++ b/renovate.json @@ -28,6 +28,10 @@ "matchDepNames": ["kubernetes/autoscaler"], "extractVersion": "^cluster-autoscaler-(?.*)$", "allowedVersions": "/^1.30.[0-9]+$/" + }, + { + "matchManagers": ["custom.regex"], + "automerge": true } ], "customManagers": [ From 93a5caa3b3fb98a42170ce36d2e28faa804e5e33 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 20:55:39 -0700 Subject: [PATCH 17/32] Migrate automerge to mergify --- .mergify.yml | 12 ++++++++++++ renovate.json | 10 +--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 4208d2b00..1ef4adfe3 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,7 +1,19 @@ +queue_rules: + - name: default + pull_request_rules: - name: Automatically approve Renovate PRs conditions: - author = renovate[bot] + - check-success=Build lab + - check-success=Validate Terraform + - check-success=Pre-commit hooks + - "#check-pending=0" + - "#check-stale=0" + - "#check-failure=0" actions: review: type: APPROVE + queue: + method: squash + name: default diff --git a/renovate.json b/renovate.json index 6404773c2..bc476febc 100644 --- a/renovate.json +++ b/renovate.json @@ -1,10 +1,6 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended", - ":rebaseStalePrs", - ":automergeStableNonMajor" - ], + "extends": ["config:recommended", ":rebaseStalePrs"], "schedule": ["on the first day of the month"], "enabledManagers": ["custom.regex", "github-actions", "terraform"], "packageRules": [ @@ -28,10 +24,6 @@ "matchDepNames": ["kubernetes/autoscaler"], "extractVersion": "^cluster-autoscaler-(?.*)$", "allowedVersions": "/^1.30.[0-9]+$/" - }, - { - "matchManagers": ["custom.regex"], - "automerge": true } ], "customManagers": [ From c6cc2175502d116051a36e7a5bce67a4901bd52e Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 21:01:04 -0700 Subject: [PATCH 18/32] Tweak auto merge status check conditions --- .mergify.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 1ef4adfe3..68e5e19f3 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -8,9 +8,7 @@ pull_request_rules: - check-success=Build lab - check-success=Validate Terraform - check-success=Pre-commit hooks - - "#check-pending=0" - - "#check-stale=0" - - "#check-failure=0" + - check-success=Build website actions: review: type: APPROVE From 16055fa9c73e6fc2f673890b378bdf94e30308ee Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:10:03 +0000 Subject: [PATCH 19/32] chore(deps): update helm release cost-analyzer to v2 (#1003) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .../modules/observability/kubecost/.workshop/terraform/vars.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/modules/observability/kubecost/.workshop/terraform/vars.tf b/manifests/modules/observability/kubecost/.workshop/terraform/vars.tf index d165649b4..7f6c78621 100644 --- a/manifests/modules/observability/kubecost/.workshop/terraform/vars.tf +++ b/manifests/modules/observability/kubecost/.workshop/terraform/vars.tf @@ -38,5 +38,5 @@ variable "kubecost_chart_version" { description = "The chart version of kubecost to use" type = string # renovate-helm: depName=cost-analyzer registryUrl=https://kubecost.github.io/cost-analyzer - default = "1.106.3" + default = "2.3.5" } \ No newline at end of file From a43c1e1729bff511ef956a880d81e3f673d6dfb8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:12:12 +0000 Subject: [PATCH 20/32] chore(deps): update dependency amazon-rds-mysql to v8.0.39 (#1063) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .../.workshop/terraform/preprovision/vars.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/modules/networking/securitygroups-for-pods/.workshop/terraform/preprovision/vars.tf b/manifests/modules/networking/securitygroups-for-pods/.workshop/terraform/preprovision/vars.tf index 065dc763b..f107e7cc3 100644 --- a/manifests/modules/networking/securitygroups-for-pods/.workshop/terraform/preprovision/vars.tf +++ b/manifests/modules/networking/securitygroups-for-pods/.workshop/terraform/preprovision/vars.tf @@ -14,5 +14,5 @@ variable "rds_engine_version" { description = "The MySQL engine version of RDS to use" type = string # renovate: datasource=endoflife-date depName=amazon-rds-mysql - default = "8.0.37" + default = "8.0.39" } \ No newline at end of file From 98bb145e49fe2c81e5f5e3a42276dec4284c4bff Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:13:50 +0000 Subject: [PATCH 21/32] chore(deps): update dependency kubernetes/kubernetes to v1.30.4 (#1074) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- lab/scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/scripts/installer.sh b/lab/scripts/installer.sh index 66b912aca..0ef21f670 100644 --- a/lab/scripts/installer.sh +++ b/lab/scripts/installer.sh @@ -3,7 +3,7 @@ set -e # renovate: depName=kubernetes/kubernetes -kubectl_version='1.30.3' +kubectl_version='1.30.4' # renovate: depName=helm/helm helm_version='3.15.3' From d7fd9dbb564c97f7a7bdc28cf3f2143a3e8d5506 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:15:23 +0000 Subject: [PATCH 22/32] chore(deps): update helm release keda to v2.15.1 (#1077) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../autoscaling/workloads/keda/.workshop/terraform/vars.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/vars.tf b/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/vars.tf index eef00d411..8806936d2 100644 --- a/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/vars.tf +++ b/manifests/modules/autoscaling/workloads/keda/.workshop/terraform/vars.tf @@ -45,5 +45,5 @@ variable "keda_chart_version" { description = "The chart version of keda to use" type = string # renovate-helm: depName=keda registryUrl=https://kedacore.github.io/charts - default = "2.15.0" + default = "2.15.1" } From 58c7e4c79f218d4770f4663c95046a38a2ba1ccd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:17:34 +0000 Subject: [PATCH 23/32] chore(deps): update dependency mikefarah/yq to v4.44.3 (#1075) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- lab/scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/scripts/installer.sh b/lab/scripts/installer.sh index 0ef21f670..a68e836c7 100644 --- a/lab/scripts/installer.sh +++ b/lab/scripts/installer.sh @@ -14,7 +14,7 @@ eksctl_version='0.188.0' kubeseal_version='0.18.4' # renovate: depName=mikefarah/yq -yq_version='4.44.2' +yq_version='4.44.3' # renovate: depName=fluxcd/flux2 flux_version='2.3.0' From e51a051878075b6be0b0a5b8af8ce81066f2432d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:19:07 +0000 Subject: [PATCH 24/32] chore(deps): update helm release aws-load-balancer-controller to v1.8.2 (#1076) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- manifests/modules/exposing/ingress/.workshop/terraform/vars.tf | 2 +- .../modules/exposing/load-balancer/.workshop/terraform/vars.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/modules/exposing/ingress/.workshop/terraform/vars.tf b/manifests/modules/exposing/ingress/.workshop/terraform/vars.tf index 42bd4d060..1d3a23016 100644 --- a/manifests/modules/exposing/ingress/.workshop/terraform/vars.tf +++ b/manifests/modules/exposing/ingress/.workshop/terraform/vars.tf @@ -38,6 +38,6 @@ variable "load_balancer_controller_chart_version" { description = "The chart version of aws-load-balancer-controller to use" type = string # renovate-helm: depName=aws-load-balancer-controller - default = "1.8.1" + default = "1.8.2" } diff --git a/manifests/modules/exposing/load-balancer/.workshop/terraform/vars.tf b/manifests/modules/exposing/load-balancer/.workshop/terraform/vars.tf index 42bd4d060..1d3a23016 100644 --- a/manifests/modules/exposing/load-balancer/.workshop/terraform/vars.tf +++ b/manifests/modules/exposing/load-balancer/.workshop/terraform/vars.tf @@ -38,6 +38,6 @@ variable "load_balancer_controller_chart_version" { description = "The chart version of aws-load-balancer-controller to use" type = string # renovate-helm: depName=aws-load-balancer-controller - default = "1.8.1" + default = "1.8.2" } From b3ec36b2e6f4dd9b9c38d076efb98651273d94d7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:21:12 +0000 Subject: [PATCH 25/32] chore(deps): update dependency aws/karpenter-provider-aws to v0.37.2 (#1064) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf | 2 +- manifests/modules/aiml/inferentia/.workshop/terraform/vars.tf | 2 +- .../autoscaling/compute/karpenter/.workshop/terraform/vars.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf b/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf index e743600c5..8855aef74 100644 --- a/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf +++ b/manifests/modules/aiml/chatbot/.workshop/terraform/vars.tf @@ -38,5 +38,5 @@ variable "karpenter_version" { description = "The version of Karpenter chart to use" type = string # renovate: datasource=github-releases depName=aws/karpenter-provider-aws - default = "0.37.0" + default = "0.37.2" } diff --git a/manifests/modules/aiml/inferentia/.workshop/terraform/vars.tf b/manifests/modules/aiml/inferentia/.workshop/terraform/vars.tf index e743600c5..8855aef74 100644 --- a/manifests/modules/aiml/inferentia/.workshop/terraform/vars.tf +++ b/manifests/modules/aiml/inferentia/.workshop/terraform/vars.tf @@ -38,5 +38,5 @@ variable "karpenter_version" { description = "The version of Karpenter chart to use" type = string # renovate: datasource=github-releases depName=aws/karpenter-provider-aws - default = "0.37.0" + default = "0.37.2" } diff --git a/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/vars.tf b/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/vars.tf index 6df3af0dd..b0d2bb770 100644 --- a/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/vars.tf +++ b/manifests/modules/autoscaling/compute/karpenter/.workshop/terraform/vars.tf @@ -38,5 +38,5 @@ variable "karpenter_version" { description = "The version of Karpenter to use" type = string # renovate: datasource=github-releases depName=aws/karpenter-provider-aws - default = "0.37.0" + default = "0.37.2" } \ No newline at end of file From ab2f5350e6a6e937e85615dffe79c08a671fff8e Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 21:21:09 -0700 Subject: [PATCH 26/32] Reverting mergify auto merge --- .mergify.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 68e5e19f3..4208d2b00 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,17 +1,7 @@ -queue_rules: - - name: default - pull_request_rules: - name: Automatically approve Renovate PRs conditions: - author = renovate[bot] - - check-success=Build lab - - check-success=Validate Terraform - - check-success=Pre-commit hooks - - check-success=Build website actions: review: type: APPROVE - queue: - method: squash - name: default From 2761959cacd2720413721df5b57bc5acbb74f631 Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Thu, 5 Sep 2024 21:26:44 -0700 Subject: [PATCH 27/32] Reinstate renovate auto merge --- renovate.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index bc476febc..caf36b5af 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,10 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:recommended", ":rebaseStalePrs"], + "extends": [ + "config:recommended", + ":rebaseStalePrs", + ":automergeStableNonMajor" + ], "schedule": ["on the first day of the month"], "enabledManagers": ["custom.regex", "github-actions", "terraform"], "packageRules": [ From 6f5dc6bdd9b71cf1588e9d10503b67005f95ad03 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:35:21 +0000 Subject: [PATCH 28/32] chore(deps): update dependency helm/helm to v3.15.4 (#1068) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- lab/scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/scripts/installer.sh b/lab/scripts/installer.sh index a68e836c7..1ffecc20f 100644 --- a/lab/scripts/installer.sh +++ b/lab/scripts/installer.sh @@ -6,7 +6,7 @@ set -e kubectl_version='1.30.4' # renovate: depName=helm/helm -helm_version='3.15.3' +helm_version='3.15.4' # renovate: depName=eksctl-io/eksctl eksctl_version='0.188.0' From 00e0305f0f32e966e99856b13443c73685a1fff6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:49:47 +0000 Subject: [PATCH 29/32] chore(deps): update dependency hashicorp/terraform to v1.9.5 (#1067) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- lab/scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/scripts/installer.sh b/lab/scripts/installer.sh index 1ffecc20f..dc0f4213a 100644 --- a/lab/scripts/installer.sh +++ b/lab/scripts/installer.sh @@ -23,7 +23,7 @@ flux_version='2.3.0' argocd_version='2.11.7' # renovate: depName=hashicorp/terraform -terraform_version='1.9.3' +terraform_version='1.9.5' # renovate: depName=aws/amazon-ec2-instance-selector ec2_instance_selector_version='2.4.1' From 87d31097b891044e02d0d500e1f782445d4d70cb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:54:48 +0000 Subject: [PATCH 30/32] chore(deps): update dependency argoproj/argo-cd to v2.12.3 (#1079) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- lab/scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/scripts/installer.sh b/lab/scripts/installer.sh index dc0f4213a..4b1e31a12 100644 --- a/lab/scripts/installer.sh +++ b/lab/scripts/installer.sh @@ -20,7 +20,7 @@ yq_version='4.44.3' flux_version='2.3.0' # renovate: depName=argoproj/argo-cd -argocd_version='2.11.7' +argocd_version='2.12.3' # renovate: depName=hashicorp/terraform terraform_version='1.9.5' From 51de74bc575d14ae89a0679008e5f2b103fac9cc Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 6 Sep 2024 07:23:45 -0700 Subject: [PATCH 31/32] Expand renovatebot to first week of the month --- renovate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index caf36b5af..288919c35 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":rebaseStalePrs", ":automergeStableNonMajor" ], - "schedule": ["on the first day of the month"], + "schedule": ["first week of the month"], "enabledManagers": ["custom.regex", "github-actions", "terraform"], "packageRules": [ { From 5fd4b470ae5e1a9616801eaeb1fe932a9dc2347f Mon Sep 17 00:00:00 2001 From: Niall Thomson Date: Fri, 6 Sep 2024 08:45:59 -0700 Subject: [PATCH 32/32] Revert schedule change --- renovate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index 288919c35..caf36b5af 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":rebaseStalePrs", ":automergeStableNonMajor" ], - "schedule": ["first week of the month"], + "schedule": ["on the first day of the month"], "enabledManagers": ["custom.regex", "github-actions", "terraform"], "packageRules": [ {