diff --git a/Makefile b/Makefile index 43dd76ca0..bd8e80d15 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ PACKER_BINARY ?= packer -PACKER_VARIABLES := aws_region ami_name binary_bucket_name binary_bucket_region kubernetes_version kubernetes_build_date kernel_version docker_version containerd_version runc_version cni_plugin_version source_ami_id source_ami_owners source_ami_filter_name arch instance_type security_group_id additional_yum_repos pull_cni_from_github sonobuoy_e2e_registry ami_regions volume_type +PACKER_VARIABLES := $(shell $(PACKER_BINARY) inspect -machine-readable eks-worker-al2.json | grep 'template-variable' | awk -F ',' '{print $$4}') K8S_VERSION_PARTS := $(subst ., ,$(kubernetes_version)) K8S_VERSION_MINOR := $(word 1,${K8S_VERSION_PARTS}).$(word 2,${K8S_VERSION_PARTS}) diff --git a/README.md b/README.md index 21c4b6641..1158ef702 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,104 @@ Provisioner](https://www.packer.io/docs/provisioners/shell.html) runs the necessary configuration tasks. Then, Packer creates an AMI from the instance and terminates the instance after the AMI is created. +### Container Image Caching + +Optionally, some container images can be cached during the AMI build process in order to reduce the latency of the node getting to a `Ready` state when launched. + +To turn on container image caching: + +``` +cache_container_images=true make 1.23 +``` + +When container image caching is enabled, the following images are cached: + - 602401143452.dkr.ecr..amazonaws.com/eks/kube-proxy:-eksbuild. + - 602401143452.dkr.ecr..amazonaws.com/eks/kube-proxy:-minimal-eksbuild. + - 602401143452.dkr.ecr..amazonaws.com/eks/pause:3.5 + - 602401143452.dkr.ecr..amazonaws.com/amazon-k8s-cni-init: + - 602401143452.dkr.ecr..amazonaws.com/amazon-k8s-cni: + +The account ID can be different depending on the region and partition you are building the AMI in. See [here](https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html) for more details. + +Since the VPC CNI is not versioned with K8s itself, the latest version of the VPC CNI and the default version, based on the response from the EKS DescribeAddonVersions at the time of the AMI build, will be cached. + +The images listed above are also tagged with each region in the partition the AMI is built in, since images are often built in one region and copied to others within the same partition. Images that are available to pull from an ECR FIPS endpoint are also tagged as such (i.e. `602401143452.dkr.ecr-fips.us-east-1.amazonaws.com/eks/pause:3.5`). + +When listing images on a node, you'll notice a long list of images. However, most of these images are simply tagged in different ways with no storage overhead. Images cached in the AMI total around 1.0 GiB. In general, a node with no images cached using the VPC CNI will use around 500 MiB of images when in a `Ready` state with no other pods running on the node. + +### IAM Permissions + +To build the EKS Optimized AMI, you will need the following permissions: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CopyImage", + "ec2:CreateImage", + "ec2:CreateKeypair", + "ec2:CreateSecurityGroup", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteKeyPair", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSnapshot", + "ec2:DeleteVolume", + "ec2:DeregisterImage", + "ec2:DescribeImageAttribute", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeRegions", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSnapshots", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DetachVolume", + "ec2:GetPasswordData", + "ec2:ModifyImageAttribute", + "ec2:ModifyInstanceAttribute", + "ec2:ModifySnapshotAttribute", + "ec2:RegisterImage", + "ec2:RunInstances", + "ec2:StopInstances", + "ec2:TerminateInstances", + "eks:DescribeAddonVersions", + "ecr:GetAuthorizationToken" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer" + ], + "Resource": "arn:aws:ecr:us-west-2:602401143452:repository/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::amazon-eks/*" + } + ] +} +``` + +You will need to use the region you are building the AMI in to specify the ECR repository resource in the second IAM statement. You may also need to change the account if you are building the AMI in a different partition or special region. You can see a mapping of regions to account ID [here](https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html). +If you're using a custom s3 bucket to vend different K8s binaries, you will need to change the resource in the third IAM statement above to reference your custom bucket. +For more information about the permissions required by Packer with different configurations, see the [docs](https://www.packer.io/plugins/builders/amazon#iam-task-or-instance-role). + ## Using the AMI If you are just getting started with Amazon EKS, we recommend that you follow diff --git a/eks-worker-al2.json b/eks-worker-al2.json index dda34dd39..6d3278cb0 100644 --- a/eks-worker-al2.json +++ b/eks-worker-al2.json @@ -13,6 +13,7 @@ "aws_session_token": "{{env `AWS_SESSION_TOKEN`}}", "binary_bucket_name": "amazon-eks", "binary_bucket_region": "us-west-2", + "cache_container_images": "false", "cni_plugin_version": "v0.8.6", "containerd_version": "1.6.6-1.amzn2.0.2", "creator": "{{env `USER`}}", @@ -23,7 +24,8 @@ "kms_key_id": "", "kubernetes_build_date": null, "kubernetes_version": null, - "launch_block_device_mappings_volume_size": "4", + "launch_block_device_mappings_volume_size": "8", + "pause_container_version": "3.5", "pull_cni_from_github": "true", "remote_folder": "", "runc_version": "1.1.3-1.amzn2.0.2", @@ -161,7 +163,10 @@ "AWS_ACCESS_KEY_ID={{user `aws_access_key_id`}}", "AWS_SECRET_ACCESS_KEY={{user `aws_secret_access_key`}}", "AWS_SESSION_TOKEN={{user `aws_session_token`}}", - "SONOBUOY_E2E_REGISTRY={{user `sonobuoy_e2e_registry`}}" + "SONOBUOY_E2E_REGISTRY={{user `sonobuoy_e2e_registry`}}", + "PAUSE_CONTAINER_VERSION={{user `pause_container_version`}}", + "KUBE_PROXY_VERSION_SUFFIX={{user `kube_proxy_version_suffix`}}", + "CACHE_CONTAINER_IMAGES={{user `cache_container_images`}}" ] }, { diff --git a/files/bootstrap.sh b/files/bootstrap.sh index 76228fa63..78e9d4fab 100755 --- a/files/bootstrap.sh +++ b/files/bootstrap.sh @@ -178,51 +178,6 @@ SERVICE_IPV6_CIDR="${SERVICE_IPV6_CIDR:-}" ENABLE_LOCAL_OUTPOST="${ENABLE_LOCAL_OUTPOST:-}" CLUSTER_ID="${CLUSTER_ID:-}" -function get_pause_container_account_for_region() { - local region="$1" - case "${region}" in - ap-east-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-800184023465}" - ;; - me-south-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-558608220178}" - ;; - cn-north-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-918309763551}" - ;; - cn-northwest-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-961992271922}" - ;; - us-gov-west-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-013241004608}" - ;; - us-gov-east-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-151742754352}" - ;; - us-iso-east-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-725322719131}" - ;; - us-isob-east-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-187977181151}" - ;; - af-south-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-877085696533}" - ;; - eu-south-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-590381155156}" - ;; - ap-southeast-3) - echo "${PAUSE_CONTAINER_ACCOUNT:-296578399912}" - ;; - me-central-1) - echo "${PAUSE_CONTAINER_ACCOUNT:-759879836304}" - ;; - *) - echo "${PAUSE_CONTAINER_ACCOUNT:-602401143452}" - ;; - esac -} - # Helper function which calculates the amount of the given resource (either CPU or memory) # to reserve in a given resource range, specified by a start and end of the range and a percentage # of the resource to reserve. Note that we return zero if the start of the resource range is @@ -314,8 +269,8 @@ if [[ "$MACHINE" != "x86_64" && "$MACHINE" != "aarch64" ]]; then exit 1 fi -PAUSE_CONTAINER_ACCOUNT=$(get_pause_container_account_for_region "${AWS_DEFAULT_REGION}") -PAUSE_CONTAINER_IMAGE=${PAUSE_CONTAINER_IMAGE:-$PAUSE_CONTAINER_ACCOUNT.dkr.ecr.$AWS_DEFAULT_REGION.$AWS_SERVICES_DOMAIN/eks/pause} +ECR_URI=$(/etc/eks/get-ecr-uri.sh "${AWS_DEFAULT_REGION}" "${AWS_SERVICES_DOMAIN}" "${PAUSE_CONTAINER_ACCOUNT:-}") +PAUSE_CONTAINER_IMAGE=${PAUSE_CONTAINER_IMAGE:-$ECR_URI/eks/pause} PAUSE_CONTAINER="$PAUSE_CONTAINER_IMAGE:$PAUSE_CONTAINER_VERSION" ### kubelet kubeconfig @@ -525,29 +480,26 @@ if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then sudo mkdir -p /etc/containerd sudo mkdir -p /etc/cni/net.d - mkdir -p /etc/systemd/system/containerd.service.d - cat << EOF > /etc/systemd/system/containerd.service.d/10-compat-symlink.conf -[Service] -ExecStartPre=/bin/ln -sf /run/containerd/containerd.sock /run/dockershim.sock -EOF if [[ -n "$CONTAINERD_CONFIG_FILE" ]]; then sudo cp -v $CONTAINERD_CONFIG_FILE /etc/eks/containerd/containerd-config.toml fi echo "$(jq '.cgroupDriver="systemd"' $KUBELET_CONFIG)" > $KUBELET_CONFIG sudo sed -i s,SANDBOX_IMAGE,$PAUSE_CONTAINER,g /etc/eks/containerd/containerd-config.toml - sudo cp -v /etc/eks/containerd/containerd-config.toml /etc/containerd/config.toml - sudo cp -v /etc/eks/containerd/sandbox-image.service /etc/systemd/system/sandbox-image.service + + # Check if the containerd config file is the same as the one used in the image build. + # If different, then restart containerd w/ proper config + if ! cmp -s /etc/eks/containerd/containerd-config.toml /etc/containerd/config.toml; then + sudo cp -v /etc/eks/containerd/containerd-config.toml /etc/containerd/config.toml + sudo cp -v /etc/eks/containerd/sandbox-image.service /etc/systemd/system/sandbox-image.service + sudo chown root:root /etc/systemd/system/sandbox-image.service + systemctl daemon-reload + systemctl enable containerd sandbox-image + systemctl restart sandbox-image containerd + fi sudo cp -v /etc/eks/containerd/kubelet-containerd.service /etc/systemd/system/kubelet.service sudo chown root:root /etc/systemd/system/kubelet.service - sudo chown root:root /etc/systemd/system/sandbox-image.service # Validate containerd config sudo containerd config dump > /dev/null - systemctl daemon-reload - systemctl enable containerd - systemctl restart containerd - systemctl enable sandbox-image - systemctl start sandbox-image - elif [[ "$CONTAINER_RUNTIME" = "dockerd" ]]; then mkdir -p /etc/docker bash -c "/sbin/iptables-save > /etc/sysconfig/iptables" diff --git a/files/get-ecr-uri.sh b/files/get-ecr-uri.sh new file mode 100755 index 000000000..f5e87b932 --- /dev/null +++ b/files/get-ecr-uri.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +# More details about the mappings in this file can be found here https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html + +region=$1 +aws_domain=$2 +if [[ $# -eq 3 ]] && [[ ! -z $3 ]]; then + acct=$3 +else + case "${region}" in + ap-east-1) + acct="800184023465" + ;; + me-south-1) + acct="558608220178" + ;; + cn-north-1) + acct="918309763551" + ;; + cn-northwest-1) + acct="961992271922" + ;; + us-gov-west-1) + acct="013241004608" + ;; + us-gov-east-1) + acct="151742754352" + ;; + us-iso-east-1) + acct="725322719131" + ;; + us-isob-east-1) + acct="187977181151" + ;; + af-south-1) + acct="877085696533" + ;; + eu-south-1) + acct="590381155156" + ;; + ap-southeast-3) + acct="296578399912" + ;; + me-central-1) + acct="759879836304" + ;; + *) + acct="602401143452" + ;; + esac +fi + +echo "${acct}.dkr.ecr.${region}.${aws_domain}" diff --git a/files/pull-image.sh b/files/pull-image.sh new file mode 100755 index 000000000..2d37d88b9 --- /dev/null +++ b/files/pull-image.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +img=$1 +region=$(echo "${img}" | cut -f4 -d ".") +MAX_RETRIES=3 + +function retry() { + local rc=0 + for attempt in $(seq 0 $MAX_RETRIES); do + rc=0 + [[ $attempt -gt 0 ]] && echo "Attempt $attempt of $MAX_RETRIES" 1>&2 + "$@" + rc=$? + [[ $rc -eq 0 ]] && break + [[ $attempt -eq $MAX_RETRIES ]] && exit $rc + local jitter=$((1 + RANDOM % 10)) + local sleep_sec="$(($((5 << $((1 + $attempt)))) + $jitter))" + sleep $sleep_sec + done +} + +ecr_password=$(retry aws ecr get-login-password --region $region) +if [[ -z ${ecr_password} ]]; then + echo >&2 "Unable to retrieve the ECR password." + exit 1 +fi +retry sudo ctr --namespace k8s.io image pull "${img}" --user AWS:${ecr_password} diff --git a/files/pull-sandbox-image.sh b/files/pull-sandbox-image.sh index 270be7d32..523e7ed4c 100644 --- a/files/pull-sandbox-image.sh +++ b/files/pull-sandbox-image.sh @@ -1,27 +1,5 @@ #!/usr/bin/env bash +set -euo pipefail -### fetching sandbox image from /etc/containerd/config.toml -sandbox_image=$(awk -F'[ ="]+' '$1 == "sandbox_image" { print $2 }' /etc/containerd/config.toml) -region=$(echo "$sandbox_image" | cut -f4 -d ".") -ecr_password=$(aws ecr get-login-password --region $region) -API_RETRY_ATTEMPTS=5 - -for attempt in $(seq 0 $API_RETRY_ATTEMPTS); do - rc=0 - if [[ $attempt -gt 0 ]]; then - echo "Attempt $attempt of $API_RETRY_ATTEMPTS" - fi - ### pull sandbox image from ecr - ### username will always be constant i.e; AWS - sudo ctr --namespace k8s.io image pull $sandbox_image --user AWS:$ecr_password - rc=$? - if [[ $rc -eq 0 ]]; then - break - fi - if [[ $attempt -eq $API_RETRY_ATTEMPTS ]]; then - exit $rc - fi - jitter=$((1 + RANDOM % 10)) - sleep_sec="$(($((5 << $((1 + $attempt)))) + $jitter))" - sleep $sleep_sec -done +sandbox_image="$(awk -F'[ ="]+' '$1 == "sandbox_image" { print $2 }' /etc/containerd/config.toml)" +/etc/eks/containerd/pull-image.sh "${sandbox_image}" diff --git a/scripts/install-worker.sh b/scripts/install-worker.sh index 6c3fa305a..154e6dfa3 100644 --- a/scripts/install-worker.sh +++ b/scripts/install-worker.sh @@ -4,6 +4,7 @@ set -o pipefail set -o nounset set -o errexit IFS=$'\n\t' +export AWS_DEFAULT_OUTPUT="json" TEMPLATE_DIR=${TEMPLATE_DIR:-/tmp/worker} @@ -30,6 +31,8 @@ validate_env_set CNI_PLUGIN_VERSION validate_env_set KUBERNETES_VERSION validate_env_set KUBERNETES_BUILD_DATE validate_env_set PULL_CNI_FROM_GITHUB +validate_env_set PAUSE_CONTAINER_VERSION +validate_env_set CACHE_CONTAINER_IMAGES ################################################################################ ### Machine Architecture ####################################################### @@ -138,46 +141,18 @@ else sudo yum install -y awscli fi -################################################################################ -### Docker ##################################################################### -################################################################################ - -sudo yum install -y device-mapper-persistent-data lvm2 - -INSTALL_DOCKER="${INSTALL_DOCKER:-true}" -if [[ "$INSTALL_DOCKER" == "true" ]]; then - sudo amazon-linux-extras enable docker - sudo groupadd -og 1950 docker - sudo useradd --gid $(getent group docker | cut -d: -f3) docker - - # install runc and lock version - sudo yum install -y runc-${RUNC_VERSION} - sudo yum versionlock runc-* - - # install containerd and lock version - sudo yum install -y containerd-${CONTAINERD_VERSION} - sudo yum versionlock containerd-* - - # install docker and lock version - sudo yum install -y docker-${DOCKER_VERSION}* - sudo yum versionlock docker-* - sudo usermod -aG docker $USER - - # Remove all options from sysconfig docker. - sudo sed -i '/OPTIONS/d' /etc/sysconfig/docker - - sudo mkdir -p /etc/docker - sudo mv $TEMPLATE_DIR/docker-daemon.json /etc/docker/daemon.json - sudo chown root:root /etc/docker/daemon.json - - # Enable docker daemon to start on boot. - sudo systemctl daemon-reload -fi - ############################################################################### ### Containerd setup ########################################################## ############################################################################### +# install runc and lock version +sudo yum install -y runc-${RUNC_VERSION} +sudo yum versionlock runc-* + +# install containerd and lock version +sudo yum install -y containerd-${CONTAINERD_VERSION} +sudo yum versionlock containerd-* + sudo mkdir -p /etc/eks/containerd if [ -f "/etc/eks/containerd/containerd-config.toml" ]; then ## this means we are building a gpu ami and have already placed a containerd configuration file in /etc/eks @@ -195,7 +170,15 @@ fi sudo mv $TEMPLATE_DIR/kubelet-containerd.service /etc/eks/containerd/kubelet-containerd.service sudo mv $TEMPLATE_DIR/sandbox-image.service /etc/eks/containerd/sandbox-image.service sudo mv $TEMPLATE_DIR/pull-sandbox-image.sh /etc/eks/containerd/pull-sandbox-image.sh +sudo mv $TEMPLATE_DIR/pull-image.sh /etc/eks/containerd/pull-image.sh sudo chmod +x /etc/eks/containerd/pull-sandbox-image.sh +sudo chmod +x /etc/eks/containerd/pull-image.sh + +sudo mkdir -p /etc/systemd/system/containerd.service.d +cat << EOF | sudo tee /etc/systemd/system/containerd.service.d/10-compat-symlink.conf +[Service] +ExecStartPre=/bin/ln -sf /run/containerd/containerd.sock /run/dockershim.sock +EOF cat << EOF | sudo tee -a /etc/modules-load.d/containerd.conf overlay @@ -208,6 +191,34 @@ net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 EOF +################################################################################ +### Docker ##################################################################### +################################################################################ + +sudo yum install -y device-mapper-persistent-data lvm2 + +INSTALL_DOCKER="${INSTALL_DOCKER:-true}" +if [[ "$INSTALL_DOCKER" == "true" ]]; then + sudo amazon-linux-extras enable docker + sudo groupadd -og 1950 docker + sudo useradd --gid $(getent group docker | cut -d: -f3) docker + + # install docker and lock version + sudo yum install -y docker-${DOCKER_VERSION}* + sudo yum versionlock docker-* + sudo usermod -aG docker $USER + + # Remove all options from sysconfig docker. + sudo sed -i '/OPTIONS/d' /etc/sysconfig/docker + + sudo mkdir -p /etc/docker + sudo mv $TEMPLATE_DIR/docker-daemon.json /etc/docker/daemon.json + sudo chown root:root /etc/docker/daemon.json + + # Enable docker daemon to start on boot. + sudo systemctl daemon-reload +fi + ################################################################################ ### Logrotate ################################################################## ################################################################################ @@ -331,6 +342,8 @@ sudo systemctl disable kubelet ################################################################################ sudo mkdir -p /etc/eks +sudo mv $TEMPLATE_DIR/get-ecr-uri.sh /etc/eks/get-ecr-uri.sh +sudo chmod +x /etc/eks/get-ecr-uri.sh sudo mv $TEMPLATE_DIR/eni-max-pods.txt /etc/eks/eni-max-pods.txt sudo mv $TEMPLATE_DIR/bootstrap.sh /etc/eks/bootstrap.sh sudo chmod +x /etc/eks/bootstrap.sh @@ -363,6 +376,92 @@ if vercmp "$KUBERNETES_VERSION" gteq "1.22.0"; then sudo mv $TEMPLATE_DIR/ecr-credential-provider-config /etc/eks/ecr-credential-provider/ecr-credential-provider-config fi +################################################################################ +### Cache Images ############################################################### +################################################################################ +if [[ "$CACHE_CONTAINER_IMAGES" == "true" && "$BINARY_BUCKET_REGION" != "us-iso-east-1" && "$BINARY_BUCKET_REGION" != "us-isob-east-1" ]]; then + AWS_DOMAIN=$(imds 'latest/meta-data/services/domain') + ECR_URI=$(/etc/eks/get-ecr-uri.sh "${BINARY_BUCKET_REGION}" "${AWS_DOMAIN}") + + PAUSE_CONTAINER="${ECR_URI}/eks/pause:${PAUSE_CONTAINER_VERSION}" + cat /etc/eks/containerd/containerd-config.toml | sed s,SANDBOX_IMAGE,$PAUSE_CONTAINER,g | sudo tee /etc/eks/containerd/containerd-cached-pause-config.toml + sudo cp -v /etc/eks/containerd/containerd-cached-pause-config.toml /etc/containerd/config.toml + sudo cp -v /etc/eks/containerd/sandbox-image.service /etc/systemd/system/sandbox-image.service + sudo chown root:root /etc/systemd/system/sandbox-image.service + sudo systemctl daemon-reload + sudo systemctl start containerd + sudo systemctl enable containerd sandbox-image + + K8S_MINOR_VERSION=$(echo "${KUBERNETES_VERSION}" | cut -d'.' -f1-2) + KUBE_PROXY_ADDON_VERSIONS=$(aws eks describe-addon-versions --addon-name kube-proxy --kubernetes-version=${K8S_MINOR_VERSION}) + + DEFAULT_KUBE_PROXY_FULL_VERSION=$(echo "${KUBE_PROXY_ADDON_VERSIONS}" | jq -r '.addons[] .addonVersions[] | select(.compatibilities[] .defaultVersion==true).addonVersion') + DEFAULT_KUBE_PROXY_VERSION=$(echo "${DEFAULT_KUBE_PROXY_FULL_VERSION}" | cut -d"-" -f1) + DEFAULT_KUBE_PROXY_PLATFORM_VERSION=$(echo "${DEFAULT_KUBE_PROXY_FULL_VERSION}" | cut -d"-" -f2) + + LATEST_KUBE_PROXY_FULL_VERSION=$(echo "${KUBE_PROXY_ADDON_VERSIONS}" | jq -r '.addons[] .addonVersions[] .addonVersion' | sort -V | tail -n1) + LATEST_KUBE_PROXY_VERSION=$(echo "${LATEST_KUBE_PROXY_FULL_VERSION}" | cut -d"-" -f1) + LATEST_KUBE_PROXY_PLATFORM_VERSION=$(echo "${LATEST_KUBE_PROXY_FULL_VERSION}" | cut -d"-" -f2) + + KUBE_PROXY_IMGS=( + ## Default kube-proxy images + "${ECR_URI}/eks/kube-proxy:${DEFAULT_KUBE_PROXY_VERSION}-${DEFAULT_KUBE_PROXY_PLATFORM_VERSION}" + "${ECR_URI}/eks/kube-proxy:${DEFAULT_KUBE_PROXY_VERSION}-minimal-${DEFAULT_KUBE_PROXY_PLATFORM_VERSION}" + + ## Latest kube-proxy images + "${ECR_URI}/eks/kube-proxy:${LATEST_KUBE_PROXY_VERSION}-${LATEST_KUBE_PROXY_PLATFORM_VERSION}" + "${ECR_URI}/eks/kube-proxy:${LATEST_KUBE_PROXY_VERSION}-minimal-${LATEST_KUBE_PROXY_PLATFORM_VERSION}" + ) + + #### Cache VPC CNI images starting with the addon default version and the latest version + VPC_CNI_ADDON_VERSIONS=$(aws eks describe-addon-versions --addon-name vpc-cni --kubernetes-version=${K8S_MINOR_VERSION}) + DEFAULT_VPC_CNI_VERSION=$(echo "${VPC_CNI_ADDON_VERSIONS}" | jq -r '.addons[] .addonVersions[] | select(.compatibilities[] .defaultVersion==true).addonVersion') + LATEST_VPC_CNI_VERSION=$(echo "${VPC_CNI_ADDON_VERSIONS}" | jq -r '.addons[] .addonVersions[] .addonVersion' | sort -V | tail -n1) + CNI_IMG="${ECR_URI}/amazon-k8s-cni" + CNI_INIT_IMG="${CNI_IMG}-init" + CNI_IMGS=( + ## Default VPC CNI Images + "${CNI_IMG}:${DEFAULT_VPC_CNI_VERSION}" + "${CNI_INIT_IMG}:${DEFAULT_VPC_CNI_VERSION}" + + ## Latest VPC CNI Images + "${CNI_IMG}:${LATEST_VPC_CNI_VERSION}" + "${CNI_INIT_IMG}:${LATEST_VPC_CNI_VERSION}" + ) + + CACHED_IMGS=( + "${PAUSE_CONTAINER}" + ${KUBE_PROXY_IMGS[@]} + ${CNI_IMGS[@]} + ) + + for img in "${CACHED_IMGS[@]}"; do + ## only kube-proxy-minimal is vended for K8s 1.24+ + if [[ "${img}" == *"kube-proxy:"* ]] && [[ "${img}" != *"-minimal-"* ]] && vercmp "${K8S_MINOR_VERSION}" gteq "1.24"; then + continue + fi + /etc/eks/containerd/pull-image.sh "${img}" + done + + #### Tag the pulled down image for all other regions in the partition + for region in $(aws ec2 describe-regions --all-regions | jq -r '.Regions[] .RegionName'); do + for img in "${CACHED_IMGS[@]}"; do + regional_img="${img/$BINARY_BUCKET_REGION/$region}" + sudo ctr -n k8s.io image tag "${img}" "${regional_img}" || : + ## Tag ECR fips endpoint for supported regions + if [[ "${region}" =~ (us-east-1|us-east-2|us-west-1|us-west-2|us-gov-east-1|us-gov-east-2) ]]; then + regional_fips_img="${regional_img/.ecr./.ecr-fips.}" + sudo ctr -n k8s.io image tag "${img}" "${regional_fips_img}" || : + sudo ctr -n k8s.io image tag "${img}" "${regional_fips_img/-eksbuild.1/}" || : + fi + ## Cache the non-addon VPC CNI images since "v*.*.*-eksbuild.1" is equivalent to leaving off the eksbuild suffix + if [[ "${img}" == *"-cni"*"-eksbuild.1" ]]; then + sudo ctr -n k8s.io image tag "${img}" "${regional_img/-eksbuild.1/}" || : + fi + done + done +fi + ################################################################################ ### SSM Agent ################################################################## ################################################################################