diff --git a/component/toolbox/Dockerfile b/component/toolbox/Dockerfile index 57865c0450..c3ed1552ee 100644 --- a/component/toolbox/Dockerfile +++ b/component/toolbox/Dockerfile @@ -6,7 +6,8 @@ RUN set -eux; \ https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_${arch}/session-manager-plugin.rpm; \ yum install -y jq -COPY ./scripts/* /usr/local/bin/si/ +COPY ./scripts/ /usr/local/bin/si/ + ENV PATH="/usr/local/bin/si:${PATH}" ENTRYPOINT ["bash", "-c"] diff --git a/component/toolbox/scripts/ssm b/component/toolbox/scripts/ssm index 004bf64e57..7198effd0c 100755 --- a/component/toolbox/scripts/ssm +++ b/component/toolbox/scripts/ssm @@ -1,4 +1,23 @@ #!/bin/bash +# --------------------------------------------------------------------------------------------------- +# Lists all the instances in a given region and upon user selection opens an interactive SSM session +# with that instance. +# --------------------------------------------------------------------------------------------------- + +# Stop immediately if anything goes wrong, let's not create too much +# mess if John's shell is poor +set -eo pipefail + +# Find & Import all the supporting functions from the supporting folder +# Get the directory of the current script to figure out where the +# Supporting funcs are +IMPORT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) + +for script in ${IMPORT_DIR}/supporting-funcs/*.sh; do + if [[ -f "$script" ]]; then + source "$script" + fi +done usage() { echo @@ -21,17 +40,6 @@ if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then usage fi -# Function to list EC2 instances with their Name tag -list_instances() { - aws ec2 describe-instances --query 'Reservations[*].Instances[?State.Name==`running`].[Tags[?Key==`Name`].Value | [0],InstanceId,InstanceType,PrivateIpAddress]' --output text -} - -# Function to start SSM session -start_ssm_session() { - instance_id=$1 - aws ssm start-session --target "$instance_id" --document-name AWS-StartInteractiveCommand --parameters command="bash -l" -} - # Parse flags while getopts ":p:r:" opt; do case ${opt} in @@ -52,25 +60,6 @@ while getopts ":p:r:" opt; do esac done -# Function to get input or use environment variable -get_param_or_env() { - local param=$1 - local env_var=$2 - local prompt=$3 - - if [ -z "$param" ]; then - if [ -z "${!env_var}" ]; then - read -p "$prompt: " value - echo "$value" - else - echo "${!env_var}" - fi - else - echo "$param" - fi -} - - # Main script profile=$(get_param_or_env "$profile" "AWS_PROFILE" "Enter the AWS profile to use") region=$(get_param_or_env "$region" "AWS_REGION" "Enter the AWS region (e.g., us-west-2)") @@ -105,6 +94,5 @@ if [ -z "$instance_id" ]; then exit 1 fi -echo "Starting SSM session with instance $instance_id..." -start_ssm_session "$instance_id" - +echo "Starting Interactive SSM session with instance $instance_id..." +start_interactive_ssm_session "$instance_id" diff --git a/component/toolbox/scripts/supporting-funcs/ec2-funcs.sh b/component/toolbox/scripts/supporting-funcs/ec2-funcs.sh new file mode 100644 index 0000000000..9688888275 --- /dev/null +++ b/component/toolbox/scripts/supporting-funcs/ec2-funcs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Function to list EC2 instances with their Name tag, either all or filtered +list_instances() { + filter=$1 + if [[ "${filter,,}" == "all" || -z "${filter}" ]]; then + aws ec2 describe-instances --query 'Reservations[*].Instances[?State.Name==`running`].[Tags[?Key==`Name`].Value | [0],InstanceId,InstanceType,PrivateIpAddress]' --output text | grep -E "sdf|veritech|pinga|rebaser" + elif [[ "${filter,,}" != "all" ]]; then + aws ec2 describe-instances --query 'Reservations[*].Instances[?State.Name==`running`].[Tags[?Key==`Name`].Value | [0],InstanceId,InstanceType,PrivateIpAddress]' --output text | grep -E "${filter}" + fi +} diff --git a/component/toolbox/scripts/supporting-funcs/inputs-funcs.sh b/component/toolbox/scripts/supporting-funcs/inputs-funcs.sh new file mode 100644 index 0000000000..609407d73b --- /dev/null +++ b/component/toolbox/scripts/supporting-funcs/inputs-funcs.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Function to get input or use environment variable +get_param_or_env() { + local param=$1 + local env_var=$2 + local prompt=$3 + + if [ -z "$param" ]; then + if [ -z "${!env_var}" ]; then + read -p "$prompt: " value + echo "$value" + else + echo "${!env_var}" + fi + else + echo "$param" + fi +} + +await_file_results() { + + results_directory=$1 + required_file_count=$2 + + timeout=60 # Timeout in seconds + start_time=$(date +%s) # Record the start time + + while true; do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if (( elapsed_time > timeout )); then + echo "Error: Timeout reached waiting for SSM document responses to arrive. Not all files are present." + exit 1 + fi + + file_count=$(ls "$results_directory" | wc -l) + + if (( file_count >= required_file_count )); then + break + fi + + # Wait for a short period before checking again + sleep 1 + done + +} + +sassy_selection_check() { + selection=${1^^} + if [ "$selection" != "Y" ]; then + echo "Don't Trust Scott and John? We're friends I promise, exiting" + exit 1 + fi +} + +concat_and_output_json() { + + results_directory=$1 + output_file=$2 + + # Check if the directory exists + if [ -d "$results_directory/" ]; then + # Aggregate all the individual json documents into one + cat $results_directory/* | jq -s '.' >> $results_directory/$output_file + cat $results_directory/$output_file | jq + echo "----------------------------------------" + echo "Results can be found within $results_directory" + else + echo "Results Directory $results_directory does not exist." + exit 1 + fi + echo "----------------------------------------" + +} \ No newline at end of file diff --git a/component/toolbox/scripts/supporting-funcs/ssm-funcs.sh b/component/toolbox/scripts/supporting-funcs/ssm-funcs.sh new file mode 100644 index 0000000000..0a14648d09 --- /dev/null +++ b/component/toolbox/scripts/supporting-funcs/ssm-funcs.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Function to start SSM session +start_and_track_ssm_session() { + + instance_id=$1 + script=$2 + service=$3 + action=$4 + results_directory=$5 + + output=$(aws ssm send-command --instance-ids "$instance_id" --document-name "$script" --parameters "Service=$service,InstanceId=$instance_id,Action=$action" 2>&1) + + status=$? + + if [ $status -ne 0 ]; then + output=$(echo "{\"instance_id\": \"$instance_id\", \"status\": \"error\", \"service\": \"$service\", \"message\": \"$output\"}") + echo $output > "$results_directory/$instance_id.json" + return + fi + + command_id=$(echo "$output" | jq -r '.Command.CommandId') + + # Poll for command status with a timeout of 60 seconds + timeout=60 + elapsed=0 + interval=1 + + while [ $elapsed -lt $timeout ]; do + status=$(check_ssm_command_status) + + if [ "$status" == "Success" ] || [ "$status" == "Failed" ] || [ "$status" == "TimedOut" ] || [ "$status" == "Cancelled" ]; then + break + fi + + sleep $interval + elapsed=$((elapsed + interval)) + done + + # Check if command was successful + if [ "$status" == "Success" ]; then + # Get the output + output=$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + | jq -r '.StandardOutputContent') + echo $output > "$results_directory/$instance_id.json" + else + echo "Command failed with status: $status" + exit_code=$(aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + | jq -r '.ResponseCode') + + echo "Exit code: $exit_code" + echo "Failure message:" + aws ssm get-command-invocation \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + | jq -r '.StandardErrorContent' + fi + +} + +# Function to start an interactive SSM session with any given instance +start_interactive_ssm_session() { + instance_id=$1 + aws ssm start-session --target "$instance_id" --document-name AWS-StartInteractiveCommand --parameters command="bash -l" +} + +# Function to check command status +check_ssm_command_status() { + status=$(aws ssm list-command-invocations \ + --command-id "$command_id" \ + --details \ + | jq -r '.CommandInvocations[0].Status') + echo "$status" +} diff --git a/component/toolbox/scripts/upgrade b/component/toolbox/scripts/upgrade index 91fceb2152..da2837a707 100755 --- a/component/toolbox/scripts/upgrade +++ b/component/toolbox/scripts/upgrade @@ -13,6 +13,18 @@ # mess if John's shell is poor set -eo pipefail +# Find & Import all the supporting functions from the supporting folder +# Get the directory of the current script to figure out where the +# Supporting funcs are +IMPORT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) + +for script in ${IMPORT_DIR}/supporting-funcs/*.sh; do + if [[ -f "$script" ]]; then + source "$script" + fi +done + +# Usage for this script usage() { echo echo "upgrade" @@ -25,7 +37,7 @@ usage() { echo "Usage: upgrade [-p profile] [-r region] [-a automatic] [-s service]" echo " -p profile AWS profile to use" echo " -r region AWS region to use" - echo " -s service [sdf/rebaser/pinga/veritech] SI Service to filter by, defaults to all" + echo " -s service [sdf/rebaser/pinga/veritech] SI Service to filter by, defaults to all" echo " -a automatic [Y/N] Run through automatically/no-interact" echo exit 1 @@ -62,162 +74,6 @@ while getopts ":p:r:a:s:" opt; do esac done -# Function to list EC2 instances with their Name tag, either all or filtered -list_instances() { - filter=$1 - if [[ "${filter,,}" == "all" || -z "${filter}" ]]; then - aws ec2 describe-instances --query 'Reservations[*].Instances[?State.Name==`running`].[Tags[?Key==`Name`].Value | [0],InstanceId,InstanceType,PrivateIpAddress]' --output text | grep -E "sdf|veritech|pinga|rebaser" - elif [[ "${filter,,}" != "all" ]]; then - aws ec2 describe-instances --query 'Reservations[*].Instances[?State.Name==`running`].[Tags[?Key==`Name`].Value | [0],InstanceId,InstanceType,PrivateIpAddress]' --output text | grep -E "${filter}" - fi -} - -# Function to check command status -check_ssm_command_status() { - status=$(aws ssm list-command-invocations \ - --command-id "$command_id" \ - --details \ - | jq -r '.CommandInvocations[0].Status') - echo "$status" -} - -# Function to start SSM session -start_and_track_ssm_session() { - - instance_id=$1 - script=$2 - service=$3 - action=$4 - results_directory=$5 - - output=$(aws ssm send-command --instance-ids "$instance_id" --document-name "$script" --parameters "Service=$service,InstanceId=$instance_id,Action=$action" 2>&1) - - status=$? - - if [ $status -ne 0 ]; then - output=$(echo "{\"instance_id\": \"$instance_id\", \"status\": \"error\", \"service\": \"$service\", \"message\": \"$output\"}") - echo $output > "$results_directory/$instance_id.json" - return - fi - - command_id=$(echo "$output" | jq -r '.Command.CommandId') - - # Poll for command status with a timeout of 60 seconds - timeout=60 - elapsed=0 - interval=1 - - while [ $elapsed -lt $timeout ]; do - status=$(check_ssm_command_status) - - if [ "$status" == "Success" ] || [ "$status" == "Failed" ] || [ "$status" == "TimedOut" ] || [ "$status" == "Cancelled" ]; then - break - fi - - sleep $interval - elapsed=$((elapsed + interval)) - done - - # Check if command was successful - if [ "$status" == "Success" ]; then - # Get the output - output=$(aws ssm get-command-invocation \ - --command-id "$command_id" \ - --instance-id "$instance_id" \ - | jq -r '.StandardOutputContent') - echo $output > "$results_directory/$instance_id.json" - else - echo "Command failed with status: $status" - exit_code=$(aws ssm get-command-invocation \ - --command-id "$command_id" \ - --instance-id "$instance_id" \ - | jq -r '.ResponseCode') - - echo "Exit code: $exit_code" - echo "Failure message:" - aws ssm get-command-invocation \ - --command-id "$command_id" \ - --instance-id "$instance_id" \ - | jq -r '.StandardErrorContent' - fi - - } - -# Function to get input or use environment variable -get_param_or_env() { - local param=$1 - local env_var=$2 - local prompt=$3 - - if [ -z "$param" ]; then - if [ -z "${!env_var}" ]; then - read -p "$prompt: " value - echo "$value" - else - echo "${!env_var}" - fi - else - echo "$param" - fi -} - -await_ssm_results() { - - results_directory=$1 - required_file_count=$2 - - timeout=60 # Timeout in seconds - start_time=$(date +%s) # Record the start time - - while true; do - current_time=$(date +%s) - elapsed_time=$((current_time - start_time)) - - if (( elapsed_time > timeout )); then - echo "Error: Timeout reached waiting for SSM document responses to arrive. Not all files are present." - exit 1 - fi - - file_count=$(ls "$results_directory" | wc -l) - - if (( file_count >= required_file_count )); then - break - fi - - # Wait for a short period before checking again - sleep 1 - done - -} - -sassy_selection_check() { - selection=${1^^} - if [ "$selection" != "Y" ]; then - echo "Don't Trust Scott and John? We're friends I promise, exiting" - exit 1 - fi -} - -concat_and_output() { - - results_directory=$1 - output_file=$2 - - # Check if the directory exists - if [ -d "$results_directory/" ]; then - # Aggregate all the individual json documents into one - cat $results_directory/* | jq -s '.' >> $results_directory/$output_file - cat $results_directory/$output_file | jq - echo "----------------------------------------" - echo "Results can be found within $results_directory" - else - echo "Results Directory $results_directory does not exist." - exit 1 - fi - echo "----------------------------------------" - -} - # --------------------------------------------------------------------------------------------------- # Main script # --------------------------------------------------------------------------------------------------- @@ -273,16 +129,16 @@ while read -r line; do ((i++)) done <<< "$instances" -await_ssm_results "$results_directory" $((i - 1)) +await_file_results "$results_directory" $((i - 1)) -concat_and_output "$results_directory" "$check_results_file" +concat_and_output_json "$results_directory" "$check_results_file" if jq -e 'all(.[]; .status == "success") and any(.[]; .upgradeable == "true")' "$results_directory/$check_results_file" > /dev/null; then [[ "${automatic,,}" == "y" ]] || read -p "Would you like to push the new binaries out to the upgradeable hosts? (Y/N) " selection [[ "${automatic,,}" == "y" ]] || sassy_selection_check $selection else echo "Error: Either none are upgradeable or one or more of the checks failed to determine whether it was possible to upgrade the node." - exit 1 + exit 2 fi # For all order 1 services, upgrade them in *sequence* @@ -302,7 +158,7 @@ jq 'map(select(.service == "veritech")) | .[]' <<< $upgrade_candidates_json | jq done # Wait until all the results arrive -await_ssm_results "$results_directory" "$upgrade_hosts_num" +await_file_results "$results_directory" "$upgrade_hosts_num" # Continue with the rest of the service nodes upgrade_hosts_num=$(jq 'map(select(.service != "veritech")) | .[]' <<< $upgrade_candidates_json | jq -c '.' | wc -l) @@ -315,8 +171,8 @@ done # Concatenate all the results together upgrade_hosts_num=$(jq '.[]' <<< $upgrade_candidates_json | jq -c '.' | wc -l) -await_ssm_results "$results_directory" "$upgrade_hosts_num" -concat_and_output "$results_directory" "$upgrade_results_file" +await_file_results "$results_directory" "$upgrade_hosts_num" +concat_and_output_json "$results_directory" "$upgrade_results_file" echo "All active binary services have been rotated, WEB IS STILL TO BE UPDATED." echo "If you are running in CI, this will be ran automatically for you. If you"