-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(fis): disk stress experiment (#54)
* feat(fis): disk stress experiment * feat(fis): cloudwatch agent install automation * fix(fis): changed disk stress target * fix(fis): adjusted waiting time between ssm commands
- Loading branch information
Showing
7 changed files
with
185 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| #!/bin/bash | ||
| aws ssm create-association --name AWS-ConfigureAWSPackage \ | ||
| --parameters 'action=Install,name=AmazonCloudWatchAgent' \ | ||
| --targets 'Key=tag:release,Values=baseline,canary' \ | ||
| --region ${region} --output text | ||
|
|
||
| sleep 30 | ||
|
|
||
| aws ssm create-association --name AmazonCloudWatch-ManageAgent \ | ||
| --parameters 'action=start' \ | ||
| --targets 'Key=tag:release,Values=baseline,canary' \ | ||
| --region ${region} --output text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| { | ||
| "tags": { | ||
| "Name": "DiskStress" | ||
| }, | ||
| "description": "Run a Disk fault injection on the specified instance", | ||
| "targets": { | ||
| "ec2-instances": { | ||
| "resourceType": "aws:ec2:instance", | ||
| "resourceTags": { | ||
| "env": "prod", | ||
| "release": "canary" | ||
| }, | ||
| "filters": [ | ||
| { | ||
| "path": "State.Name", | ||
| "values": ["running"] | ||
| } | ||
| ], | ||
| "selectionMode": "COUNT(1)" | ||
| } | ||
| }, | ||
| "actions": { | ||
| "DiskStress": { | ||
| "actionId": "aws:ssm:send-command", | ||
| "description": "run disk stress using ssm", | ||
| "parameters": { | ||
| "duration": "PT1M", | ||
| "documentArn": "${doc_arn}", | ||
| "documentParameters": "{\"DurationSeconds\": \"60\", \"Workers\": \"4\", \"Percent\": \"70\", \"InstallDependencies\": \"True\"}" | ||
| }, | ||
| "targets": { | ||
| "Instances": "ec2-instances" | ||
| } | ||
| } | ||
| }, | ||
| "stopConditions": [ | ||
| { | ||
| "source": "aws:cloudwatch:alarm", | ||
| "value": "${alarm}" | ||
| } | ||
| ], | ||
| "roleArn": "${role}" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| --- | ||
| description: | | ||
| ### Document name - FIS-Run-Disk-Stress | ||
| ## What does this document do? | ||
| It runs disk stress on an instance via stress-ng tool. | ||
| ## Input Parameters | ||
| * DurationSeconds: (Required) The duration - in seconds - of the disk stress. | ||
| * Workers: The number of virtual disk stressors (default: 1). | ||
| * Percent: The percentage of virtual disk to use (required). | ||
| * InstallDependencies: If set to True, Systems Manager installs the required dependencies on the target instances. (default: True). | ||
| ## Output Parameters | ||
| None. | ||
| schemaVersion: '2.2' | ||
| parameters: | ||
| DurationSeconds: | ||
| type: String | ||
| description: "(Required) The duration - in seconds - of the disk stress." | ||
| allowedPattern: "^[0-9]+$" | ||
| Workers: | ||
| type: String | ||
| description: "The number of disk stressors (default: 1)." | ||
| default: "1" | ||
| allowedPattern: "^[0-9]+$" | ||
| Percent: | ||
| type: String | ||
| description: "The percentage of disk to use (required)." | ||
| allowedPattern: "^[0-9]+$" | ||
| InstallDependencies: | ||
| type: String | ||
| description: "If set to True, Systems Manager installs the required dependencies on the target instances. (default: True)." | ||
| default: 'True' | ||
| allowedValues: | ||
| - 'True' | ||
| - 'False' | ||
| mainSteps: | ||
| - action: aws:runShellScript | ||
| name: InstallDependencies | ||
| precondition: | ||
| StringEquals: | ||
| - platformType | ||
| - Linux | ||
| description: | | ||
| ## Parameter: InstallDependencies | ||
| If set to True, this step installs the required dependecy via operating system's repository. It supports both | ||
| Debian (apt) and CentOS (yum) based package managers. | ||
| inputs: | ||
| runCommand: | ||
| - | | ||
| #!/bin/bash | ||
| if [[ "{{ InstallDependencies }}" == True ]] ; then | ||
| if [[ "$( which stress-ng 2>/dev/null )" ]] ; then echo Dependency is already installed. ; exit ; fi | ||
| echo "Installing required dependencies" | ||
| if [ -f "/etc/system-release" ] ; then | ||
| if cat /etc/system-release | grep -i 'Amazon Linux' ; then | ||
| sudo amazon-linux-extras install testing | ||
| sudo yum -y install stress-ng | ||
| else | ||
| echo "There was a problem installing dependencies." | ||
| exit 1 | ||
| fi | ||
| elif cat /etc/issue | grep -i Ubuntu ; then | ||
| sudo apt-get update -y | ||
| sudo DEBIAN_FRONTEND=noninteractive sudo apt-get install -y stress-ng | ||
| else | ||
| echo "There was a problem installing dependencies." | ||
| exit 1 | ||
| fi | ||
| fi | ||
| - action: aws:runShellScript | ||
| name: ExecuteStressNg | ||
| precondition: | ||
| StringEquals: | ||
| - platformType | ||
| - Linux | ||
| description: | | ||
| ## Parameters: DurationSeconds, Workers and Percent | ||
| This step will run a disk stress test on the instance for the specified DurationSeconds time. | ||
| It will start `Workers` number of workers, using `Percent` of the total available disk. | ||
| inputs: | ||
| maxAttempts: 1 | ||
| runCommand: | ||
| - | | ||
| if [ {{ DurationSeconds }} -lt 1 ] || [ {{ DurationSeconds }} -gt 43200 ] ; then echo DurationSeconds parameter value must be between 1 and 43200 && exit; fi | ||
| pgrep stress-ng && echo Another stress-ng command is running, exiting... && exit | ||
| echo Initiating disk stress for {{ DurationSeconds }} seconds, {{ Workers }} workers, using {{ Percent }} percent of total available disk... | ||
| stress-ng --fallocate {{ Workers }} --fallocate-bytes {{ Percent }}% -t {{ DurationSeconds }}s --metrics | ||
| echo Finished disk stress. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters