feat: add integration tests & integration test framework (#33)

.github/workflows/tests-integration.yaml

@@ -0,0 +1,115 @@
+name: Observe Agent Integration Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+#Special permissions required for OIDC authentication
+permissions:
+    id-token: write
+    contents: write
+    pull-requests: write
+
+env:
+    TF_VAR_name_format: "tf-observe-agent-test-%s"   
+    TF_VAR_PUBLIC_KEY_PATH: "./test_key.pub"
+    TF_VAR_PRIVATE_KEY_PATH: "./test_key.pem" 
+    TF_VAR_OBSERVE_URL: ${{secrets.OBSERVE_URL}}
+    TF_VAR_OBSERVE_TOKEN: ${{secrets.OBSERVE_TOKEN}}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable       
+      - name: Install msitools
+        run: sudo apt-get install -y wixl    
+      - name: Install qemu
+        uses: docker/setup-qemu-action@v3
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          distribution: goreleaser-pro
+          version: '~> v2'
+          args: release --prepare --clean --snapshot --verbose --parallelism 10
+        env:         
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+      - run: ls -l && ls -l dist/
+      - name: Upload build artifacts 
+        uses: actions/upload-artifact@v4        
+        with:
+          name: dist
+          path: dist/
+          retention-days: 10
+
+  observe-agent-integration-tests:
+    name: observe-agent-integration-tests-${{ matrix.AWS_MACHINE}}
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+        fail-fast: false 
+        matrix:
+            #AWS_MACHINE: ["AMAZON_LINUX_2023", "RHEL_8_4_0"]
+            AWS_MACHINE: ["AMAZON_LINUX_2023", "UBUNTU_22_04_LTS"]
+    defaults:
+        run:
+            working-directory: integration #Terrafrom commands and tests are ran from integration directory
+    steps:
+        - name: Checkout code
+          uses: actions/checkout@v4        
+        - run: git branch && df -h
+        - name: Configure AWS Credentials ## Terraform provider automatically uses these creds 
+          uses: aws-actions/configure-aws-credentials@v4
+          with:
+            role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+            role-session-name: ${{ github.sha }}
+            aws-region: ${{ secrets.AWS_REGION }}
+
+        - name: Setup Python
+          uses: actions/setup-python@v5
+          with:
+            python-version: '3.10' 
+        - run: pip install -r scripts/requirements.txt           
+
+        - name: Setup SSH Keys
+          run: |
+            echo "${{ secrets.SSH_PRIVATE_KEY }}" > ${{env.TF_VAR_PRIVATE_KEY_PATH}}
+            chmod 600 ${{env.TF_VAR_PRIVATE_KEY_PATH}}
+            echo "${{ secrets.SSH_PUBLIC_KEY }}" >${{env.TF_VAR_PUBLIC_KEY_PATH}}
+            chmod 644 ${{env.TF_VAR_PUBLIC_KEY_PATH}}
+                        
+        - name: Setup additional terraform variable
+          run:  echo "TF_VAR_AWS_MACHINE=$(echo ${{matrix.AWS_MACHINE}})" >> $GITHUB_ENV
+
+        - name: Download build artifacts 
+          uses: actions/download-artifact@v4        
+          id: artifact-download-step
+          with:
+            name: dist         
+            path: dist
+
+        - name: Setup Terraform
+          uses: hashicorp/setup-terraform@v3
+
+        - name: Terraform Init
+          id: init
+          run: terraform init
+
+        - name: Terraform Validate
+          id: validate
+          run: terraform validate -no-color
+
+        - name: Terraform Test
+          id: test
+          run: |               
+            terraform test -verbose 
+
+
+        

.gitignore

@@ -4,4 +4,25 @@ output-logs
 dist/
 agent.exe
 agent.exe~
-.DS_Store
+.DS_Store
+
+#integration
+*.pem
+*.pub 
+*.env 
+
+
+#terraform
+integration/.terraform
+.terraform/
+*.tfstate*
+terraform.tfvars
+*.auto.tfvars
+tfplan
+.terraform.lock.hcl
+*_override.tf
+
+
+#python
+*pycache*
+*.pyc

integration/README.md

@@ -0,0 +1,73 @@
+## Integration Tests 
+
+
+The root of this module location is intended to run integration tests using the terraform test framework. The tests are located at `integration/tests`
+
+The tests are run using the `terraform test -verbose` command from this folder `observe-agent/integration` 
+
+When the above command is run, the tests in the `integration/tests` directory are ran using the variables provided. The tests are ran in the order of the run blocks provided in `<test>.tftest.hcl` 
+
+Generally a test will do the following for any given EC2 Machine:
+- Create a machine using the variables provided below in `us-west-1`
+- Run a test using `observeinc/collection/aws//modules/testing/exec` module to accept python scripts located at `integration/tests/scripts` 
+
+
+### Variables 
+
+The tests are run using the following variables. These can be set in the `integration/tests.auto.tfvars` file for local testing. 
+
+```
+name_format        = "tf-observe-agent-test-%s"
+AWS_MACHINE= "AMAZON_LINUX_2023"  #Choose the AWS Machine to run the tests on 
+PUBLIC_KEY_PATH  = "./test_key.pub" #Path to Public Key for EC2
+PRIVATE_KEY_PATH = "./test_key.pem" #Path to Private Key for EC2
+OBSERVE_URL = "https://<TENANT_ID.collect.observe-staging.com>" #Observe URL to use for testing
+OBSERVE_TOKEN ="<DATASTREAM_TOKEN_TO_TEST_OBSERVE_AGENT_WITH">
+``` 
+
+
+Note: You must also set the provider correctly. We use the following settings:
+- Region: `us-west-1`
+- Profile: `blunderdome`
+- IAM Role Assumed: `gh-observe_agent-repo` 
+  - The above role has permissions to create and destroy EC2 instances. See `modules/setup_aws_backend/role.tf` for more details.
+
+The provider can be directly set in the `integration/tests/integration.tftest.hcl` as below:
+
+```
+provider "aws" {
+  region  = "us-west-1" # Specify the AWS region
+  profile = "blunderdome"
+  assume_role {
+      role_arn = "arn:aws:iam::<member_account>:role/gh-observe_agent-repo"
+  }
+}
+
+```
+
+or through a `provider_override.tf` placed in `modules/create_ec2` directory.
+
+
+### Local Testing
+
+Any of the python scripts in the `/scripts` directory can be tested by running them directly, granted an EC2 Machine exists. As the scripts rely on the outputs of `create_ec2` and `setup_observe_variables` modules to be passed in as environment variables, these environment variables can be manually set if the modules are not ran.
+
+The `/scripts/<test_xyz.py` expects the following environment variables to be set:
+
+```
+HOST="54.177.249.99" #HOST IP Address 
+USER="ubuntu" #HOST user to login as 
+KEY_FILENAME="./test_key.pem" #Private path to key 
+MACHINE_NAME="UBUNTU_22_04_LTS" #Machine name to test 
+MACHINE_CONFIG="ami_description:Ubuntu Server 22.04 LTS (HVM)- EBS General Purpose (SSD) Volume Type. Support available from Canonical,ami_id:ami-036cafe742923b3d9,ami_instance_type:t3.small,architecture:amd64,default_user:ubuntu,distribution:debian,package_type:.deb,sleep:120,user_data:user_data/aptbased.sh" #Machine config 
+OBSERVE_URL="" #Observe URL to use for testing
+OBSERVE_TOKEN="" #Observe Token to use for testing
+```
+
+
+### Architecture
+
+The architecture diagram can be found ![here](Observe-Agent.png)
+
+
+

integration/main.tf

@@ -0,0 +1,8 @@
+resource "null_resource" "main" {
+  provisioner "local-exec" {
+    command = <<EOF
+            echo "This root module does nothing and is intended for testing purposes for terraform test commands"
+            echo "Please call terrafrom test -verbose to run tests from this location" 
+            EOF
+  }
+}

integration/modules/create_ec2/README.md

@@ -0,0 +1,22 @@
+## Create EC2 
+
+This module sets up an EC2 Instance and EC2 Key Pair attached to the instance, for agent integration testsing. 
+
+It takes in the following variables to create EC2 Instance + Key Pair:
+- name_format
+- AWS_MACHINE
+- PUBLIC_KEY_PATH
+- PRIVATE_KEY_PATH 
+
+
+It then generates outputs that can be used in other modules (eg: `tests/*` for terraform test)
+
+### Dependencies
+
+It relies on the existence of the following in `us-west-1` 
+- Security Group Name: `tf-observe-agent-test-ec2_sg` 
+- Subnet Name: `tf-observe-agent-test-subnet`
+
+The above are used to attach to the EC2 Instance the module creates. 
+
+

integration/modules/create_ec2/data.tf

@@ -0,0 +1,13 @@
+
+
+data "aws_security_group" "ec2_public" {
+  name = "tf-observe-agent-test-ec2_sg"
+}
+
+data "aws_subnet" "subnet_public" {
+  filter {
+    name   = "tag:Name"
+    values = ["tf-observe-agent-test-subnet"]
+  }
+}
+

integration/modules/create_ec2/locals.tf

@@ -0,0 +1,118 @@
+
+#Map of allowed machines 
+locals {
+  AWS_MACHINE_CONFIGS = {
+    UBUNTU_22_04_LTS = {
+      # ami used in testing
+      ami_instance_type = "t3.small"
+      ami_id            = "ami-036cafe742923b3d9"
+      ami_description   = "Ubuntu Server 22.04 LTS (HVM)- EBS General Purpose (SSD) Volume Type. Support available from Canonical"
+      default_user      = "ubuntu"
+      sleep             = 120
+      user_data         = "user_data/aptbased.sh"
+      distribution      = "debian"
+      package_type      = ".deb"
+      architecture      = "amd64"
+    }
+
+    # UBUNTU_20_04_LTS = {
+    #   # ami used in testing
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-0892d3c7ee96c0bf7"
+    #   ami_description   = "Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on 2021-11-29"
+    #   default_user      = "ubuntu"
+    #   sleep             = 120
+    #   user_data         = "user_data/aptbased.sh"
+    # }
+
+    # UBUNTU_18_04_LTS = {
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-0cfa91bdbc3be780c"
+    #   ami_description   = "Canonical, Ubuntu, 18.04 LTS, amd64 bionic image build on 2022-04-11"
+    #   default_user      = "ubuntu"
+    #   sleep             = 120
+    #   user_data         = "user_data/aptbased.sh"
+    # }
+
+    # AMAZON_LINUX_2 = {
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-02b92c281a4d3dc79"
+    #   ami_description   = "Amazon Linux 2 Kernel 5.10 AMI 2.0.20220419.0 x86_64 HVM gp2"
+    #   default_user      = "ec2-user"
+    #   sleep             = 60
+    #   user_data         = "user_data/yumbased.sh"
+    # }
+
+    AMAZON_LINUX_2023 = {
+      ami_instance_type = "t3.small"
+      ami_id            = "ami-0a2781a262879e465"
+      ami_description   = "Amazon Linux 2023 AMI 2023.4.20240528.0 x86_64 HVM kernel-6.1"
+      default_user      = "ec2-user"
+      sleep             = 60
+      user_data         = "user_data/yumbased.sh"
+      distribution      = "redhat"
+      package_type      = ".rpm"
+      architecture      = "x86_64"
+    }
+
+    # RHEL_8_4_0 = {
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-054965c6cd7c6e462"
+    #   ami_description   = "Red Hat Enterprise Linux 8 (HVM), SSD Volume Type"
+    #   default_user      = "ec2-user"
+    #   sleep             = 120
+    #   user_data         = "user_data/yumbased.sh"
+    # }
+
+    # CENT_OS_7 = {
+    #   # https://wiki.centos.org/Cloud/AWS
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-0686851c4e7b1a8e1"
+    #   ami_description   = "CentOS 7.9.2009 x86_64 ami-0686851c4e7b1a8e1"
+    #   default_user      = "centos"
+    #   sleep             = 120
+    #   user_data         = "user_data/yumbased.sh"
+    # }
+
+    # WINDOWS_SERVER_2016_BASE = {
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-0e87182c1094f2344"
+    #   ami_description   = "Microsoft Windows Server 2016 with Desktop Experience Locale English AMI provided by Amazon"
+    #   default_user      = "Administrator"
+    #   sleep             = 120
+    #   user_data         = "user_data/windows.ps"
+    # }
+
+    # WINDOWS_SERVER_2019_BASE = {
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-01dc5695dfebe46cc"
+    #   ami_description   = "Microsoft Windows Server 2019 with Desktop Experience Locale English AMI provided by Amazon"
+    #   default_user      = "Administrator"
+    #   sleep             = 120
+    #   user_data         = "user_data/windows.ps"
+    # }
+
+    # WINDOWS_SERVER_2022_BASE = {
+    #   ami_instance_type = "t3.small"
+    #   ami_id            = "ami-091f300417a06d788"
+    #   ami_description   = "Microsoft Windows Server 2022 Full Locale English AMI provided by Amazon"
+    #   default_user      = "Administrator"
+    #   sleep             = 120
+    #   user_data         = "user_data/windows.ps"
+    # }
+
+
+  }
+}
+
+
+#Map of tags we'll attach
+locals {
+  BASE_TAGS = {
+    owner        = "Observe"
+    createdBy    = "terraform"
+    team         = "Product Specialists "
+    purpose      = "observe-agent integration tests"
+    git_repo_url = "https://github.com/observeinc/observe-agent"
+  }
+}