feat: add docker image tests (#51)

### Description Ticket: https://observe.atlassian.net/browse/OB-33428 - Adds docker image integration tests - Updates to support `cloud-init` for Ubuntu (debian) machine for docker purposes - Docker images are also now part of build step, for upload. They are packaged as `.tar` for each distribution to "install" on the debian machine (See https://docs.docker.com/reference/cli/docker/image/save/ & https://docs.docker.com/reference/cli/docker/image/load/) - Minor wrapper helper function for common work between windows, linux, docker ### Checklist - [x] Created tests which fail without the change (if possible) - [x] Extended the README / documentation, if necessary
observeinc · Jul 23, 2024 · e0db84f · e0db84f
1 parent 15e0b9f
commit e0db84f
Show file tree

Hide file tree

Showing 9 changed files with 250 additions and 76 deletions.
diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml
@@ -45,7 +45,15 @@ jobs:
           args: release --prepare --clean --snapshot --verbose --parallelism 8
         env:         
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
-      - run: ls -l && ls -l dist/
+      - name: Save docker as .tar #Save docker images to tar
+        run: |
+          amd64_image=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep "amd64"| grep "observe-agent" | sort -r -k4 | head -n 1)
+          arm64v8_image=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep "arm64v8"| grep "observe-agent" | sort -r -k4 | head -n 1)
+          echo "amd64_image: ${amd64_image}"
+          echo "arm64v8_image: ${arm64v8_image}"
+          docker save -o dist/observe-agent_docker_arm64v8.tar ${arm64v8_image}
+          docker save -o dist/observe-agent_docker_amd64.tar ${amd64_image}
+      - run: ls -l && ls -l dist/         
       - name: Upload build artifacts 
         uses: actions/upload-artifact@v4        
         with:
@@ -60,7 +68,7 @@ jobs:
     strategy:
         fail-fast: false 
         matrix:
-            AWS_MACHINE: ["AMAZON_LINUX_2023", "UBUNTU_22_04_LTS", "WINDOWS_SERVER_2016_BASE", "WINDOWS_SERVER_2019_BASE", "WINDOWS_SERVER_2022_BASE"]
+            AWS_MACHINE: ["AMAZON_LINUX_2023", "UBUNTU_22_04_LTS", "WINDOWS_SERVER_2016_BASE", "WINDOWS_SERVER_2019_BASE", "WINDOWS_SERVER_2022_BASE", "DOCKER_AMD64_UBUNTU_22_04_LTS"]
     defaults:
         run:
             working-directory: integration #Terrafrom commands and tests are ran from integration directory

diff --git a/integration/modules/create_ec2/locals.tf b/integration/modules/create_ec2/locals.tf
@@ -15,6 +15,19 @@ locals {
       architecture      = "amd64"
     }
 
+    DOCKER_AMD64_UBUNTU_22_04_LTS = {
+      # ami used in testing
+      ami_instance_type = "t3.small"
+      ami_id            = "ami-036cafe742923b3d9"
+      ami_description   = "Used for Docker testing - Ubuntu Server 22.04 LTS (HVM)- EBS General Purpose (SSD) Volume Type. Support available from Canonical"
+      default_user      = "ubuntu"
+      sleep             = 120
+      user_data         = "user_data/aptbased_docker.sh"
+      distribution      = "docker"
+      package_type      = ".tar"
+      architecture      = "amd64"
+    }
+
     # UBUNTU_20_04_LTS = {
     #   # ami used in testing
     #   ami_instance_type = "t3.small"
@@ -54,7 +67,7 @@ locals {
       package_type      = ".rpm"
       architecture      = "x86_64"
     }
-
+         
     # RHEL_8_4_0 = {
     #   ami_instance_type = "t3.small"
     #   ami_id            = "ami-054965c6cd7c6e462"

diff --git a/integration/modules/create_ec2/user_data/aptbased_docker.sh b/integration/modules/create_ec2/user_data/aptbased_docker.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+apt-get update -y
+apt-get install wget curl -y
+sudo apt-get install ca-certificates curl
+
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update -y 
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y 
+sudo usermod -a -G docker $(whoami)
diff --git a/integration/scripts/test_configure.py b/integration/scripts/test_configure.py
@@ -3,10 +3,10 @@
 import sys
 import re
 import time 
-from utils import *
+import utils as u
 
-@print_test_decorator
-def run_test_windows(remote_host: Host, env_vars: dict) -> None:  
+@u.print_test_decorator
+def run_test_windows(remote_host: u.Host, env_vars: dict) -> None:  
 
     """
     Test to validate connection of observe-agent to Observe 
@@ -39,8 +39,41 @@ def run_test_windows(remote_host: Host, env_vars: dict) -> None:
 
     pass   
 
-@print_test_decorator
-def run_test_linux(remote_host: Host, env_vars: dict) -> None:    
+@u.print_test_decorator
+def run_test_docker(remote_host: u.Host, env_vars: dict) -> None:  
+    docker_prefix='sudo docker run \
+        --mount type=bind,source=/proc,target=/hostfs/proc,readonly \
+        --mount type=bind,source=/snap,target=/hostfs/snap,readonly \
+        --mount type=bind,source=/var/lib,target=/hostfs/var/lib,readonly \
+        --mount type=bind,source=/var/log,target=/hostfs/var/log,readonly \
+        --mount type=bind,source=/var/lib/docker/containers,target=/var/lib/docker/containers,readonly \
+        --mount type=bind,source=$(pwd)/observe-agent.yaml,target=/etc/observe-agent/observe-agent.yaml \
+        --pid host \
+        $(sudo docker images --format "{{.Repository}}:{{.Tag}}" | grep SNAPSHOT)'
+
+    init_command='init-config --token {} --observe_url {}'.format(env_vars["observe_token"], env_vars["observe_url"])
+    diagnose_command='diagnose'
+
+    #Set up correct config with observe url and token 
+    result = remote_host.run_command(docker_prefix + ' ' + init_command)
+
+    #Check diagnose command
+    result = remote_host.run_command(docker_prefix + ' ' + diagnose_command)
+    observe_val = False
+    for line in result.stdout.splitlines():      
+        if "Request to test URL responded with response code 200" in line:
+            print (" ✅ observe-agent -> observe validation passed! ")
+            observe_val = True
+            break        
+    if not observe_val:
+        print(result)
+        raise ValueError(f"❌ Failed: observe-agent -> observe validation")
+
+
+    pass
+
+@u.print_test_decorator
+def run_test_linux(remote_host: u.Host, env_vars: dict) -> None:    
 
     """
     Test to validate connection of observe-agent to Observe 
@@ -74,8 +107,8 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
 
 if __name__ == '__main__':
 
-    env_vars = get_env_vars(need_observe=True)
-    remote_host = Host(host_ip=env_vars["host"],
+    env_vars = u.get_env_vars(need_observe=True)
+    remote_host = u.Host(host_ip=env_vars["host"],
                        username=env_vars["user"],
                        key_file_path=env_vars["key_filename"],
                        password=env_vars["password"])    
@@ -87,6 +120,8 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
         run_test_linux(remote_host, env_vars)
     elif "windows" in env_vars["machine_config"]["distribution"]:
         run_test_windows(remote_host, env_vars)
+    elif "docker" in env_vars["machine_config"]["distribution"]:
+        run_test_docker(remote_host, env_vars)
 
     pass 
 

diff --git a/integration/scripts/test_ec2_connection.py b/integration/scripts/test_ec2_connection.py
@@ -5,10 +5,11 @@
 import sys
 import re
 import time 
-from utils import *
+import utils as u
 
-@print_test_decorator
-def run_test_windows(remote_host: Host, env_vars: dict) -> None:  
+
+@u.print_test_decorator
+def run_test_windows(remote_host: u.Host, env_vars: dict) -> None:  
 
     """
     This test validates that the UserdataExecution.log finished successfully 
@@ -58,10 +59,15 @@ def run_test_windows(remote_host: Host, env_vars: dict) -> None:
             time.sleep(1)        
         raise RuntimeError("❌ The UserdataExecution file did not finish successfully in time")  
 
+@u.print_test_decorator
+def run_test_docker(remote_host: u.Host, env_vars: dict) -> None:  
+    #Since our test is being done on a linux EC2, we can just check it initializes and runs similar to linux test
+    run_test_linux(remote_host, env_vars)
+    pass 
+
 
-
-@print_test_decorator
-def run_test_linux(remote_host: Host, env_vars: dict) -> None:    
+@u.print_test_decorator
+def run_test_linux(remote_host: u.Host, env_vars: dict) -> None:    
     """
     This test validates that the cloud-init file finished successfully 
     and ec2 instance is in stable state prior to running other
@@ -94,8 +100,8 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
 
 if __name__ == '__main__':
 
-    env_vars = get_env_vars()
-    remote_host = Host(host_ip=env_vars["host"],
+    env_vars = u.get_env_vars()
+    remote_host = u.Host(host_ip=env_vars["host"],
                        username=env_vars["user"],
                        key_file_path=env_vars["key_filename"],
                        password=env_vars["password"])    
@@ -108,5 +114,5 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
         run_test_linux(remote_host, env_vars)
     elif "windows" in env_vars["machine_config"]["distribution"]:
         run_test_windows(remote_host, env_vars)
-        
-
+    elif "docker" in env_vars["machine_config"]["distribution"]:
+        run_test_docker(remote_host, env_vars)
diff --git a/integration/scripts/test_install.py b/integration/scripts/test_install.py
@@ -6,11 +6,10 @@
 import re
 import time 
 import inspect 
-from utils import * 
+import utils as u
 
 
-
-def get_installation_package(env_vars: dict) -> tuple:
+def _get_installation_package(env_vars: dict) -> tuple:
     """Returns the full path and filename to the built distribution package
 
     Args:
@@ -43,8 +42,8 @@ def get_installation_package(env_vars: dict) -> tuple:
             print(f"Found matching file {filename} at: {full_path}")
             return filename, full_path
 
-@print_test_decorator
-def run_test_windows(remote_host: Host, env_vars: dict) -> None:  
+@u.print_test_decorator
+def run_test_windows(remote_host: u.Host, env_vars: dict) -> None:  
 
     """
     Test to install local observe-agent on a windows ec2 instance and validate command ran successfully 
@@ -57,7 +56,7 @@ def run_test_windows(remote_host: Host, env_vars: dict) -> None:
         RuntimeError: Installation error in powershell script
     """
     # Get built dist. installation package path for machine 
-    filename, full_path = get_installation_package(env_vars)    
+    filename, full_path = _get_installation_package(env_vars)    
 
     # Set windows home dir paths for consistency 
     home_dir = r"/C:/Users/{}".format(env_vars["user"]) #for user in sftp 
@@ -85,10 +84,24 @@ def run_test_windows(remote_host: Host, env_vars: dict) -> None:
     else:        
         print("✅ Installation test passed")
 
-
 
-@print_test_decorator
-def run_test_linux(remote_host: Host, env_vars: dict):       
+@u.print_test_decorator
+def run_test_docker(remote_host: u.Host, env_vars: dict) -> None:  
+
+    filename, full_path= _get_installation_package(env_vars)
+    home_dir = "/home/{}".format(env_vars["user"])
+
+    remote_host.put_file(full_path, home_dir)
+    result = remote_host.run_command('sudo docker load --input {}'.format(filename))
+    if result.stderr:
+        print(result)
+        raise RuntimeError("❌ Installation error in docker load")
+    else:
+        print("✅ Installation test passed")
+
+
+@u.print_test_decorator
+def run_test_linux(remote_host: u.Host, env_vars: dict):       
     """
     Test to install local observe-agent on a linux ec2 instance and validate command ran successfully 
 
@@ -99,7 +112,7 @@ def run_test_linux(remote_host: Host, env_vars: dict):
     Raises:
         RuntimeError: Unknown distribution type passed  
     """
-    filename, full_path= get_installation_package(env_vars)
+    filename, full_path= _get_installation_package(env_vars)
     home_dir = "/home/{}".format(env_vars["user"])
 
     remote_host.put_file(full_path, home_dir)
@@ -117,8 +130,8 @@ def run_test_linux(remote_host: Host, env_vars: dict):
 
 if __name__ == '__main__':
 
-    env_vars = get_env_vars()
-    remote_host = Host(host_ip=env_vars["host"],
+    env_vars = u.get_env_vars()
+    remote_host = u.Host(host_ip=env_vars["host"],
                        username=env_vars["user"],
                        key_file_path=env_vars["key_filename"],
                        password=env_vars["password"])    
@@ -130,6 +143,9 @@ def run_test_linux(remote_host: Host, env_vars: dict):
         run_test_linux(remote_host, env_vars)
     elif "windows" in env_vars["machine_config"]["distribution"]:
         run_test_windows(remote_host, env_vars)
+    elif "docker" in env_vars["machine_config"]["distribution"]:
+        run_test_docker(remote_host, env_vars)
+