feat: add windows to integration tests (#45)

### Description

- Adds Windows (2016/2019 Server) machines to observe-agent integration
tests
- Updates Readme
- Renamed tests to match closer to our readme wording 

### Checklist
- [x] Created tests which fail without the change (if possible)
- [x] Extended the README / documentation, if necessary

.github/workflows/tests-integration.yaml

@@ -60,8 +60,7 @@ jobs:
     strategy:
         fail-fast: false 
         matrix:
-            #AWS_MACHINE: ["AMAZON_LINUX_2023", "RHEL_8_4_0"]
-            AWS_MACHINE: ["AMAZON_LINUX_2023", "UBUNTU_22_04_LTS"]
+            AWS_MACHINE: ["AMAZON_LINUX_2023", "UBUNTU_22_04_LTS", "WINDOWS_SERVER_2016_BASE", "WINDOWS_SERVER_2019_BASE", "WINDOWS_SERVER_2022_BASE"]
     defaults:
         run:
             working-directory: integration #Terrafrom commands and tests are ran from integration directory
@@ -101,6 +100,8 @@ jobs:
 
         - name: Setup Terraform
           uses: hashicorp/setup-terraform@v3
+          with:
+            terraform_version: 1.8.5
 
         - name: Terraform Init
           id: init

integration/README.md

@@ -50,7 +50,7 @@ or through a `provider_override.tf` placed in `modules/create_ec2` directory.
 
 ### Local Testing
 
-Any of the python scripts in the `/scripts` directory can be tested by running them directly, granted an EC2 Machine exists. As the scripts rely on the outputs of `create_ec2` and `setup_observe_variables` modules to be passed in as environment variables, these environment variables can be manually set if the modules are not ran.
+Any of the python scripts in the `/scripts` directory can be tested by running them directly, granted an EC2 Machine exists. As the scripts rely on the outputs of `create_ec2` and `setup_observe_variables` modules to be passed in as environment variables, these environment variables can be manually set if the set up modules are not ran.
 
 The `/scripts/<test_xyz.py` expects the following environment variables to be set:
 
@@ -62,8 +62,29 @@ MACHINE_NAME="UBUNTU_22_04_LTS" #Machine name to test
 MACHINE_CONFIG="ami_description:Ubuntu Server 22.04 LTS (HVM)- EBS General Purpose (SSD) Volume Type. Support available from Canonical,ami_id:ami-036cafe742923b3d9,ami_instance_type:t3.small,architecture:amd64,default_user:ubuntu,distribution:debian,package_type:.deb,sleep:120,user_data:user_data/aptbased.sh" #Machine config 
 OBSERVE_URL="" #Observe URL to use for testing
 OBSERVE_TOKEN="" #Observe Token to use for testing
+PASSWORD="WindowsPassword to be used for testing" # Set to None for testing 
+
+```
+
+Run the scripts from the folder as below:
+```
+➜  integration git:(nikhil/integration-testing-windows) ✗ pwd
+/Users/nikhil.dua/Documents/observe-repos/observe-agent/integration
+➜  integration git:(nikhil/integration-testing-windows) ✗ python3 scripts/test_installation.py
 ```
 
+Note: If testing Windows machines, the RDP password is redacted by default in the python scripts. 
+This can be turned off when disabling mask by setting below environment variable to `False` before running these scripts
+```
+export MASK=False
+python3 scripts/test_ec2_connection.py
+------------------------------
+Masking Disabled
+Env vars set to: 
+ {'host': '54.177.26.178', 'user': 'Administrator', 'key_filename': './test_key.pem', 'password': '<exposed_password>, 'machine_name': 'WINDOWS_SERVER_2016_BASE', 'machine_config': {'ami_description': 'Microsoft Windows Server 2016 with Desktop Experience Locale English AMI provided by Amazon', 'ami_id': 'ami-07357c8c8d7501f94', 'ami_instance_type': 't3.small', 'architecture': 'x86_64', 'default_user': 'Administrator', 'distribution': 'windows', 'package_type': '.zip', 'sleep': '120', 'user_data': 'user_data/windows.ps'}, 'observe_url': 'https://179969258044.collect.observe-staging.com/', 'observe_token': '<exposed_token>}
+------------------------------
+Testing SSH connection to host 54.177.26.178 with timeout 120s
+```
 
 ### Architecture
 

integration/modules/create_ec2/locals.tf

@@ -74,34 +74,45 @@ locals {
     #   user_data         = "user_data/yumbased.sh"
     # }
 
-    # WINDOWS_SERVER_2016_BASE = {
-    #   ami_instance_type = "t3.small"
-    #   ami_id            = "ami-0e87182c1094f2344"
-    #   ami_description   = "Microsoft Windows Server 2016 with Desktop Experience Locale English AMI provided by Amazon"
-    #   default_user      = "Administrator"
-    #   sleep             = 120
-    #   user_data         = "user_data/windows.ps"
-    # }
+    WINDOWS_SERVER_2016_BASE = {
+      ami_instance_type = "t3.small"
+      ami_id            = "ami-07357c8c8d7501f94"
+      ami_description   = "Microsoft Windows Server 2016 with Desktop Experience Locale English AMI provided by Amazon"
+      default_user      = "Administrator"
+      sleep             = 120
+      user_data         = "user_data/windows.ps"
+      distribution      = "windows"
+      package_type      = ".zip"
+      architecture      = "x86_64"
+    }
 
-    # WINDOWS_SERVER_2019_BASE = {
-    #   ami_instance_type = "t3.small"
-    #   ami_id            = "ami-01dc5695dfebe46cc"
-    #   ami_description   = "Microsoft Windows Server 2019 with Desktop Experience Locale English AMI provided by Amazon"
-    #   default_user      = "Administrator"
-    #   sleep             = 120
-    #   user_data         = "user_data/windows.ps"
-    # }
 
-    # WINDOWS_SERVER_2022_BASE = {
-    #   ami_instance_type = "t3.small"
-    #   ami_id            = "ami-091f300417a06d788"
-    #   ami_description   = "Microsoft Windows Server 2022 Full Locale English AMI provided by Amazon"
-    #   default_user      = "Administrator"
-    #   sleep             = 120
-    #   user_data         = "user_data/windows.ps"
-    # }
+    WINDOWS_SERVER_2019_BASE = {
+      ami_instance_type = "t3.small"
+      ami_id            = "ami-006bce2026b3f63b8"
+      ami_description   = "Microsoft Windows Server 2019 with Desktop Experience Locale English AMI provided by Amazon"
+      default_user      = "Administrator"
+      sleep             = 120
+      user_data         = "user_data/windows.ps"
+      distribution      = "windows"
+      package_type      = ".zip"
+      architecture      = "x86_64"
+    }    
 
+
+     WINDOWS_SERVER_2022_BASE = {
+      ami_instance_type = "t3.small"
+      ami_id            = "ami-0fdf7c7a70369b831"
+      ami_description   = "Microsoft Windows Server 2022 Full Locale English AMI provided by Amazon"
+      default_user      = "Administrator"
+      sleep             = 120
+      user_data         = "user_data/windows.ps"
+      distribution      = "windows"
+      package_type      = ".zip"
+      architecture      = "x86_64"
+    }    
 
+
   }
 }
 

integration/modules/create_ec2/outputs.tf

@@ -38,6 +38,9 @@ output "private_key_path" {
 
 output "public_ssh_link" {
   value = "ssh -i ${var.PRIVATE_KEY_PATH} ${local.AWS_MACHINE_CONFIGS[var.AWS_MACHINE].default_user}@${aws_instance.observe_agent_instance.public_ip}"
-
 }
 
+output "password" {
+  value = can(regex("WINDOWS", var.AWS_MACHINE)) ? rsadecrypt(aws_instance.observe_agent_instance.password_data, file(var.PRIVATE_KEY_PATH)) : "None" 
+  sensitive = true   
+}

integration/scripts/install_windows.ps1

@@ -0,0 +1,42 @@
+# This scripts take an $local_install parameter, unzips the observe-agent .zip file and
+# and copies the relevant files to C:\Program Files\Observe\observe-agent
+# It's intended to only install observe-agent on a windows machine and ensure no issues take place
+
+#$local_installer="C:\Users\Administrator\observe-agent_Windows_x86_64.zip" This is set from python 
+
+param (
+    [Parameter(Mandatory=$true)]
+    [string]$local_installer
+)
+#Parameter is local_installer .zip file (which should already exist on machine)
+Write-Output "Local installer path is located at: $local_installer"
+
+$program_data_filestorage="C:\ProgramData\Observe\observe-agent\filestorage"
+$observeagent_install_dir="$env:ProgramFiles\Observe\observe-agent"
+$temp_dir="C:\temp"
+
+#Create directories for temp & observe-agent installation ls
+New-Item -ItemType Directory -Force -Path $temp_dir
+New-Item -ItemType Directory -Force -Path $observeagent_install_dir
+New-Item -ItemType Directory -Force -Path $observeagent_install_dir\config
+New-Item -ItemType Directory -Force -Path $program_data_filestorage
+
+# Stop the observe agent if its running so that we can copy the new .exe
+if((Get-Service ObserveAgent -ErrorAction SilentlyContinue)){
+    Write-Output "Observe Agent is running, Stopping Observe Agent..."
+    Stop-Service ObserveAgent
+}
+
+# Unzip the installer .zip to C:\temp\observe-agent_extract
+# Eg: Unzip C:\Users\Administrator\observe-agent_Windows_x86_64.zip to C:\temp\observe-agent_extract
+Write-Output "Unzipping installer $local_installer to $temp_dir\observe-agent_extract"
+Expand-Archive -Force -LiteralPath $local_installer  -DestinationPath "$temp_dir\observe-agent_extract"
+
+# Copy relevant files from C:\temp\observe-agent_extract to C:\Program Files\Observe\observe-agent
+Write-Output "Copying files from $temp_dir\observe-agent_extract to $observeagent_install_dir"
+Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.exe -Destination $observeagent_install_dir
+Copy-Item -Force -Path $temp_dir\observe-agent_extract\observe-agent.yaml -Destination $observeagent_install_dir
+Copy-Item -Force -Path $temp_dir\observe-agent_extract\otel-collector.yaml -Destination $observeagent_install_dir\config\otel-collector.yaml
+Copy-Item -Force -Path $temp_dir\observe-agent_extract\connections\ -Destination $observeagent_install_dir\connections -Recurse
+
+

integration/scripts/start_agent_windows.ps1

@@ -0,0 +1,36 @@
+##This script is responsible for creating & starting the observe agent service 
+
+$observeagent_install_dir="$env:ProgramFiles\Observe\observe-agent"
+
+if(-not (Get-Service ObserveAgent -ErrorAction SilentlyContinue)){
+    Write-Output "Creating ObserveAgent Service...."
+    $params = @{
+        Name = "ObserveAgent"
+        BinaryPathName =  "`"${observeagent_install_dir}\observe-agent.exe`" `"${observeagent_install_dir}\observe-agent.yaml`""
+        DisplayName = "Observe Agent"
+        StartupType = "Automatic"
+        Description = "Observe Agent based on OpenTelemetry collector"
+      }
+
+    New-Service @params
+    Write-Output "Starting ObserveAgent Service..."
+    Start-Service ObserveAgent
+    }
+else{
+    Write-Output "ObserveAgent Service already exists, restarting service..."
+    Stop-Service ObserveAgent
+    Start-Service ObserveAgent
+}
+
+## Placeholder below for future use 
+## Delete ObserveAgent service if needed 
+
+# if (Get-Service "ObserveAgent" -ErrorAction 'SilentlyContinue')
+# {
+#     $service = Get-WmiObject -Class Win32_Service -Filter "Name='ObserveAgent'"
+#     $service.delete()
+#     Write-Host "ObserveAgent service deleted"
+# }
+# else{
+#     write-host "No service found."
+# }

integration/scripts/test_diagnosis.py → integration/scripts/test_configure.py

@@ -5,7 +5,39 @@
 import time 
 from utils import *
 
+@print_test_decorator
+def run_test_windows(remote_host: Host, env_vars: dict) -> None:  
+
+    """
+    Test to validate connection of observe-agent to Observe 
+
+    Args:
+        remote_host (Host): instance to ssh into 
+        env_vars (dict): environment variables passed into for testing
+
+    Raises:
+        ValueError: Something failed with initial config or observe-agent -> observe connection 
+    """
+
+    init_command='Set-Location "C:\Program Files\Observe\observe-agent"; ./observe-agent init-config --token {} --observe_url {}'.format(env_vars["observe_token"], env_vars["observe_url"])
+    diagnose_command='Set-Location "C:\Program Files\Observe\observe-agent"; ./observe-agent diagnose'
+
+    #Set up correct config with observe url and token 
+    result = remote_host.run_command(init_command)
 
+    #Check diagnose command
+    result = remote_host.run_command(diagnose_command)
+    observe_val = False
+    for line in result.stdout.splitlines():      
+        if "Request to test URL responded with response code 200" in line:
+            print (" ✅ observe-agent -> observe validation passed! ")
+            observe_val = True
+            break        
+    if not observe_val:
+        print(result)
+        raise ValueError(f"❌ Failed: observe-agent -> observe validation")
+
+    pass   
 
 @print_test_decorator
 def run_test_linux(remote_host: Host, env_vars: dict) -> None:    
@@ -32,7 +64,7 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
     observe_val = False
     for line in result.stdout.splitlines():      
         if "Request to test URL responded with response code 200" in line:
-            print (" ✅ observe-agent -> observe valdation passed! ")
+            print (" ✅ observe-agent -> observe validation passed! ")
             observe_val = True
             break        
     if not observe_val:
@@ -45,13 +77,16 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
     env_vars = get_env_vars(need_observe=True)
     remote_host = Host(host_ip=env_vars["host"],
                        username=env_vars["user"],
-                       key_file_path=env_vars["key_filename"])    
+                       key_file_path=env_vars["key_filename"],
+                       password=env_vars["password"])    
 
     #Test SSH Connection before starting test of interest 
     remote_host.test_conection(int(env_vars["machine_config"]["sleep"]))   
 
     if "redhat" in env_vars["machine_config"]["distribution"] or "debian" in env_vars["machine_config"]["distribution"]:
         run_test_linux(remote_host, env_vars)
+    elif "windows" in env_vars["machine_config"]["distribution"]:
+        run_test_windows(remote_host, env_vars)
 
     pass 
 

integration/scripts/test_ec2_connection.py

@@ -7,6 +7,58 @@
 import time 
 from utils import *
 
+@print_test_decorator
+def run_test_windows(remote_host: Host, env_vars: dict) -> None:  
+
+    """
+    This test validates that the UserdataExecution.log finished successfully 
+    and ec2 instance is in stable state prior to running other
+
+
+    Args:
+        remote_host (Host): instance to ssh into 
+        env_vars (dict): environment variables passed into for testing
+
+    Raises:
+        RuntimeError: Failed to verify UserdataExecution.log or agent.logfile
+    """
+
+
+    tmp_file = "/tmp/UserdataExecution.log"
+    cloud_init_file_timeout = 240 # 4 minutes    
+
+    if "2022" in env_vars["machine_name"]: #Windows 2022 -  Test windows cloud-init file finished successfully
+        print("Windows 2022 detected")
+        cloud_init_file = r'/C:/ProgramData/Amazon/EC2Launch/log/agent.log'
+
+        for _ in range(cloud_init_file_timeout):        
+            remote_host.get_file(cloud_init_file, tmp_file) # This command will automatically test connection 
+            with open(tmp_file) as file: #No encoding for windows 2022 needed 
+                content = file.read().lower()
+                if "script execution finished successfully"  in content:
+                    print(" ✅ Verified agent.log had completed successfully!")
+                    return 
+                else:
+                    print(" Looking for the agent.log file to finish completing...")
+            time.sleep(1)        
+        raise RuntimeError("❌ The agent.log file did not finish successfully in time")  
+    else: # Windows 2016/2019 -   Test windows cloud-init file finished successfully
+        print("Windows 2016 or 2019 detected")
+        cloud_init_file = r'/C:/ProgramData/Amazon/EC2-Windows/Launch/Log/UserdataExecution.log'
+
+        for _ in range(cloud_init_file_timeout):        
+            remote_host.get_file(cloud_init_file, tmp_file) # This command will automatically test connection 
+            with open(tmp_file, encoding="utf-16") as file:
+                content = file.read().lower()
+                if "user data script completed"  in content:
+                    print(" ✅ Verified UserdataExecution had completed successfully!")
+                    return 
+                else:
+                    print(" Looking for the UserdataExecution.log file to finish completing...")
+            time.sleep(1)        
+        raise RuntimeError("❌ The UserdataExecution file did not finish successfully in time")  
+
+
 
 @print_test_decorator
 def run_test_linux(remote_host: Host, env_vars: dict) -> None:    
@@ -45,12 +97,16 @@ def run_test_linux(remote_host: Host, env_vars: dict) -> None:
     env_vars = get_env_vars()
     remote_host = Host(host_ip=env_vars["host"],
                        username=env_vars["user"],
-                       key_file_path=env_vars["key_filename"])    
+                       key_file_path=env_vars["key_filename"],
+                       password=env_vars["password"])    
 
     #Test SSH Connection before starting test of interest 
     remote_host.test_conection(int(env_vars["machine_config"]["sleep"]))   
+
 
     if "redhat" in env_vars["machine_config"]["distribution"] or "debian" in env_vars["machine_config"]["distribution"]:
         run_test_linux(remote_host, env_vars)
+    elif "windows" in env_vars["machine_config"]["distribution"]:
+        run_test_windows(remote_host, env_vars)