diff --git a/modules/azure.psm1 b/modules/azure.psm1 index 1bc56f3..60a60ed 100644 --- a/modules/azure.psm1 +++ b/modules/azure.psm1 @@ -1,7 +1,7 @@ #Common fuctions for Azure operations function GetSasToken($storageAccountName, $resourceGroup, $subscriptionName) -{ +{ $StartTime = Get-Date $EndTime = $startTime.AddYears(100) @@ -22,11 +22,11 @@ function GetSasToken($storageAccountName, $resourceGroup, $subscriptionName) { Write-Host "No matching sub found for $subscriptionName" VarDump($foundSubs) - Exit(1) + Exit(1) } $sub = Select-AzureRmSubscription -SubscriptionName $subscriptionName -ErrorAction Stop - + $keys = Get-AzureRmStorageAccountKey -AccountName $storageAccountName -ResourceGroupName $resourceGroup -ErrorAction Stop $key = $keys[0].Value @@ -60,18 +60,18 @@ function Remove-RmVMFull .SYNOPSIS This function is used to remove any Azure VMs as well as any attached disks. By default, this function creates a job due to the time it takes to remove an Azure VM. - + .EXAMPLE PS> Get-AzureRmVm -Name 'AS-GS111' | Remove-RmVMFull OR Remove-RmVMFull -Name $VMName -ResourceGroupName $azureResourceGroupName -Wait - + This example removes the Azure VM As-GS111 as well as any disks attached to it. - + .PARAMETER VMName The name of an Azure VM. This has an alias of Name which can be used as pipeline input from the Get-AzureRmVM cmdlet. - + .PARAMETER ResourceGroupName The name of the resource group the Azure VM is a part of. - + .PARAMETER Wait If you'd rather wait for the Azure VM to be removed before returning control to the console, use this switch parameter. If not, it will create a job and return a PSJob back. @@ -83,7 +83,7 @@ function Remove-RmVMFull [ValidateNotNullOrEmpty()] [Alias('Name')] [string]$VMName, - + [Parameter(Mandatory,ValueFromPipelineByPropertyName)] [ValidateNotNullOrEmpty()] [string]$ResourceGroupName, @@ -91,7 +91,7 @@ function Remove-RmVMFull [Parameter()] [ValidateNotNullOrEmpty()] [switch]$Wait - + ) process { try @@ -104,7 +104,7 @@ function Remove-RmVMFull 'ResourceGroupName' = $ResourceGroupName } $vm = Get-AzureRmVm @commonParams - + #region Get the VM ID $azResourceParams = @{ 'ResourceName' = $VMName @@ -114,7 +114,7 @@ function Remove-RmVMFull $vmResource = Get-AzureRmResource @azResourceParams $vmId = $vmResource.Properties.VmId #endregion - + #region Remove the boot diagnostics disk if ($vm.DiagnosticsProfile.bootDiagnostics) { @@ -132,11 +132,11 @@ function Remove-RmVMFull 'ResourceGroupName' = $diagSaRg 'Name' = $diagSa } - + Get-AzureRmStorageAccount @saParams | Get-AzureStorageContainer | where { $_.Name-eq $diagContainerName } | Remove-AzureStorageContainer -Force } #endregion - + Write-Host 'Removing the Azure VM...' $null = $vm | Remove-AzureRmVM -Force Write-Host 'Removing the Azure network interface...' @@ -150,25 +150,25 @@ function Remove-RmVMFull { Write-Host 'Removing the Public IP Address...' Remove-AzureRmPublicIpAddress -ResourceGroupName $vm.ResourceGroupName -Name $ipConfig.PublicIpAddress.Id.Split('/')[-1] -Force - } + } } - } + } + - ## Remove the OS disk Write-Host 'Removing OS disk...' $osDiskUri = $vm.StorageProfile.OSDisk.Vhd.Uri $osDiskContainerName = $osDiskUri.Split('/')[-2] - - ## TODO: Does not account for resouce group + + ## TODO: Does not account for resouce group $osDiskStorageAcct = Get-AzureRmStorageAccount | where { $_.StorageAccountName -eq $osDiskUri.Split('/')[2].Split('.')[0] } $osDiskStorageAcct | Remove-AzureStorageBlob -Container $osDiskContainerName -Blob $osDiskUri.Split('/')[-1] -ea Ignore - + #region Remove the status blob Write-Host 'Removing the OS disk status blob...' $osDiskStorageAcct | Get-AzureStorageBlob -Container $osDiskContainerName -Blob "$($vm.Name)*.status" | Remove-AzureStorageBlob #endregion - + ## Remove any other attached disks if ($vm.DataDiskNames.Count -gt 0) { @@ -180,7 +180,7 @@ function Remove-RmVMFull } } } - + if ($Wait.IsPresent) { & $scriptBlock -VMName $VMName -ResourceGroupName $ResourceGroupName @@ -193,7 +193,7 @@ function Remove-RmVMFull 'ArgumentList' = @($VMName, $ResourceGroupName) 'Name' = "Azure VM $VMName Removal" } - Start-Job @jobParams + Start-Job @jobParams } } catch @@ -238,3 +238,11 @@ function DownloadAzureStorageBlob { Write-Warning -Message ('Failed to download Azure Storage blob' -f $Blob); } } + +function GetKeyForStorageAccount { + param ([string] $resourceGroup, [string] $accountName) + + Write-Host "Getting Access Key for Storage Account: $accountName" + $accessKey = (Get-AzureRmStorageAccountKey -ResourceGroupName $resourceGroup -AccountName $accountName).Value[0]; + return $accessKey +} diff --git a/modules/remoting-alt.psm1 b/modules/remoting-alt.psm1 new file mode 100644 index 0000000..8de066d --- /dev/null +++ b/modules/remoting-alt.psm1 @@ -0,0 +1,48 @@ +function CopyUsingSSHSFTP { +#will recurse switch break single files? + <# + .Synopsis + Copy files and folders from source to destination server + + .Example + DownloadAzureStorageBlob $StorageAccountName $StorageAccountKey $Destination; + #> + [CmdletBinding()] + param ( + $orig, + $destFolder, + $ComputerName, + $port, + $username, + $password + ) + + Write-Host "Copy $orig to $destFolder in VM $ComputerName using psftp." + #dos2unix $orig + Write-Host "Starting copy" + + $program = "..\..\libraries\putty\psftp.exe" + $cmd = @( + "cd /$destFolder", + "put -r $orig" + "bye" + ) + # Detect hostkey + $output1 = [String]($cmd | & $program -batch -pw $password "$username@$ComputerName" 2>&1 ) + $output1 | Out-File "temp.txt" -Force # It is horrible but I am in a hurry + $output2 = Get-Content "temp.txt" + foreach ($line in $output2) + { + if ($line -like "ssh-rsa 2048 *") + { + $hostkey = $line -replace "ssh-rsa 2048 ","" + Write-Host "Hostkey detected." + } + } + Write-Host "Hostkey=$hostkey." + + # Do the copy with Hostkey + [String]($cmd | & $program -batch -pw $password "$username@$ComputerName" -hostkey $hostkey 2>&1 ) + + Write-Host "Copy finished." +} diff --git a/modules/resource-locator.psm1 b/modules/resource-locator.psm1 new file mode 100644 index 0000000..929f7af --- /dev/null +++ b/modules/resource-locator.psm1 @@ -0,0 +1,74 @@ +<# + .SYNOPSIS + Functions that return locations of external resources that comply with our new naming convention/ + This is basically a dynamic config file + Author: Matthew Tyas +#> + +#Azure Resources +function GetVMImageName($environment, $branch) +{ + return "GameServerBaseImage_${environment}_${branch}" +} + +function GetVmName($environment) +{ + return "as-gstemplate-$environment" +} + +function GetASLAddress($environment, $region) +{ + return "https://as-${environment}-asl-${region}.azurewebsites.net/api" +} + +function GetVirtualNetwork($environment, $region) +{ + return "as-${environment}-vnet-GS-${region}" +} + +function GetNetworkSecurityGroupName($environment) +{ + return "as-${environment}-nsg-GS" +} + +function GetAzureResourceGroupName($environment, $region) +{ + return "as-${environment}-rg-GS-${region}" +} + +function GetEmulatorStorageResourceGroupName($environment, $region) +{ + return "as-${environment}-rg-storage-${region}" +} +# Storage Accounts + +function GetImageStorageAccount($environment, $region) +{ + return "as${environment}rggs${region}disks01" +} + +function GetDiagnosticsStorageAccountName($environment, $region) +{ + return "as${environment}rggs${region}diag01" +} + +# Azure storage account name for the emulators and roms +function GetEmulatorStorageAccount($environment, $region) +{ + return "as${environment}storage${region}".ToLower() +} + +function GetEmulatorFileShareName() +{ + return "binaries" +} + +function GetGameSettingsStorageAccount() +{ + return "asdevblob" +} + +function GetRomStorageAccount($environment, $region) +{ + return "as${environment}storage${region}".ToLower() +} diff --git a/src/Build/ConfigureGSTemplate.ps1 b/src/Build/ConfigureGSTemplate.ps1 new file mode 100644 index 0000000..35021d0 --- /dev/null +++ b/src/Build/ConfigureGSTemplate.ps1 @@ -0,0 +1,250 @@ +param( + [switch] $Help = $false, # Allows help to be shown + $azurePassword, # Azure Password + $azureUsername, # i.e: provision-serviceaccount@adminantstream.onmicrosoft.com + $azureSubscriptionName, + $vmAdministratorUsername, + $vmAdministratorPassword, + $gameServerUsername, # CGDeploy Directory USER for demo files. Used by apache. (i.e: cginternal_tOuwMfFmBx8e1OI44SZ7) + $gameServerPassword, # CGDeploy Directory password for demo files. Used by apache. + $aslApiUsername, # CGDeploy ASL API Username. It will be stored in curlparams file. (i.e: ASLVMAPI) + $aslApiPassword, # CGDeploy ASL API Password. It will be stored in curlparams file. + $aslAddress, # ASL URL to use. It will be stored in curlparams and asladdress files. # nameSSH-Sessions of the share with the emulator + $environment, + $region # Region of the VM +) +# Useful links +# Create Linux VM https://azure.microsoft.com/en-gb/documentation/articles/virtual-machines-ps-create-preconfigure-linux-vms +# Create Windows VM https://azure.microsoft.com/en-gb/documentation/articles/virtual-machines-ps-create-preconfigure-windows-vms/ +# VMImages https://azure.microsoft.com/en-gb/blog/vm-image-blog-post/ +# SSH connector http://www.powershelladmin.com/wiki/SSH_from_PowerShell_using_the_SSH.NET_library +# SSH connector http://www.jonathanmedd.net/2013/08/using-ssh-to-access-linux-servers-in-powershell.html + +function Usage() +{ + Write-Host 'Expected parameters:' + Write-Host ' -Help : Shows this help.' + exit 0 +} + +function ExecuteSSH($command, $ComputerName, $port, $username, $password, $verbose) +{ + Write-Host "Executing command '$command ' in VM $ComputerName" + $session = Get-SshSession -ComputerName $ComputerName + if ($session.'SSH Connected' -ne "True") + { + #Get-SshSession -ComputerName $ComputerName | ft -AutoSize + Write-Host "Seems that SSH session to $ComputerName was closed. Reopenning" + New-SshSession -ComputerName $ComputerName -Username $username -Password "$password" -Port $port -Reconnect + } + + $result = Invoke-SshCommand -ComputerName $ComputerName -Command "echo $password | sudo -S $command" + if ($verbose) + { + Write-Host $result + } +} + +function dos2unix($file) +{ + Write-Host "Converting file $file from dos format to unix format (dos2unix)" + $x = get-content -raw -path $file + $x -replace "`r`n","`n" | set-content -path $file + Write-Host "Converted." +} + +function WaitVMToHaveStatus($azureResourceGroupName, $vmName, $desiredStatus, $retrySeconds, [int] $maxNumberRetries) +{ + [bool] $keepPolling = $true + [int] $numberRetries = 0 + while ($keepPolling) + { + Start-Sleep $retrySeconds + $numberRetries++ + $vmstatus = Get-AzureRmVM -ResourceGroupName $azureResourceGroupName -Name $vmName -Status -ErrorAction Ignore + $vmPowerStatus = $vmstatus.Statuses[1].DisplayStatus + if ($vmPowerStatus -eq $desiredStatus) + { + Write-Host "VM Machine $vmName is finally running in $vmPowerStatus state. Attempt $numberRetries" + break + } + else + { + Write-Host "VM Machine $vmName is not YET running. State = $vmPowerStatus. Attempt $numberRetries" + } + + if ($numberRetries -gt $maxNumberRetries) + { + throw "Max number of retries exceeded ($numberRetries of maximum $maxNumberRetries attempts) waiting for $VM to get to state $desiredStatus" + } + } +} + +function WaitForSSH($ComputerName, $vmAdministratorUsername, $vmAdministratorPassword, $port) +{ + [bool] $keepPolling = $true + [int] $numberRetries = 0 + [int] $retrySeconds = 5 + [int] $maxNumberRetries = 100 + + Write-Host "Waiting for SSH." + while ($keepPolling) + { + # Function made by Steve Cottam + New-SshSession -ComputerName $ComputerName -Username $vmAdministratorUsername -Password "$vmAdministratorPassword" -Port $port -Scp -Sftp -ErrorAction SilentlyContinue -WarningAction SilentlyContinue | Out-Null + $session = Get-SshSession -ComputerName $ComputerName + if (($session.'SSH Connected' -eq "True") -and (($session.'SCP Connected' -eq "True")) -and (($session.'SFTP Connected' -eq "True"))) + { + Write-Host "SSH Connected - closing this connection." + Remove-SshSession -ComputerName $ComputerName + break + } + else + { + Write-Host "SSH Not Yet Connected. Attempt $numberRetries" + } + + Start-Sleep $retrySeconds + $numberRetries++ + + if ($numberRetries -gt $maxNumberRetries) + { + throw "Max number of retries exceeded ($numberRetries of maximum $maxNumberRetries attempts) waiting for $VM to get to state $desiredStatus" + } + } +} + +# Global variables +$scriptDir = Split-Path ($MyInvocation.MyCommand.Definition) -parent +Set-Location $scriptDir + +$moduleDir = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($scriptDir, "..\..\modules\")) +$libraryDir = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($scriptDir, "..\..\libraries\")) + +Import-Module "$moduleDir\azure.psm1" -Force +Import-Module "$libraryDir\SSH-SessionsP\SSH-Sessions" -Force +Import-Module "$moduleDir\remoting-alt.psm1" -Force #I can only apologise for this file existing +Import-Module "$moduleDir\resource-locator.psm1" -Force + +$scpInstalled = Get-Module -ListAvailable -Name Posh-SSH +if (-Not $scpInstalled) { + Install-PackageProvider NuGet -Force + Import-PackageProvider NuGet -Force +} + +#Main +$ErrorActionPreference = "Stop" +try +{ + #Preparation steps + $startTime = Get-Date + + $vmName = GetVmName $environment + Write-Host -ForegroundColor Cyan "This deploy GameServer files to templateVM '$vmName'" + Write-Host "Script started ($startTime)" + DetectPowershellVersion 4 + DetectAzurePowerShellVersion + $azurePassword = $azurePassword | ConvertTo-SecureString -asPlainText -Force + $azureCredentials = new-object -typename System.Management.Automation.PSCredential -argumentlist $azureUsername,$azurePassword + Disable-AzureDataCollection -WarningAction SilentlyContinue + + #1 + Write-Host -ForegroundColor Cyan "Step 1: Logging in to Azure..." + Add-AzureRmAccount -credential $azureCredentials | Out-Null + Write-Host "Logged in." + + #2 + Write-Host -ForegroundColor Cyan "Step 2: Selecting subscription..." + $AzureSub = Select-AzureRmSubscription -SubscriptionName $azureSubscriptionName -Verbose + Write-Host "We are now in $AzureSub.Subscription" + + $azureResourceGroupName = GetAzureResourceGroupName -environment $environment -region $region + + WaitVMToHaveStatus $azureResourceGroupName $vmName "VM running" 10 180 + + #3 Deploy + Write-Host -ForegroundColor Cyan "Step 3: Connecting via SSH" + Write-Host "Locating VM: $vmName in Resource Group: $azureResourceGroupName" + $vm = Get-AzureRmVM -ResourceGroupName $azureResourceGroupName -Name $vmName + if ($vm -eq $null) { throw "Cannot find VM: $vmName in Resource Group: $azureResourceGroupName" } + + #Detect ComputerName and port + $ComputerName = $vm | Get-AzureRmPublicIPAddress + $ComputerName = $ComputerName.IPAddress + $port = "22" + Write-Host "Detected IP Address of VM $vmName --> $ComputerName" + Write-Host "Detected SSH port: $port" + + $romResourceGroup = "as-${environment}-rg-storage-${region}" + $romSubscription = "as-sub-working" + $settingsResourceGroup = "as-asl-working" + $settingsSubscription = "as-sub-working" + + Write-Host -ForegroundColor Cyan "Step 4: Generating SAS tokens" + + $settingsStorageAccount = GetGameSettingsStorageAccount + Write-Host "Game Settings Storage Account: " + $settingsStorageAccount + + $romStorageAccount = GetRomStorageAccount -environment $environment -region $region + + Write-Host "Rom Storage Account: ${romStorageAccount}" + + $romSasKey = GetSasToken $romStorageAccount $romResourceGroup $romSubscription + $settingsSasKey = GetSasToken $settingsStorageAccount $settingsResourceGroup $settingsSubscription + + Write-Host -ForegroundColor Cyan "Getting Eventhubs Key" + $eventHubResourceGroupName = "as-$environment-rg-logs-GLOBAL" + $eventHubNameSpaceName = "as-$environment-eh-logs-$region" + + $eventHubsKey = (Get-AzureRmEventHubKey -ResourceGroupName $eventHubResourceGroupName -Namespace $eventHubNameSpaceName -Name "RootManageSharedAccessKey").PrimaryKey + + Write-Host "Connecting to computer: $ComputerName" + WaitForSSH $ComputerName $vmAdministratorUsername $vmAdministratorPassword $port + New-SshSession -ComputerName $ComputerName -Username $vmAdministratorUsername -Password "$vmAdministratorPassword" -Port $port -Scp -Sftp + Write-Host "Connected." + + # Copy Provisioning files and scripts to VM + Write-Host "Copying over assets" + + CopyUsingSSHSFTP "..\Provisioning" "/tmp" $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword -ErrorAction Continue + + # Update Ubuntu + ExecuteSSH "apt-get update | tee /home/antmin/cgdeploy2.log" $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword $false + ExecuteSSH 'sudo apt-get -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade | tee /home/antmin/cgdeploy3.log' $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword $false + ExecuteSSH "cp -v /usr/sbin/waagent.save /usr/sbin/waagent | tee /home/antmin/cgdeploy4.log" $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword $true + + $emulatorStorageAccount = GetEmulatorStorageAccount -environment $environment -region $region + + Write-Host "Getting Emulator Storage Account Access Key" + $emulatorStorageResourceGroup = GetEmulatorStorageResourceGroupName -environment $environment -region $region + + Write-Host "Emulator Storage Account: ${emulatorStorageAccount}, Emulator Resource Group: ${emulatorStorageResourceGroup}" + $emulatorAccessKey = GetKeyForStorageAccount -resourceGroup $emulatorStorageResourceGroup -accountName $emulatorStorageAccount + + $emulatorShareName = GetEmulatorFileShareName + Write-Host "Emultor File Share Name: ${emulatorShareName}" + # Run the povisioning script + $cgdeployparameters="`"$gameServerUsername`" `"$gameServerPassword`" `"$aslAddress`" `"$aslApiUsername`" `"$aslApiPassword`" `"$emulatorStorageAccount`" `"$emulatorAccessKey`" `"$emulatorShareName`" `"$region`" `"$romStorageAccount`" `"$romSasKey`" `"$settingsStorageAccount`" `"$settingsSasKey`" `"$eventHubsKey`"" + ExecuteSSH "sudo /bin/bash /tmp/Provisioning/base.sh $cgdeployparameters 2>&1 | tee /home/antmin/cgdeploy5.log" $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword $false + + # Switch off: sudo waagent -deprovision && sudo shutdown -h now + ExecuteSSH "waagent -deprovision+user -force | tee /home/antmin/cgdeploy8.log" $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword $true + ExecuteSSH "shutdown -h now | tee /home/antmin/cgdeploy9.log" $ComputerName $port $vmAdministratorUsername $vmAdministratorPassword $true + + Remove-SshSession -ComputerName $ComputerName + # END + $endTime = Get-Date + Write-Host "Script finished ($endTime)" + $elapsedTime = new-timespan $startTime $endTime + Write-Host -ForegroundColor Cyan "SUCCESS: Script time = $elapsedTime" + + exit 0 +} +catch [Exception] +{ + Write-Host -ForegroundColor Magenta $psItem.Exception + Write-Host -ForegroundColor Magenta $psItem.ErrorDetails + Write-Host -ForegroundColor Magenta "Error source line:" $psItem.InvocationInfo.ScriptLineNumber + Write-Error -Exception $psItem.Exception + Exit 1 +}