diff --git a/docs/Proxmox.md b/docs/Proxmox.md new file mode 100644 index 0000000..4a492d7 --- /dev/null +++ b/docs/Proxmox.md @@ -0,0 +1,162 @@ +# Using Proxmox with TopoMojo + +[Proxmox Virtual Environment (PVE)](https://pve.proxmox.com/wiki/Main_Page) is an open source server virtualization management solution based on QEMU/KVM and LXC. TopoMojo can be configured to use a PVE cluster to deploy QEMU virtual machines rather than it's traditional VMware based virtual machines. + +## Proxmox Setup + +There are a few things you wil need to do within Proxmox to prepare it for use with TopoMojo. + +### Installation + +- Install Proxmox on one or more nodes. + - Add all of the nodes that you want to be used by TopoMojo to a single Proxmox cluster. + +### Generate an Access Token + +TopoMojo requires a Proxmox Access Token in order to authenticate with the Proxmox API. + +- From the Proxmox Web UI, generate an API Token by clicking on Datacenter and navigating to Permissions -> API Tokens. + - Ensure Privilege Separation is unchecked if you want to use the privileges of the token user. Otherwise, you will need to select individual permissions to give to the token. + - Copy the Secret and the Token ID. This will need to be added to appsettings later. + +### Create an SDN Zone + +TopoMojo uses Proxmox's Software Defined Networking (SDN) feature to manage the networks of the virtual labs. You will need to create an SDN Zone in Proxmox and configure TopoMojo to use it. + +- In the Proxmox Web UI navigate to Datacenter -> SDN -> Zones. +- Add a new VXLAN Zone. + - VXLAN is the only type currently supported by TopoMojo. + - The ID is the name you want to use for this Zone. You will need it when configuring TopoMojo's appsettings. + - Under Peer Address List, enter a comma separated list of the IP Addresses of all of the nodes in your cluster. + - If you add a new node to your cluster, you will need to add it to the SDN Zone as well. + +### Configure NGINX + +You will need to configure a reverse proxy on the node that TopoMojo will communicate with in order to access the API and allow viewing of consoles. This will allow the Proxmox API to be accessed over port 443 as well as provide the required authentication headers for accessing consoles through an external application. Instructions for doing this with NGINX are provided below. + +- Install NGINX on your main Proxmox node and configure it to run on startup. + - `sudo apt install nginx` + - `sudo systemctl enable nginx` +- Use the following NGINX configuration as a reference. This will allow your Proxmox Web UI and API to be reached over port 443 as well as allow console access to work through TopoMojo. + - Replace "pve.local" with your Node's hostname + - Replace with the API Token you generated earlier. This should be in the format `user@system!TokenId=Secret` e.g. `root@pam!Topo=4c4fbe1e-b31e-55a9-9fg0-2de4a411cd23` + +``` +upstream proxmox { + server "pve.local"; +} + +server { + listen 80 default_server; + rewrite ^(.*) https://$host$1 permanent; +} + +server { + listen 443; + server_name _; + ssl on; + ssl_certificate /etc/pve/local/pve-ssl.pem; + ssl_certificate_key /etc/pve/local/pve-ssl.key; + proxy_redirect off; + + location ~ /api2/json/nodes/.+/qemu/.+/vncwebsocket.* { + proxy_set_header "Authorization" "PVEAPIToken="; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass https://localhost:8006; + proxy_buffering off; + client_max_body_size 0; + proxy_connect_timeout 3600s; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + send_timeout 3600s; + } + + location / { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass https://localhost:8006; + proxy_buffering off; + client_max_body_size 0; + proxy_connect_timeout 3600s; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + send_timeout 3600s; + } +} +``` + +## TopoMojo Setup + +This section describes the appsettings that will need to be set to configure TopoMojo to use Proxmox + +### Required Settings + +- Pod__HypervisorType + - Set this to `Proxmox` to enable Proxmox mode. + - Each TopoMojo instance currently operates either entirely in Vsphere or Proxmox mode. +- Pod__Url + - Set this to the url of your main Proxmox node. (i.e. https://my.prox.local) +- Pod__AccessToken + - Set this to the access token generated above. It should be the same one set in your NGINX config. + - E.g. `root@pam!Topo=4c4fbe1e-b31e-55a9-9fg0-2de4a411cd23` +- Pod__SDNZone + - Set this to the name of the SDN Zone you created in Proxmox + +### Optional Settings + +- Pod_Password + - Set this to the password of the **root** user account only to enable Guest Settings support (discussed in detail below). + - If no password or an invalid root password is provided, Guest Settings will be disabled. +- Pod__Vlan__ResetDebounceDuration + - The integer number of milliseconds TopoMojo will wait after a virtual network operation is initiated before reloading Proxmox's SDN. As reloading is a synchronous process that can take up 10 seconds, we offer this setting to reduce aggregate wait times by debouncing changes into batches. +- Pod__Vlan__ResetDebounceMaxDuration + - The integer number of milliseconds that describes the maximum amount of time TopoMojo will debounce before it reloads Proxmox's SDN following a network operation. + +#### ISOs + +TopoMojo can optionally allow uploading of ISO files that can be mounted to virtual machines. You will need to set these settings to enable this feature. + +- Pod__IsoStore + - Set this to the name of the shared storage in your Proxmox cluster that ISOs will be sourced from for mounting to virtual machines. + - e.g. `iso` +- FileUpload_IsoRoot + - Set this to a path that is mounted to the TopoMojo API container that ISOs uploaded through TopoMojo will be saved to. + - This should map to the same underlying storage as `Pod_IsoStore` above. + - Proxmox creates a particular directory structure for ISO stores, so this path needs to end in /template/iso. + - e.g. `/mnt/isos/template/iso` +- FileUpload_SupportsSubFolders + - Set this to `false` for Proxmox since Proxmox does not allow sub folders in it's ISO stores + +## Guest Settings + +TopoMojo templates have a Guest Settings section, allowing the user to set key value pairs that are injected into the virtual machine. + +In Vsphere, this is done by setting guestinfo values in the virtual machine's ExtraConfig that can be retrieved inside the virtual machine using open-vm-tools or vmware tools with the command `vmtoolsd --cmd "info-get guestinfo.variable"`. + +In Proxmox, a similar functionality is achieved using the QEMU Firmware Configuration (fw_cfg) Device. Guest Settings are injected into the virtual machine and can be accessed with the command `sudo cat /sys/firmware/qemu_fw_cfg/by_name/opt/guestinfo.variable/raw`, where `variable` is the key of the Guest Setting. + +Note: This currently only works on Linux Guests. There is an open source [Windows driver](https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/fwcfg64) that has some basic fw_cfg support, but does not support reading user-defined /opt values at this time. + +As described in the Settings section, this currently requires the use of the root user and password. There is a [patch](https://bugzilla.proxmox.com/show_bug.cgi?id=4068) available for Proxmox that would make this no longer necessary, but it has not been merged into a release. Currently, if a root password is not provided, Guest Settings will be skipped when virtual machines are deployed. + +## Using Proxmox Templates + +In Vsphere, TopoMojo templates point to virtual disks. In Proxmox, TopoMojo templates point to Proxmox Templates. To get started with a new installation, create one or more Proxmox Templates manually by deploying a virtual machine and converting it into a template. Then create a TopoMojo template and set the `Template` value of it's Detail property to the name of the Proxmox template you created. When deplying this TopoMojo template, TopoMojo will create a linked clone of the Proxmox template to the same storage location that the template exists on and reconfigure appropriate values from the TopoMojo template such as memory, cpus, networks, etc. + +### Windows Virtual Machines + +Windows virtual machines require the installation of VirtIO drivers for compatibility with QEMU/KVM. Details can be found at https://pve.proxmox.com/wiki/Windows_VirtIO_Drivers. + +### Clipboard Support + +TopoMojo supports clipboard access to Proxmox Virtual machines, if the appropriate pre-requisites are set. + +- On the Proxmox template, the VNC Clipboard must be enabled. + - To do this, in the template's Hardware tab, Edit the Display and set Clipboard to VNC + - There is currently a known QEMU limitation where a virtual machine with the Clipboard set to VNC cannot be migrated to another Node. You may need to temporarily disable the VNC clipboard, perform the migration, and re-enable it if you need to move a vm to another Node. +- The [SPICE Guest Tools](https://www.spice-space.org/download.html) must be installed in the virtual machines + - This is installed by default on some Linux distributions. + - This must be installed manually in Windows and has been tested and works the same as Linux clipboard support. diff --git a/src/TopoMojo.Agent/EventHandler.cs b/src/TopoMojo.Agent/EventHandler.cs index 5e834bb..8f85d9d 100644 --- a/src/TopoMojo.Agent/EventHandler.cs +++ b/src/TopoMojo.Agent/EventHandler.cs @@ -72,8 +72,9 @@ private async Task ProcessInitial() var since = DateTimeOffset.UtcNow.AddMinutes(-5).ToString("u"); var dispatches = await Mojo.ListDispatchesAsync( - Config.GroupId, + Config.GroupId, since, + false, "", null, null, "", new string[] { "pending" } ); @@ -165,7 +166,7 @@ internal async Task Connect() { try { - + if (!connected) { await Hub.StartAsync(); diff --git a/src/TopoMojo.Api/AppSettings.cs b/src/TopoMojo.Api/AppSettings.cs index 0942ba1..dd9dda0 100644 --- a/src/TopoMojo.Api/AppSettings.cs +++ b/src/TopoMojo.Api/AppSettings.cs @@ -79,6 +79,7 @@ public class FileUploadOptions public string TopoRoot { get; set; } = "wwwroot"; public string IsoRoot { get; set; } = "wwwroot/isos"; public string DocRoot { get; set; } = "wwwroot/docs"; + public bool SupportsSubfolders { get; set; } = true; } public class HeaderOptions diff --git a/src/TopoMojo.Api/Features/File/FileController.cs b/src/TopoMojo.Api/Features/File/FileController.cs index e79af60..496acdb 100644 --- a/src/TopoMojo.Api/Features/File/FileController.cs +++ b/src/TopoMojo.Api/Features/File/FileController.cs @@ -143,18 +143,26 @@ await _uploader.Process( private string BuildDestinationPath(string filename, string key) { - string path = Path.Combine( - _config.IsoRoot, - key.SanitizePath() - ); - - if (!Directory.Exists(path)) - Directory.CreateDirectory(path); - - return Path.Combine( - path, - filename.Replace(" ", "").SanitizeFilename() - ); + if (_config.SupportsSubfolders) + { + string path = Path.Combine( + _config.IsoRoot, + key.SanitizePath() + ); + + if (!Directory.Exists(path)) + Directory.CreateDirectory(path); + + return Path.Combine( + path, + filename.Replace(" ", "").SanitizeFilename() + ); + } + else + { + var fileName = $"{key.SanitizePath()}#{filename.Replace(" ", "").SanitizeFilename()}"; + return Path.Combine(_config.IsoRoot, fileName); + } } } diff --git a/src/TopoMojo.Api/Features/Template/TemplateUtility.cs b/src/TopoMojo.Api/Features/Template/TemplateUtility.cs index 736de6a..bba0c99 100644 --- a/src/TopoMojo.Api/Features/Template/TemplateUtility.cs +++ b/src/TopoMojo.Api/Features/Template/TemplateUtility.cs @@ -181,6 +181,14 @@ public void LocalizeDiskPaths(string workspaceKey, string templateKey) i += 1; } + + // Proxmox + // TODO: Move to separate method? + if (!string.IsNullOrEmpty(_template.Template)) + { + _template.ParentTemplate = _template.Template; + _template.Template = _template.Name; + } } public override string ToString() diff --git a/src/TopoMojo.Api/Startup.cs b/src/TopoMojo.Api/Startup.cs index cb2c2e3..2e384fa 100644 --- a/src/TopoMojo.Api/Startup.cs +++ b/src/TopoMojo.Api/Startup.cs @@ -94,7 +94,6 @@ public void ConfigureServices(IServiceCollection services) // Configure Auth services.AddConfiguredAuthentication(Settings.Oidc); services.AddConfiguredAuthorization(); - } public void Configure(IApplicationBuilder app) diff --git a/src/TopoMojo.Api/Structure/TopoMojoStartupExtensions.cs b/src/TopoMojo.Api/Structure/TopoMojoStartupExtensions.cs index bdec05e..9dce85a 100644 --- a/src/TopoMojo.Api/Structure/TopoMojoStartupExtensions.cs +++ b/src/TopoMojo.Api/Structure/TopoMojoStartupExtensions.cs @@ -10,6 +10,7 @@ using TopoMojo.Api.Data; using TopoMojo.Hypervisor; using TopoMojo.Api.Services; +using TopoMojo.Hypervisor.Proxmox; namespace Microsoft.Extensions.DependencyInjection { @@ -135,7 +136,15 @@ Func podConfig } else { - services.AddSingleton(); + if (config.HypervisorType == HypervisorType.Proxmox) + { + // give proxmox Random.Shared since it's not directly available in netstandard2.0 + services.AddProxmoxHypervisor(Random.Shared); + } + else + { + services.AddSingleton(); + } } services.AddSingleton(sp => config); diff --git a/src/TopoMojo.Api/appsettings.conf b/src/TopoMojo.Api/appsettings.conf index 49445ea..600ad35 100644 --- a/src/TopoMojo.Api/appsettings.conf +++ b/src/TopoMojo.Api/appsettings.conf @@ -94,6 +94,7 @@ ## Hypervisor #################### +# Pod_HypervisorType = Vsphere # Pod__DebugVerbose = false ## Example Url: https://vcenter.local or https://esxi[1-4].local (supports ranges) @@ -106,6 +107,7 @@ ## credentials for user # Pod__User = # Pod__Password = +# Pod__AccessToken = ## Example PoolPath: "datacenter/cluster/pool" (uses first-found for any empty segments # Pod__PoolPath = @@ -156,6 +158,11 @@ # Pod__Sddc__AuthUrl = # Pod__Sddc__AuthTokenHeader = csp-auth-token +## these settings currently only apply to Proxmox (not vSphere) +## Set how long to wait for more network create/delete calls before reloading networking in Proxmox +# Pod__Vlan__ResetDebounceDuration = 2000 +# Pod__Vlan__ResetDebounceMaxDuration = 5000 + #################### ## Logging #################### @@ -206,3 +213,16 @@ # Logging__LogLevel__Microsoft.Hosting.Lifetime = Information # Logging__LogLevel__Microsoft = Warning + +################### +## Proxmox Example. See docs/Proxmox.md for details. +################### + +# Pod__HypervisorType = Proxmox +# Pod__Password = changeme +# Pod__AccessToken = root@pam!Topo=4c4fbe1e-b31e-55a9-9fg0-2de4a411cd23 +# Pod__Host = pve1.local +# Pod__SDNZone = topomojo + +# FileUpload__IsoRoot = /mnt/isos/template/iso +# FileUpload__SupportsSubFolders = false diff --git a/src/TopoMojo.Hypervisor/Common/DateTimeOffsetRange.cs b/src/TopoMojo.Hypervisor/Common/DateTimeOffsetRange.cs new file mode 100644 index 0000000..978f530 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Common/DateTimeOffsetRange.cs @@ -0,0 +1,10 @@ +using System; + +namespace TopoMojo.Hypervisor.Common +{ + public sealed class DateTimeOffsetRange + { + public DateTimeOffset Start { get; set; } + public DateTimeOffset End { get; set; } + } +} diff --git a/src/TopoMojo.Hypervisor/Common/DebouncePool.cs b/src/TopoMojo.Hypervisor/Common/DebouncePool.cs new file mode 100644 index 0000000..a608970 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Common/DebouncePool.cs @@ -0,0 +1,132 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Threading.Tasks; +using System.Threading; + +namespace TopoMojo.Hypervisor.Common +{ + public sealed class DebouncePool + { + private string _id; + private DateTimeOffsetRange _currentDebounce = null; + private readonly object _currentDebounceLock = new object(); + private readonly SemaphoreSlim _semaphoreLock = new SemaphoreSlim(1); + private readonly ConcurrentBag _items = new ConcurrentBag(); + + public DebouncePool() + { + DebouncePeriod = 0; + } + + public DebouncePool(int debounceDurationMs) + { + DebouncePeriod = debounceDurationMs; + } + + public DebouncePool(int debounceDurationMs, int maxTotalDebounceDurationMs) + { + DebouncePeriod = debounceDurationMs; + MaxTotalDebounce = maxTotalDebounceDurationMs; + } + + public int DebouncePeriod { get; set; } + public int? MaxTotalDebounce { get; set; } = null; + + public Task> Add(T item, CancellationToken cancellationToken) + => AddRange(new T[] { item }, cancellationToken); + + public async Task> AddRange(IEnumerable items, CancellationToken cancellationToken) + { + // add the item to the collection immediately (independent of debounce settings) + foreach (var item in items.ToArray()) + { + _items.Add(item); + } + + var nowish = DateTimeOffset.UtcNow; + lock (_currentDebounceLock) + { + if (_currentDebounce is null) + { + // start a new debounce if there's not one currently in the hopper + _currentDebounce = new DateTimeOffsetRange + { + Start = nowish, + End = nowish.AddMilliseconds(this.DebouncePeriod) + }; + + _id = Guid.NewGuid().ToString(); + } + else + { + // if there's a current debounce happening, refresh the period length (e.g. if the debounce period is 300ms and an item is added 250ms after the last one, + // the debounce timer should reset to 300ms after the second item is added) + _currentDebounce.End = nowish.AddMilliseconds(this.DebouncePeriod); + + // BUT if there's a maximum total debounce time, we have to ensure that we don't overflow it, so clamp the value to the maximum remaining if it would + if (this.MaxTotalDebounce.HasValue) + { + var maxDebounceEnds = _currentDebounce.Start.AddMilliseconds(this.MaxTotalDebounce.Value); + if (maxDebounceEnds < _currentDebounce.End) + { + _currentDebounce.End = maxDebounceEnds; + } + } + } + } + + // after waiting for the appropriate delay, return the contents of the collection + try + { + await _semaphoreLock.WaitAsync(cancellationToken); + + if (_currentDebounce != null) + { + int delayLength; + do + { + + // lock on the current debounce when we read its `.End`, because + // another thread could be writing it (above) + lock (_currentDebounceLock) + { + delayLength = (int)Math.Ceiling((_currentDebounce.End - DateTimeOffset.UtcNow).TotalMilliseconds); + } + + if (delayLength > 0) + await Task.Delay(delayLength, cancellationToken); + } + while (delayLength > 0); + } + + var itemsThreadSafe = Array.Empty(); + + lock (_currentDebounceLock) + { + _currentDebounce = null; + + // get a new array that points to the contents of _items for thread safety + itemsThreadSafe = this._items.ToArray(); + + // clear the collection (.Clear() isn't supported in .netstandard2.0) + foreach (var item in _items) + { + _items.TryTake(out _); + } + } + + return new DebouncePoolBatch + { + Id = _id, + Items = itemsThreadSafe + }; + } + finally + { + _semaphoreLock.Release(); + } + } + } +} diff --git a/src/TopoMojo.Hypervisor/Common/DebouncePoolBatch.cs b/src/TopoMojo.Hypervisor/Common/DebouncePoolBatch.cs new file mode 100644 index 0000000..91b6016 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Common/DebouncePoolBatch.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; + +namespace TopoMojo.Hypervisor +{ + public sealed class DebouncePoolBatch + { + public string Id { get; set; } + public IEnumerable Items { get; set; } + } +} diff --git a/src/TopoMojo.Hypervisor/HypervisorServiceConfiguration.cs b/src/TopoMojo.Hypervisor/HypervisorServiceConfiguration.cs index 329185a..81ef31f 100644 --- a/src/TopoMojo.Hypervisor/HypervisorServiceConfiguration.cs +++ b/src/TopoMojo.Hypervisor/HypervisorServiceConfiguration.cs @@ -5,20 +5,23 @@ namespace TopoMojo.Hypervisor { - public class HypervisorServiceConfiguration { + public class HypervisorServiceConfiguration + { public bool IsVCenter { get; set; } + public HypervisorType HypervisorType { get; set; } public string Type { get; set; } - public string Url { get; set;} - public string Host { get; set;} + public string Url { get; set; } + public string Host { get; set; } public string User { get; set; } public string Password { get; set; } + public string AccessToken { get; set; } public string PoolPath { get; set; } public string Uplink { get; set; } = "dvs-topomojo"; public string VmStore { get; set; } = "[topomojo] _run/"; public string DiskStore { get; set; } = "[topomojo]"; public string IsoStore { get; set; } = "[topomojo] iso/"; - public string TicketUrlHandler { get; set; } = "querystring"; //"local-app", "external-domain", "host-map", "none" - public Dictionary TicketUrlHostMap { get; set; } = new Dictionary(); + public string TicketUrlHandler { get; set; } = "querystring"; //"local-app", "external-domain", "host-map", "none" + public Dictionary TicketUrlHostMap { get; set; } = new Dictionary(); public VlanConfiguration Vlan { get; set; } = new VlanConfiguration(); public int KeepAliveMinutes { get; set; } = 10; public string ExcludeNetworkMask { get; set; } = "topomojo"; @@ -26,6 +29,7 @@ public class HypervisorServiceConfiguration { public bool IsNsxNetwork { get; set; } public bool DebugVerbose { get; set; } public bool IgnoreCertificateErrors { get; set; } + public string SDNZone { get; set; } = "topomojo"; public SddcConfiguration Sddc { get; set; } = new SddcConfiguration(); } @@ -46,7 +50,9 @@ public class SddcConfiguration public class VlanConfiguration { public string Range { get; set; } = ""; - public Vlan[] Reservations { get; set; } = new Vlan[] {}; + public Vlan[] Reservations { get; set; } = new Vlan[] { }; + public int ResetDebounceDuration { get; set; } = 2000; + public int? ResetDebounceMaxDuration { get; set; } = 5000; } public class Vlan diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/CreatePveVnet.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/CreatePveVnet.cs new file mode 100644 index 0000000..7c30408 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/CreatePveVnet.cs @@ -0,0 +1,13 @@ +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + public sealed class CreatePveVnet + { + public string Alias { get; set; } + public string Zone { get; set; } + + /// + /// An integer tag will be automatically generated during creation if this isn't set + /// + public int? Tag { get; set; } = null; + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveIso.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveIso.cs new file mode 100644 index 0000000..16696bd --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveIso.cs @@ -0,0 +1,27 @@ +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + public class PveIso + { + public string Volid { get; set; } + public string Format { get; set; } + public string Content { get; set; } + public int Ctime { get; set; } + public long Size { get; set; } + + public string Name + { + get + { + return this.Volid.Split('/')[1]; + } + } + + public string DisplayName + { + get + { + return this.Name.Replace('#', '/'); + } + } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveNic.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveNic.cs new file mode 100644 index 0000000..7125180 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveNic.cs @@ -0,0 +1,15 @@ +namespace TopoMojo.Hypervisor.Proxmox +{ + public sealed class PveNic + { + public int Index { get; set; } + public string MacAddress { get; set; } + + /// + /// Proxmox supports several "models" for network devices (e.g. virtio, Intel E1000, VMWare vmxnet3, etc.) + /// Proxmox stores the model along with the current network bridge the device is using in the format + /// {MODEL=MAC_ADDRESS,BRIDGE=PROXMOX_BRIDGE_ID} on the Extension properties of the VM. + /// + public string PveModel { get; set; } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveNodeTask.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveNodeTask.cs new file mode 100644 index 0000000..c94120e --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveNodeTask.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Dynamic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using System.Xml; +using Corsinvest.ProxmoxVE.Api; + +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + internal class PveNodeTask + { + public string Id { get; set; } + public string Action { get; set; } + public int Progress { get; private set; } + + // The Line Number that was last processed to get the current Progress + public int? LastLine { get; private set; } = null; + + public DateTimeOffset WhenCreated { get; set; } + + public void SetProgress(PveNodeTaskLog log) + { + // Hack to ensure topo ui keeps polling until task is removed + if (log.Progress > 99) + { + Progress = 99; + } + else + { + Progress = log.Progress; + } + + var lastEntry = log.Entries.LastOrDefault(); + + if (lastEntry != null) + { + LastLine = Convert.ToInt32(lastEntry.LineNumber); + } + } + + public void SetProgress(Result result) + { + var exitStatus = result.Response.data.exitstatus; + + if (exitStatus == "OK") + { + Progress = 99; + } + else + { + Progress = -1; + } + } + } + + internal class PveNodeTaskLog + { + public PveNodeTaskLog(Result result) + { + if (!result.RequestResource.Contains("/tasks/")) + throw new ArgumentException(); + + List data = result.ToData(); + Entries = data.Select(x => new PveNodeTaskLogEntry(x as ExpandoObject)).ToArray(); + SetProgress(); + } + + public PveNodeTaskLogEntry[] Entries { get; private set; } + + public int Progress { get; private set; } + + private void SetProgress() + { + // Tasks do not have a progress property so we have to parse the text output + // of the task to find progress. The output varies for different types of tasks, + // so we look for any text in the format of a number ending with a % character. + // e.g. "drive-scsi0: transferred 25.8 GiB of 32.0 GiB (80.40%) in 2m 12s" + // would return 80% + double preciseProgress = 0; + foreach (var entry in Entries.OrderByDescending(x => x.LineNumber)) + { + var matches = Regex.Matches(entry.Text, @"(\d+(\.\d+)?%)"); + foreach (Match match in matches) + { + if (double.TryParse(match.Value.TrimEnd('%'), out preciseProgress)) + { + Progress = Convert.ToInt32(preciseProgress); + return; + } + } + } + } + } + + internal class PveNodeTaskLogEntry + { + public PveNodeTaskLogEntry(ExpandoObject obj) + { + LineNumber = ((dynamic)obj).n; + Text = ((dynamic)obj).t; + } + + [JsonPropertyName("n")] + public long LineNumber { get; set; } + [JsonPropertyName("t")] + public string Text { get; set; } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveVmConfig.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVmConfig.cs new file mode 100644 index 0000000..5cecd94 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVmConfig.cs @@ -0,0 +1,19 @@ + +using System; +using System.Collections.Generic; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public sealed class PveVmConfig + { + public string Boot { get; set; } + public int Cores { get; set; } + public string Cpu { get; set; } + public string Digest { get; set; } + public IEnumerable Nics { get; set; } = Array.Empty(); + public string OsType { get; set; } + public long MemoryInBytes { get; set; } + public string Smbios1 { get; set; } + public int Sockets { get; set; } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveVmUpdateConfig.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVmUpdateConfig.cs new file mode 100644 index 0000000..d537fb3 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVmUpdateConfig.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public sealed class PveVmUpdateConfig + { + public IDictionary NetAssignments { get; private set; } = new Dictionary(); + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnet.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnet.cs new file mode 100644 index 0000000..acbbf6c --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnet.cs @@ -0,0 +1,11 @@ +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + public class PveVnet + { + public int Tag { get; set; } + public string Type { get; set; } + public string Vnet { get; set; } + public string Zone { get; set; } + public string Alias { get; set; } + } +} \ No newline at end of file diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperation.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperation.cs new file mode 100644 index 0000000..452a6fa --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperation.cs @@ -0,0 +1,36 @@ +using System; + +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + internal sealed class PveVnetOperation : IEquatable + { + public PveVnetOperationType Type { get; private set; } = PveVnetOperationType.Create; + public string NetworkName { get; private set; } = string.Empty; + + public PveVnetOperation(string networkName, PveVnetOperationType type) + { + NetworkName = networkName; + Type = type; + } + + public override bool Equals(object obj) + { + if (obj.GetType() != typeof(PveVnetOperation)) + return false; + + var typedObj = obj as PveVnetOperation; + + return Equals(typedObj); + } + + public bool Equals(PveVnetOperation other) + { + return + other.NetworkName == this.NetworkName && + other.Type == this.Type; + } + + public override int GetHashCode() + => (NetworkName + Type.ToString()).GetHashCode(); + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperationResult.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperationResult.cs new file mode 100644 index 0000000..e930b76 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperationResult.cs @@ -0,0 +1,14 @@ +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + public sealed class PveVnetOperationResult + { + public string NetName { get; set; } + public PveVnet Vnet { get; set; } + public PveVnetOperationType Type { get; set; } = PveVnetOperationType.Create; + + public override string ToString() + { + return $"{NetName} :: {Vnet.Zone}.{Vnet.Alias} :: {Type}"; + } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperationType.cs b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperationType.cs new file mode 100644 index 0000000..10f997b --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/Models/PveVnetOperationType.cs @@ -0,0 +1,8 @@ +namespace TopoMojo.Hypervisor.Proxmox.Models +{ + public enum PveVnetOperationType + { + Create, + Delete + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxClient.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxClient.cs new file mode 100644 index 0000000..bfba684 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxClient.cs @@ -0,0 +1,1025 @@ +// Copyright 2021 Carnegie Mellon University. All Rights Reserved. +// Released under a 3 Clause BSD-style license. See LICENSE.md in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Corsinvest.ProxmoxVE.Api; +using Corsinvest.ProxmoxVE.Api.Extension; +using Corsinvest.ProxmoxVE.Api.Shared.Models.Cluster; +using Corsinvest.ProxmoxVE.Api.Shared.Models.Vm; +using TopoMojo.Hypervisor.Proxmox.Models; +using TopoMojo.Hypervisor.Extensions; +using Corsinvest.ProxmoxVE.Api.Shared.Models.Node; +using System.Text; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public class ProxmoxClient + { + public ProxmoxClient( + HypervisorServiceConfiguration options, + ConcurrentDictionary vmCache, + ILogger logger, + IProxmoxNameService nameService, + IProxmoxVlanManager vnetService, + Random random + ) + { + _logger = logger; + _config = options; + _logger.LogDebug($"Constructing Client {_config.Url}"); + _tasks = new Dictionary(); + _vmCache = vmCache; + _random = random; + + if (_config.Tenant == null) + _config.Tenant = ""; + + int port = 443; + string host = _config.Url; + if (Uri.TryCreate(_config.Url, UriKind.RelativeOrAbsolute, out Uri result) && result.IsAbsoluteUri) { + host = result.Host; + port = result.Port; + } + _config.Host = host; + _hostPrefix = host.Split('.').FirstOrDefault(); + + _pveClient = new PveClient(host, port) + { + ApiToken = options.AccessToken + }; + + _rootPveClient = new PveClient(host, port); + + _nameService = nameService; + _vlanManager = vnetService; + Task sessionMonitorTask = MonitorSession(); + Task taskMonitorTask = MonitorTasks(); + } + + private readonly ILogger _logger; + Dictionary _tasks; + private ConcurrentDictionary _vmCache; + private readonly IProxmoxNameService _nameService; + private readonly IProxmoxVlanManager _vlanManager; + private readonly HypervisorServiceConfiguration _config = null; + // int _pollInterval = 1000; + int _syncInterval = 30000; + int _taskMonitorInterval = 3000; + string _hostPrefix = ""; + // DateTimeOffset _lastAction; + private PveClient _pveClient; + private PveClient _rootPveClient; + private readonly Random _random; + private readonly bool _enableHA = false; + private readonly Object _lock = new object(); + private const string deleteTag = "delete"; // tags are always lower-case + + public async Task Refresh(VmTemplate template) + { + string target = template.Name + "#" + template.IsolationTag; + var resources = await _pveClient.GetResourcesAsync(ClusterResourceType.Vm); + + var pveVm = resources.Where(x => x.Name == _nameService.ToPveName(template.Name)).FirstOrDefault(); + + if (pveVm != null) + { + return new Vm + { + Name = _nameService.FromPveName(pveVm.Name), + Id = pveVm.VmId.ToString(), + State = pveVm.IsRunning ? VmPowerState.Running : VmPowerState.Off + }; + } + else + { + // todo: check for in progress cloning of parent template + if (resources.Where(x => x.Name == template.Template).Any()) + { + return new Vm + { + Name = target, + Status = "initialized" + }; + } + else + { + return new Vm + { + Name = target, + Status = "created" + }; + } + } + } + + public async Task CreateTemplate(VmTemplate template) + { + // await this.ReloadVmCache(); + + var vmTemplate = _vmCache + .Where(x => x.Value.Name == template.Template) + .FirstOrDefault() + .Value; + + if (vmTemplate != null) + { + throw new InvalidOperationException("Template already exists"); + } + + var parentTemplate = _vmCache + .Where(x => x.Value.Name == template.ParentTemplate) + .FirstOrDefault() + .Value; + + if (parentTemplate == null) + { + throw new InvalidOperationException("Parent Template does not exist"); + } + + var nextId = await GetNextId(); + var pveId = Int32.Parse(nextId); + var name = _nameService.ToPveName(template.Template); + + // full clone parent template + var task = await _pveClient.Nodes[parentTemplate.Host].Qemu[parentTemplate.Id].Clone.CloneVm( + pveId, + full: true, + name: name, + target: parentTemplate.Host); + await _pveClient.WaitForTaskToFinish(task); + + if (task.IsSuccessStatusCode) + { + // convert new vm to template + task = await _pveClient.Nodes[parentTemplate.Host].Qemu[nextId].Template.Template(); + await _pveClient.WaitForTaskToFinish(task); + } + else + { + throw new Exception(task.ReasonPhrase); + } + + Vm vm = new Vm() + { + Name = name, + Id = nextId, + State = VmPowerState.Off, + Status = "deployed", + Host = parentTemplate.Host, + HypervisorType = HypervisorType.Proxmox + }; + + _vmCache.AddOrUpdate(vm.Id, vm, (k, v) => (v = vm)); + + return vm; + } + + public async Task Deploy(VmTemplate template) + { + Result task; + Vm vm = null; + + _logger.LogDebug("deploy: create vm..."); + var targetNode = await GetTargetNode(); + var vmTemplate = _vmCache + .Where(x => x.Value.Name == template.Template && + (x.Value.Tags == null || !x.Value.Tags.Contains(deleteTag))) + .FirstOrDefault() + .Value; + + var nextId = await GetNextId(); + var pveId = Int32.Parse(nextId); + + var cloneTask = _pveClient.Nodes[vmTemplate.Host].Qemu[vmTemplate.Id].Clone.CloneVm( + pveId, + full: false, + name: _nameService.ToPveName(template.Name), + target: targetNode); + + _logger.LogDebug($"deploy: virtual networks (id {template.Id})..."); + var vnetsTask = _vlanManager.Provision(template.Eth.Select(n => n.Net)); + var isoTask = this.GetIso(template); + + // We can clone vm and provision networks concurrently since we don't set the network until + // the configure step after the clone is finished. Isos are also not dependent on the other tasks. + try + { + await Task.WhenAll(cloneTask, vnetsTask, isoTask); + } + catch (Exception ex) + { + if (cloneTask.IsFaulted) + { + throw ex; + } + } + + task = cloneTask.Result; + + if (!vnetsTask.IsFaulted) + { + _logger.LogDebug($"deploy: {vnetsTask.Result.Count()} networks deployed."); + } + + await _pveClient.WaitForTaskToFinish(task); + + if (task.IsSuccessStatusCode) + { + if (isoTask.IsFaulted || vnetsTask.IsFaulted) + { + var exceptions = new List(); + + if (isoTask.IsFaulted) + { + exceptions.Add(isoTask.Exception); + } + + if (vnetsTask.IsFaulted) + { + exceptions.Add(vnetsTask.Exception); + } + + var destroyTask = await _pveClient.Nodes[targetNode].Qemu[nextId].DestroyVm(); + await _pveClient.WaitForTaskToFinish(destroyTask); + + throw new AggregateException($"Exception deploying {template.Name}", exceptions); + } + + // Proxmox requires using the root user account directly (not an access token) + // for setting the args field to add arbitrary arguments to the QEMU command line. This is currently the only + // way to set the fw_cfg property that we need for Guest Settings. + // A Patch is available to add direct fw_cfg support but has not been merged into a release. + // https://bugzilla.proxmox.com/show_bug.cgi?id=4068 + // If no root password is provided, skip setting args. + var client = _pveClient; + var setGuestSettings = false; + + if (!string.IsNullOrEmpty(_config.Password)) + { + if (await _rootPveClient.LoginAsync("root", _config.Password)) + { + client = _rootPveClient; + setGuestSettings = true; + } + else + { + _logger.LogError("Error logging in with root password. Skipping Guest Settings."); + } + } + else + { + _logger.LogDebug("No root password provided. Skipping Guest Settings"); + } + + var nics = await this.GetNics(template); + var memory = this.GetMemory(template); + var sockets = this.GetSockets(template); + var coresPerSocket = this.GetCoresPerSocket(template); + string args = null; + + if (setGuestSettings) + { + args = this.GetArgs(template); + } + + task = await client.Nodes[targetNode].Qemu[nextId].Config.UpdateVmAsync( + netN: nics, + memory: memory, + sockets: sockets, + cores: coresPerSocket, + cdrom: isoTask.Result, + args: args); + await _pveClient.WaitForTaskToFinish(task); + + if (!task.IsSuccessStatusCode) + { + _logger.LogError($"Error reconfiguring vm {template.Name} ({nextId}) - {task.ReasonPhrase}"); + var destroyTask = await _pveClient.Nodes[targetNode].Qemu[nextId].DestroyVm(); + await _pveClient.WaitForTaskToFinish(destroyTask); + + throw new Exception(task.ReasonPhrase); + } + + _logger.LogDebug("deploy: load vm..."); + + vm = new Vm() + { + Name = template.Name, + Id = nextId, + State = VmPowerState.Off, + Status = "deployed", + Host = targetNode, + HypervisorType = HypervisorType.Proxmox + }; + + if (vm.Name.Contains("#").Equals(false) || vm.Name.ToTenant() != _config.Tenant) + return null; + + _vmCache.AddOrUpdate(vm.Id, vm, (k, v) => v = vm); + + if (_enableHA) + { + task = await _pveClient.Cluster.Ha.Resources.Create(nextId); + } + + if (template.AutoStart && task.IsSuccessStatusCode) + { + _logger.LogDebug("deploy: start vm..."); + vm = await Start(vm.Id); + } + } + else + { + throw new Exception(task.ReasonPhrase); + } + + return vm; + } + + public async Task Start(string id) + { + Vm vm = _vmCache[id]; + + var task = await _pveClient.Nodes[vm.Host].Qemu[vm.GetId()].Status.Start.VmStart(); + await _pveClient.WaitForTaskToFinish(task); + + if (task.IsSuccessStatusCode) + { + vm.State = VmPowerState.Running; + } + else + { + throw new Exception(task.ReasonPhrase); + } + + _vmCache.TryUpdate(vm.Id, vm, vm); + + return vm; + } + + public async Task Stop(string id) + { + Vm vm = _vmCache[id]; + + var task = await _pveClient.Nodes[vm.Host].Qemu[vm.GetId()].Status.Stop.VmStop(); + await _pveClient.WaitForTaskToFinish(task); + + if (task.IsSuccessStatusCode) + { + vm.State = VmPowerState.Off; + } + else + { + throw new Exception(task.ReasonPhrase); + } + + _vmCache.TryUpdate(vm.Id, vm, vm); + + return vm; + } + + public async Task DeleteTemplate(string templateName) + { + Vm vm = _vmCache.Where(x => x.Value.Name == templateName).FirstOrDefault().Value; + + if (vm == null) + return null; + + return await Delete(vm.Id); + } + + public async Task Delete(string id) + { + Result task; + var pveId = long.Parse(id); + Vm vm = _vmCache[id]; + var status = await _pveClient.Nodes[vm.Host].Qemu[pveId].Status.Current.GetAsync(); + + if (_enableHA) + { + task = await _pveClient.Cluster.Ha.Resources[pveId].Delete(); + await _pveClient.WaitForTaskToFinish(task); + } + + if (status.IsRunning) + { + task = await _pveClient.ChangeStatusVmAsync(pveId, VmStatus.Stop); + await _pveClient.WaitForTaskToFinish(task); + } + + task = await _pveClient.Nodes[vm.Host].Qemu[id].DestroyVm(); + await _pveClient.WaitForTaskToFinish(task); + + if (task.IsSuccessStatusCode) + { + vm.Status = "initialized"; + } + else + { + throw new Exception(task.ReasonPhrase); + } + + // Don't set vm to the result here, because if we get unlucky and the sync task removed + // this vm from the cache first, we'll get a null value, which will cause errors in the + // calling method + _vmCache.TryRemove(vm.Id, out _); + + return vm; + } + + public async Task> GetTicket(string id) + { + string url; + string ticket; + var vm = _vmCache[id]; + + var result = await _pveClient.Nodes[vm.Host].Qemu[id].Vncproxy.Vncproxy(websocket: true); + + if (result.IsSuccessStatusCode) + { + string urlFragment = $"/api2/json/nodes/{vm.Host}/qemu/{id}/vncwebsocket?port={result.Response.data.port}&vncticket={WebUtility.UrlEncode(result.Response.data.ticket)}"; + url = $"wss://{_config.Host}{urlFragment}"; + ticket = result.Response.data.ticket; + } + else + { + throw new Exception(result.GetError()); + } + + return new Tuple(url, ticket); + } + + public async Task Save(string id) + { + Vm vm = _vmCache[id]; + + if (vm != null) + { + var config = await _pveClient.Nodes[vm.Host].Qemu[vm.Id].Config.GetAsync(); + + var disk = config.Disks.ElementAt(0); + var storageItems = await _pveClient.Nodes[vm.Host].Storage[disk.Storage].Content.GetAsync(); + + var pveDisk = storageItems.Where(x => disk.FileName == x.FileName).FirstOrDefault(); + + if (pveDisk != null) + { + var parent = storageItems.Where(x => x.FileName == pveDisk.GetParentFilename()).FirstOrDefault(); + + if (parent != null && parent.Parent == null) + { + // check if anything else is using the template + var count = storageItems.Where(x => x.Parent != null && x.GetParentFilename() == parent.FileName).Count(); + + if (count > 1) + { + throw new InvalidOperationException("Base Template is in use"); + } + + var template = _vmCache[parent.VmId.ToString()]; + + if (template != null) + { + // Full Clone vm + var nextId = Int32.Parse(await this.GetNextId()); + var task = await _pveClient.Nodes[vm.Host].Qemu[vm.Id].Clone.CloneVm(newid: nextId, full: true, target: template.Host, name: template.Name); + + var t = new PveNodeTask { Id = task.Response.data, Action = "saving", WhenCreated = DateTimeOffset.UtcNow }; + vm.Task = new VmTask { Name = "saving", WhenCreated = DateTime.UtcNow, Progress = t.Progress }; + _tasks.Add(vm.Id, t); + + _ = CompleteSave(task, id, nextId, template, vm.Id); + } + } + } + } + + return vm; + } + + private async Task CompleteSave(Result task, string oldId, int nextId, Vm template, string vmId) + { + try + { + await _pveClient.WaitForTaskToFinish(task); + + if (!task.IsSuccessStatusCode) + throw new Exception($"Clone failed: {task.ReasonPhrase}"); + + // Convert to template + task = await _pveClient.Nodes[template.Host].Qemu[nextId].Template.Template(); + await _pveClient.WaitForTaskToFinish(task); + + if (!task.IsSuccessStatusCode) + { + var destroyTask = await _pveClient.Nodes[template.Host].Qemu[nextId].DestroyVm(); + await _pveClient.WaitForTaskToFinish(destroyTask); + throw new Exception($"Convert to template failed: {task.ReasonPhrase}"); + } + + // Delete old vm + await this.Delete(oldId); + + // Tag old template + // Janitor will delete anything with this tag if deletion fails now + task = await _pveClient + .Nodes[template.Host] + .Qemu[template.Id] + .Config + .UpdateVmAsync(tags: deleteTag); + await _pveClient.WaitForTaskToFinish(task); + + if (!task.IsSuccessStatusCode) + throw new Exception($"Rename old template failed: {task.ReasonPhrase}"); + + // delete old template + task = await _pveClient.Nodes[template.Host].Qemu[template.Id].DestroyVm(); + await _pveClient.WaitForTaskToFinish(task); + + if (!task.IsSuccessStatusCode) + throw new Exception($"Delete old template failed: {task.ReasonPhrase}"); + + await this.ReloadVmCache(); + } + finally + { + _tasks.Remove(vmId); + } + } + + public async Task GetFiles() + { + var node = await this.GetRandomNode(); + + var task = await _pveClient + .Nodes[node] + .Storage[_config.IsoStore] + .Content + .Index(content: "iso"); + await _pveClient.WaitForTaskToFinish(task); + + var isos = task.ToModel(); + + return isos; + } + + public Task GetVmConfig(Vm vm) + => GetVmConfig(vm.Host, vm.GetId()); + + public async Task GetVmConfig(string node, long vmId) + { + var vmConfig = await _pveClient + .Nodes[node] + .Qemu[vmId] + .Config + .GetAsync(true); + + var nics = new List(); + + // our proxmox package stuffs NIC info into the ExtensionData property of + // the config call, so we rely on the fact that (current) proxmox documentation + // says that NICs start with "net" and are followed by a number + var nicDataRegex = new Regex(@"net(\d)+"); + var nicModelMacRegex = new Regex(@"(?[^=]+)=(?[0-9A-Fa-f:]{17})"); + + if (vmConfig.ExtensionData.Any(d => nicDataRegex.IsMatch(d.Key))) + { + foreach (var extensionItem in vmConfig.ExtensionData) + { + var match = nicDataRegex.Match(extensionItem.Key); + + if (match.Success && int.TryParse(match.Groups[1].Value, out var nicIndex)) + { + var modelMacMatch = nicModelMacRegex.Match(extensionItem.Value.ToString()); + + if (modelMacMatch.Success) + { + nics.Add(new PveNic + { + Index = nicIndex, + MacAddress = modelMacMatch.Groups["mac"].Value, + PveModel = modelMacMatch.Groups["model"].Value + }); + } + } + } + } + + return new PveVmConfig + { + Boot = vmConfig.Boot, + Cores = vmConfig.Cores, + Cpu = vmConfig.Cpu, + Nics = nics, + OsType = vmConfig.OsType, + MemoryInBytes = vmConfig.Memory, + Smbios1 = vmConfig.Smbios1, + Sockets = vmConfig.Sockets + }; + } + + public async Task PushVmConfigUpdate(long vmId, PveVmUpdateConfig update) + { + var vm = _vmCache[vmId.ToString()]; + var currentConfig = await this.GetVmConfig(vm); + + // if there are any net assignment updates, we need to resolve their IDs from the names + // passed in. We make a new dictionary to hold them rather than mutate the argument. + var vnetAssignmentIds = new Dictionary(); + + if (update.NetAssignments.Any()) + { + var vnets = await _vlanManager.GetVnets(); + + foreach (var netUpdate in update.NetAssignments) + { + var resolvedName = _vlanManager.ResolvePveNetName(netUpdate.Value); + var vnet = vnets.FirstOrDefault(v => v.Alias == _vlanManager.ResolvePveNetName(netUpdate.Value)) + ?? throw new Exception($"Couldn't resolve an ID for virtual network {netUpdate.Value}"); + + var nic = currentConfig.Nics.SingleOrDefault(n => n.Index == netUpdate.Key) + ?? throw new Exception($"Couldn't resolve a NIC on the host machine with index {netUpdate.Key}."); + + var updateValue = $"{nic.PveModel}={nic.MacAddress},bridge={vnet.Vnet}"; + vnetAssignmentIds.Add(netUpdate.Key, updateValue); + } + } + + var updateTask = await _pveClient + .Nodes[vm.Host] + .Qemu[vm.Id] + .Config + .UpdateVmAsync + ( + netN: vnetAssignmentIds.Any() ? vnetAssignmentIds : null + ); + + await _pveClient.WaitForTaskToFinish(updateTask); + + if (!updateTask.IsSuccessStatusCode) + { + throw new Exception($"VM Id {vmId}: failed to push update to the VM. The API returned a failed status code ({updateTask.StatusCode}): {updateTask.ReasonPhrase}"); + } + + return vm; + } + + /// + /// Selects a Node to deploy to. Randomly picks among all online nodes with less than 50% memory usage, or the + /// Node with the least memory usage if none are less than 50%. + /// + /// + private async Task GetTargetNode() + { + string target = null; + var nodes = await _pveClient.GetNodesAsync(); + + if (nodes.Count() > 0) + { + IClusterResourceNode targetNode; + var targetNodes = nodes.Where(x => + x.IsOnline && + x.MemoryUsagePercentage <= 50); + + if (targetNodes.Any()) + { + targetNode = targetNodes.ElementAt(_random.Next(0, targetNodes.Count() - 1)); + } + else + { + targetNode = nodes + .OrderBy(x => x.MemoryUsagePercentage) + .Where(x => x.IsOnline) + .FirstOrDefault(); + } + + target = targetNode.Node; + } + + return target; + } + + private async Task GetRandomNode() + { + var nodes = await _pveClient.GetNodesAsync(); + var randomNum = _random.Next(0, nodes.Count() - 1); + return nodes.ElementAt(randomNum).Node; + } + + private string GetArgs(VmTemplate template) + { + if (!template.GuestSettings.Any()) + return null; + + var args = new StringBuilder(); + + // Default settings + // TODO: Fix duplication with vsphere? + args.Append($"-fw_cfg name=opt/guestinfo.isolationTag,string={template.IsolationTag} "); + args.Append($"-fw_cfg name=opt/guestinfo.templateSource,string={template.Id} "); + args.Append($"-fw_cfg name=opt/guestinfo.hostname,string={template.Name.Untagged()} "); + + foreach (var setting in template.GuestSettings) + { + // TODO: rework this quick fix for injecting isolation specific settings + if (setting.Key.StartsWith("iftag.") && !setting.Value.Contains(template.IsolationTag)) + { + continue; + } + + args.Append($"-fw_cfg name=opt/{setting.Key},string={setting.Value} "); + } + + return args.ToString().TrimEnd(); + } + + private async Task GetIso(VmTemplate template) + { + var isos = await this.GetFiles(); + + var iso = isos + .Where(x => x.Volid == template.Iso) + .FirstOrDefault(); + + if (iso != null) + { + return iso.Volid; + } + else + { + return null; + } + } + + private string GetMemory(VmTemplate template) + { + return ((template.Ram > 0) ? template.Ram * 1024 : 1024).ToString(); + } + + private int? GetCoresPerSocket(VmTemplate template) + { + string[] p = template.Cpu.Split('x'); + int coresPerSocket = 1; + + if (p.Length > 1) + { + if (!Int32.TryParse(p[1], out coresPerSocket)) + { + coresPerSocket = 1; + } + } + + return coresPerSocket; + } + + private int? GetSockets(VmTemplate template) + { + string[] p = template.Cpu.Split('x'); + var sockets = 1; + + if (!Int32.TryParse(p[0], out sockets)) + { + sockets = 1; + } + + return sockets; + } + + private async Task GetNextId() + { + string nextId = null; + + for (int i = 0; i < 10; i++) + { + var randomId = _random.Next(1, 999999999); + var task = await _pveClient.Cluster.Nextid.Nextid(randomId); + + if (task.IsSuccessStatusCode) + { + nextId = task.Response.data; + break; + } + } + + return nextId; + } + + private async Task> GetNics(VmTemplate template) + { + Dictionary nics = new Dictionary(); + + if (template.Eth.IsEmpty()) + return nics; + + var vnets = await _vlanManager.GetVnets(); + + for (int i = 0; i < template.Eth.Length; i++) + { + var eth = template.Eth[i]; + + var vnet = vnets.Where(x => x.Alias == _nameService.ToPveName(eth.Net)).FirstOrDefault(); + if (vnet != null) + { + var netString = new StringBuilder(); + netString.Append(eth.Type); + + if (!string.IsNullOrEmpty(eth.Mac)) + { + netString.Append($"={eth.Mac.ToUpper()}"); + } + + nics.Add(i, $"{netString},bridge={vnet.Vnet}"); + } + } + + return nics; + } + + private Vm LoadVm(IClusterResourceVm pveVm) + { + Vm vm = new Vm() + { + Name = pveVm.Name == null ? "" : _nameService.FromPveName(pveVm.Name), + Id = pveVm.VmId.ToString(), + State = pveVm.IsRunning ? VmPowerState.Running : VmPowerState.Off, + Status = "deployed", + Host = pveVm.Node, + Tags = pveVm.Tags == null ? new string[] { } : pveVm.Tags.Split(' '), + HypervisorType = HypervisorType.Proxmox + }; + + if (_tasks.ContainsKey(vm.Id)) + { + var t = _tasks[vm.Id]; + vm.Task = new VmTask { Name = t.Action, WhenCreated = t.WhenCreated, Progress = t.Progress }; + } + + // Proxmox Vm names are null for a few seconds when first deployed. + // We still want to add to cache when in this state. + if (!pveVm.IsTemplate && !string.IsNullOrEmpty(vm.Name) && vm.Name.Contains("#").Equals(false) || + vm.Name.ToTenant() != _config.Tenant) + return null; + + _vmCache.AddOrUpdate(vm.Id, vm, (k, v) => v = vm); + + return vm; + } + + private async Task ReloadVmCache() + { + List existing = _vmCache.Values + .Select(o => o.Id) + .ToList(); + + List list = new List(); + + var pveVms = await _pveClient.GetVmsAsync(); + + //iterate through the collection of Vm's + foreach (var pveVm in pveVms) + { + Vm vm = LoadVm(pveVm); + + if (vm != null) + { + list.Add(vm); + } + } + + List active = list.Select(o => o.Id).ToList(); + _logger.LogDebug($"refreshing cache [{_config.Host}] existing: {existing.Count} active: {active.Count}"); + + foreach (string key in existing.Except(active)) + { + if (_vmCache.TryRemove(key, out Vm stale)) + { + _logger.LogDebug($"removing stale cache entry [{_config.Host}] {stale.Name}"); + } + } + + //return an array of vm's + return list.ToArray(); + } + + private async Task MonitorSession() + { + _logger.LogDebug($"{_config.Host}: starting cache loop"); + await _vlanManager.Initialize(); + int step = 0; + + while (true) + { + try + { + await ReloadVmCache(); + if (step == 0) + { + await _vlanManager.Clean(_vmCache); + await DeleteUnusedTemplates(); + } + } + catch (Exception ex) + { + _logger.LogError(0, ex, $"Failed to refresh cache for {_config.Host}"); + } + finally + { + await Task.Delay(_syncInterval); + } + + step = (step + 1) % 2; + } + // _logger.LogDebug("sessionMonitor ended."); + } + + private async Task DeleteUnusedTemplates() + { + var tasks = new List(); + + foreach (var taggedVm in _vmCache) + { + if (taggedVm.Value.Tags.Contains(deleteTag)) + { + _logger.LogInformation($"Deleting vm with deleteTag: {taggedVm.Value?.Name} ({taggedVm.Key})"); + tasks.Add(this.Delete(taggedVm.Key)); + } + } + + if (tasks.Count > 0) + { + await Task.WhenAll(tasks); + } + } + + private async Task MonitorTasks() + { + _logger.LogDebug($"{_config.Host}: starting task monitor"); + while (true) + { + try + { + foreach (string key in _tasks.Keys.ToArray()) + { + var t = _tasks[key]; + var info = await _pveClient.Nodes[PveClientBase.GetNodeFromTask(t.Id)] + .Tasks[t.Id] + .Status + .ReadTaskStatus(); + + var nodeTask = info.ToModel(); + + switch (nodeTask.Status) + { + case "running": + var taskLog = await _pveClient + .Nodes[PveClientBase.GetNodeFromTask(t.Id)] + .Tasks[t.Id] + .Log + .ReadTaskLog(start: t.LastLine); + + var log = new PveNodeTaskLog(taskLog); + t.SetProgress(log); + break; + + case "stopped": + t.SetProgress(info); + //_tasks.Remove(key); + break; + + default: + t.SetProgress(info); + //_tasks.Remove(key); + break; + } + + if (_vmCache.ContainsKey(key)) + { + Vm vm = _vmCache[key]; + if (vm.Task == null) + vm.Task = new VmTask(); + vm.Task.Progress = t.Progress; + vm.Task.Name = t.Action; + _vmCache.TryUpdate(key, vm, vm); + } + } + } + catch (Exception ex) + { + _logger.LogError(0, ex, $"Error in task monitor of {_config.Host}"); + } + finally + { + await Task.Delay(_taskMonitorInterval); + } + } + // _logger.LogDebug("taskMonitor ended."); + } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxExtensions.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxExtensions.cs new file mode 100644 index 0000000..72d6643 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxExtensions.cs @@ -0,0 +1,118 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Corsinvest.ProxmoxVE.Api; +using Corsinvest.ProxmoxVE.Api.Shared.Models.Node; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public static class ProxmoxExtensions + { + public static long GetId(this Vm vm) + { + return long.Parse(vm.Id); + } + + public static string GetParentFilename(this NodeStorageContent content) + { + if (string.IsNullOrEmpty(content.Parent)) + { + return null; + } + + // raw format parent: base--disk-@__base__ + if (content.Parent.Contains('@')) + { + // return e.g. base-100-disk-0 + return content.Parent.Split('@')[0]; + } + // qcow2 format: ..//base--disk-.qcow2 + else if (content.Parent.StartsWith("../")) + { + // return e.g. 100/base-100-disk0.qcow2 + return content.Parent.Split(new[] { '/' }, 2)[1]; + } + else + { + throw new InvalidOperationException("Unsupported NodeStorageContent type"); + } + } + + /// + /// Adds support for the Proxmox hypervisor to Topomojo. + /// + /// The app's service collection. + /// + /// An instance of Random which will be used across the hypervisor's implementation. Where available, + /// The thread-safe Random.Shared instance is recommended. If no instance is supplied, a default + /// will be created. + /// + /// + public static IServiceCollection AddProxmoxHypervisor(this IServiceCollection services, Random random = null) + { + return services + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(_ => random ?? new Random()); + } + + /// + /// Wait for task to finish. + /// + /// Modification of the Corsinvest implementation that adds additional check + /// that the passed in result actually contains a task id to avoid throwing errors. + /// https://github.com/Corsinvest/cv4pve-api-dotnet/blob/master/src/Corsinvest.ProxmoxVE.Api/PveClientBase.cs + /// TODO: Make PR to their repo + /// + /// The result representing the task to wait for + /// + /// + /// + public static async Task WaitForTaskToFinish(this PveClient client, Result result, int wait = 2000, long timeout = 3600 * 1000) + => !(result != null && + !result.ResponseInError && + timeout > 0 && + result.ToData() is string && + ((string)result.ToData()).StartsWith("UPID:")) || + await WaitForTaskToFinish(client, result.ToData(), wait, timeout); + + /// + /// Wait for task to finish. + /// + /// Modification of Corsinvest implementation that swaps Thread.Sleep with Task.Delay + /// and fixes timeout. + /// TODO: Make PR to their repo + /// + /// Task identifier + /// Millisecond wait next check + /// Millisecond timeout + /// + public static async Task WaitForTaskToFinish(this PveClient client, string task, int wait = 2000, long timeout = 3600 * 1000) + { + var isRunning = true; + if (wait <= 0) { wait = 500; } + if (timeout < wait) { timeout = wait + 5000; } + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + while (isRunning && stopwatch.ElapsedMilliseconds < timeout) + { + await Task.Delay(wait); + isRunning = await client.TaskIsRunningAsync(task); + } + + stopwatch.Stop(); + + //check timeout + return stopwatch.ElapsedMilliseconds < timeout; + } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs new file mode 100644 index 0000000..b1996a9 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs @@ -0,0 +1,494 @@ +// Copyright 2021 Carnegie Mellon University. All Rights Reserved. +// Released under a 3 Clause BSD-style license. See LICENSE.md in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using TopoMojo.Hypervisor.Extensions; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public class ProxmoxHypervisorService : IHypervisorService + { + public ProxmoxHypervisorService( + HypervisorServiceConfiguration options, + IProxmoxNameService nameService, + IProxmoxVlanManager vlanManager, + ILoggerFactory mill, + Random random + ) + { + _options = options; + _mill = mill; + _logger = _mill.CreateLogger(); + // _hostCache = new ConcurrentDictionary(); + // _affinityMap = new Dictionary(); + _vlanManager = vlanManager; + _vmCache = new ConcurrentDictionary(); + + NormalizeOptions(_options); + + _pveClient = new ProxmoxClient( + options, + _vmCache, + _mill.CreateLogger(), + nameService, + vlanManager, + random); + } + + private readonly HypervisorServiceConfiguration _options; + + private readonly ILogger _logger; + private readonly ILoggerFactory _mill; + //private ConcurrentDictionary _hostCache; + private DateTimeOffset _lastCacheUpdate = DateTimeOffset.MinValue; + //private Dictionary _affinityMap; + private readonly ConcurrentDictionary _vmCache; + private readonly ProxmoxClient _pveClient; + private readonly IProxmoxVlanManager _vlanManager; + + public HypervisorServiceConfiguration Options { get { return _options; } } + + public async Task Deploy(VmTemplate template, bool privileged = false) + { + var vm = await LoadVm(template.Name + "#" + template.IsolationTag); + if (vm != null) + return vm; + + _logger.LogDebug("deploy: host " + _options.Host); + NormalizeTemplate(template, Options, privileged); + _logger.LogDebug("deploy: normalized " + template.Name); + + _logger.LogDebug("deploy: " + template.Name + " " + Options.Host); + return await _pveClient.Deploy(template); + } + + public async Task> Deploy(IEnumerable templates, bool privileged = false) + { + var virtualNetworks = templates + .SelectMany(t => t.Eth) + .Select(eth => eth.Net) + .ToArray(); + var vms = new List(); + var undeployedTemplates = new List(); + + foreach (var template in templates) + { + var vm = await LoadVm(template.Name + "#" + template.IsolationTag); + if (vm is null) + { + _logger.LogDebug("deploy: host " + _options.Host); + NormalizeTemplate(template, Options, privileged); + _logger.LogDebug("deploy: normalized " + template.Name); + + undeployedTemplates.Add(template); + } + + _logger.LogDebug($"deploy (host: {Options.Host}, templates: {undeployedTemplates.Count}): {string.Join(",", undeployedTemplates.Select(t => t.Name).ToArray())}"); + _logger.LogDebug("deploy: " + template.Name + " " + Options.Host); + vms.Add(await _pveClient.Deploy(template)); + } + + return vms; + } + + public async Task GetVmNetOptions(string id) + { + var hostVnets = await _vlanManager.GetVnets(); + + return new VmOptions { Net = hostVnets.Select(n => n.Alias).ToArray() }; + } + + public string Version + { + get + { + return "TopoMojo Pod Manager for Proxmox, v1.0.0"; + } + } + + private void NormalizeTemplate(VmTemplate template, HypervisorServiceConfiguration option, bool privileged = false) + { + if (!template.Iso.HasValue()) + { + // need to have a backing file to add the cdrom device + template.Iso = option.IsoStore + "null.iso"; + } + + var isoPath = template.Iso.Replace('/', '#'); + isoPath = $"{option.IsoStore.Replace("/", String.Empty)}:iso/{isoPath}"; + template.Iso = isoPath; + + foreach (VmDisk disk in template.Disks) + { + if (!disk.Path.StartsWith(option.DiskStore) + ) + { + DatastorePath dspath = new DatastorePath(disk.Path); + dspath.Merge(option.DiskStore); + disk.Path = dspath.ToString(); + } + + if (disk.Source.HasValue() && !disk.Source.StartsWith(option.DiskStore) + ) + { + DatastorePath dspath = new DatastorePath(disk.Source); + dspath.Merge(option.DiskStore); + disk.Source = dspath.ToString(); + } + } + + if (template.IsolationTag.HasValue()) + { + var tag = "#" + template.IsolationTag; + var rgx = new Regex("#.*"); + + if (!template.Name.EndsWith(template.IsolationTag)) + template.Name = rgx.Replace(template.Name, "") + tag; + + foreach (var requestedNetwork in template.Eth) + { + if (privileged && _vlanManager.IsReserved(requestedNetwork.Net)) + continue; + + requestedNetwork.Net = rgx.Replace(requestedNetwork.Net, "") + tag; + } + } + } + + public async Task Delete(string id) + { + _logger.LogDebug("deleting " + id); + Vm vm = await LoadVm(id); + return await _pveClient.Delete(vm.Id); + } + + public async Task Display(string id) + { + var info = new VmConsole(); + + try + { + var vm = await LoadVm(id); + + info = new VmConsole + { + Id = vm.Id, + Name = vm.Name.Untagged(), + IsolationId = vm.Name.Tag(), + IsRunning = vm.State == VmPowerState.Running + }; + + // throws if powered off + var ticket = await _pveClient.GetTicket(GetId(vm.Id)); + info.Url = ticket.Item1; + info.Ticket = ticket.Item2; + + } + catch { } + + return info; + } + + private string GetId(string id) + { + var pveId = id.Split('/').Last(); + return pveId; + } + + protected class HostVmCount + { + public string Name { get; set; } + public int Count { get; set; } + } + + private void NormalizeOptions(HypervisorServiceConfiguration options) + { + var regex = new Regex("(]|/)$"); + + if (!regex.IsMatch(options.VmStore)) + options.VmStore += "/"; + + if (!regex.IsMatch(options.DiskStore)) + options.DiskStore += "/"; + + if (!regex.IsMatch(options.IsoStore)) + options.IsoStore += "/"; + } + + public async Task Load(string id) + { + return await LoadVm(id, false); + } + + private Task LoadVm(string id, bool returnNull = true) + { + Vm vm = _vmCache.Values.Where(o => o.Id == id || o.Name == id).FirstOrDefault(); + + if (vm == null && !returnNull) + { + vm = new Vm() + { + Id = null, + HypervisorType = HypervisorType.Proxmox + }; + } + + return Task.FromResult(vm); + } + + private void CheckProgress(Vm vm) + { + if (vm.Task != null && (vm.Task.Progress < 0 || vm.Task.Progress > 99)) + { + vm.Task = null; + _vmCache.TryUpdate(vm.Id, vm, vm); + } + } + + private Vm[] CheckProgress(Vm[] vms) + { + foreach (Vm vm in vms) + CheckProgress(vm); + + return vms; + } + + public async Task Start(string id) + { + var vm = await LoadVm(id); + return await _pveClient.Start(vm.Id); + } + + public async Task Stop(string id) + { + var vm = await LoadVm(id); + return await _pveClient.Stop(vm.Id); + } + + public async Task Save(string id) + { + var vm = await LoadVm(id); + return await _pveClient.Save(vm.Id); + } + + public Task Revert(string id) + { + throw new NotImplementedException(); + } + + public async Task StartAll(string target) + { + _logger.LogDebug("starting all matching " + target); + var tasks = new List(); + foreach (var vm in await Find(target)) + { + tasks.Add(Start(vm.Id)); + } + + if (tasks.Count > 0) + { + await Task.WhenAll(tasks.ToArray()); + } + } + + public async Task StopAll(string target) + { + _logger.LogDebug("stopping all matching " + target); + var tasks = new List(); + foreach (var vm in await Find(target)) + { + tasks.Add(Stop(vm.Id)); + } + + if (tasks.Count > 0) + { + await Task.WhenAll(tasks.ToArray()); + } + } + + public async Task DeleteAll(string target) + { + _logger.LogDebug("deleting all matching " + target); + var tasks = new List(); + + foreach (var vm in await Find(target)) + { + tasks.Add(this.Delete(vm.Id)); + } + + if (tasks.Count > 0) + { + await Task.WhenAll(tasks); + } + } + + public async Task ChangeState(VmOperation op) + { + Vm vm = null; + var id = GetId(op.Id); + switch (op.Type) + { + case VmOperationType.Start: + vm = await Start(op.Id); + break; + + case VmOperationType.Reset: + vm = await Stop(op.Id); + vm = await Start(op.Id); + break; + + case VmOperationType.Stop: + vm = await Stop(op.Id); + break; + + case VmOperationType.Save: + vm = await Save(id); + break; + + case VmOperationType.Revert: + vm = await Revert(op.Id); + break; + + case VmOperationType.Delete: + vm = await Delete(id); + break; + } + + return vm; + } + + public async Task ChangeConfiguration(string id, VmKeyValue change, bool privileged = false) + { + if (!long.TryParse(id, out var vmId)) + { + throw new ArgumentException($"Couldn't parse virtual machine ID to a long.", nameof(id)); + } + + var configUpdate = new PveVmUpdateConfig(); + + switch (change.Key) + { + case "net": + // for NIC/network changes, the value contains the (topo) name of a virtual network. + // topo may also append a colon and the the zero-based index of the NIC to target, so + // we need to check if we're being asked to target a specific NIC. Defaults to the first + // NIC if not. + var nicIndex = 0; + var delimitedValue = change.Value.Split(':'); + var netName = change.Value; + + if (delimitedValue.Length == 2) + { + if (int.TryParse(delimitedValue[1], out nicIndex)) + { + netName = delimitedValue[0]; + } + } + + configUpdate.NetAssignments[nicIndex] = netName.Trim(); + break; + default: + throw new NotImplementedException($"Updating configuration property '{change.Key}' is not supported on Proxmox."); + } + + return await _pveClient.PushVmConfigUpdate(vmId, configUpdate); + } + + public Task SetAffinity(string isolationTag, Vm[] vms, bool start) + { + throw new NotImplementedException(); + } + + public async Task Refresh(VmTemplate template) + { + string target = template.Name + "#" + template.IsolationTag; + var vm = await LoadVm(target); + + if (vm == null) + { + if (_vmCache.Where(x => x.Value.Name == template.Template).Any()) + { + return new Vm + { + Name = target, + Status = "initialized" + }; + } + else + { + return new Vm + { + Name = target, + Status = "created" + }; + } + } + + return vm; + } + + public Task Find(string term) + { + IEnumerable q = _vmCache.Values; + + if (term.HasValue()) + q = q.Where(o => o.Id.Contains(term) || o.Name.Contains(term)); + + return Task.FromResult(q.ToArray()); + } + + public async Task CreateDisks(VmTemplate template) + { + // Clone template + var vm = await _pveClient.CreateTemplate(template); + + if (vm != null) + { + return 0; + } + else + { + return 100; + } + } + + public Task VerifyDisks(VmTemplate template) + { + throw new NotImplementedException(); + } + + public async Task DeleteDisks(VmTemplate template) + { + await _pveClient.DeleteTemplate(template.Template); + } + + public Task Answer(string id, VmAnswer answer) + { + throw new NotImplementedException(); + } + + public async Task GetVmIsoOptions(string key) + { + var isos = await this._pveClient.GetFiles(); + + return new VmOptions + { + Iso = isos + .Where(x => x.Name.StartsWith(key) || x.Name.StartsWith(Guid.Empty.ToString())) + .Select(x => x.DisplayName) + .ToArray() + }; + } + + public Task ReloadHost(string host) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxNameService.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxNameService.cs new file mode 100644 index 0000000..bb07e32 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxNameService.cs @@ -0,0 +1,20 @@ +namespace TopoMojo.Hypervisor.Proxmox +{ + public interface IProxmoxNameService + { + string ToPveName(string name); + string FromPveName(string pveName); + } + + public class ProxmoxNameService : IProxmoxNameService + { + public bool IsPveName(string name) + => name.Contains("--"); + + public string ToPveName(string name) + => name.Replace("#", "--"); + + public string FromPveName(string pveName) + => pveName.Replace("--", "#"); + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxVlanManager.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxVlanManager.cs new file mode 100644 index 0000000..234cca3 --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxVlanManager.cs @@ -0,0 +1,334 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using TopoMojo.Hypervisor.Common; +using TopoMojo.Hypervisor.Extensions; +using TopoMojo.Hypervisor.Proxmox.Models; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public interface IProxmoxVlanManager + { + Task> DeleteVnets(IEnumerable vnetNames, bool force); + Task> DeleteVnetsByTerm(string term); + Task> GetVnets(); + bool IsReserved(string networkName); + Task> Provision(IEnumerable vnetNames); + string ResolvePveNetName(string topoName); + Task Clean(ConcurrentDictionary vmCache, string tag = null); + Task Initialize(); + } + + public class ProxmoxVlanManager : IProxmoxVlanManager + { + // ABOUT THE STATIC MEMBERS + // this service is currently placed into the DI container as a singleton, but I wanted to support the case that it ever becomes + // scoped. the static members are important to how the class works across all instances, where the individual ones are either + // injected or trivial + private readonly static Lazy _deploySemaphore = new Lazy(() => new SemaphoreSlim(1)); + // defaults to a debounce period of 300ms, but can be changed using the `Pod__Vnet__ResetDebounceDuration`. A maximum + // debounce can be set using `Pod__VNet__ResetDebounceMaxDuration`. + private readonly static Lazy> _vnetOpsPool = new Lazy>(() => new DebouncePool()); + private readonly static IMemoryCache _recentVnetOpsCache = new MemoryCache(new MemoryCacheOptions { }); + private readonly static IDictionary _reservedVnetIds = new Dictionary(); + private readonly static IMemoryCache _recentVnetCache = new MemoryCache(new MemoryCacheOptions { }); + + private readonly int _cacheDurationMs; + private readonly int _recentExpirationMinutes = 5; + private readonly int _lastReloadMaxMinutes = 30; + private readonly HypervisorServiceConfiguration _hypervisorOptions; + private readonly ILogger _logger; + private readonly IProxmoxNameService _nameService; + private readonly IProxmoxVnetsClient _vnetsApi; + + private DateTimeOffset _lastReload = DateTimeOffset.UtcNow; + + public ProxmoxVlanManager + ( + HypervisorServiceConfiguration hypervisorOptions, + ILogger logger, + IProxmoxNameService nameService, + IProxmoxVnetsClient vnetsApi + ) + { + _hypervisorOptions = hypervisorOptions; + _logger = logger; + _nameService = nameService; + _vnetsApi = vnetsApi; + + // update the debounce pool to use settings from config + _vnetOpsPool.Value.DebouncePeriod = _hypervisorOptions.Vlan.ResetDebounceDuration; + _vnetOpsPool.Value.MaxTotalDebounce = _hypervisorOptions.Vlan.ResetDebounceMaxDuration; + + // cache this - we need this to remain at least long as the maximum possible debounce (if it's defined). If it is, + // add a couple seconds for safety. if not, just double the min debounce up to a minimum of two seconds + _cacheDurationMs = _hypervisorOptions.Vlan.ResetDebounceMaxDuration != null ? + _hypervisorOptions.Vlan.ResetDebounceMaxDuration.Value + 2000 : + Math.Max(_hypervisorOptions.Vlan.ResetDebounceDuration * 2, 2000); + + // reserve the vlans specified as "global" in the application's config + Reserve(hypervisorOptions.Vlan.Reservations); + } + + /// + /// Delete the specified vnets + /// + /// + /// Force a reload check by continuing even with an empty list + /// + public async Task> DeleteVnets(IEnumerable vnetNames, bool force = false) + { + _logger.LogDebug($"Requested to delete vnets: {string.Join(",", vnetNames)}"); + vnetNames = vnetNames + .Where(n => !string.IsNullOrWhiteSpace(n)) + .Distinct(); + + if (!force && !vnetNames.Any()) + { + _logger.LogDebug($"No vnet names passed. Cancelling vnet delete."); + return Array.Empty(); + } + + // create the nets + var results = await DebounceVnetOperations(vnetNames.Select(name => new PveVnetOperation(name, PveVnetOperationType.Delete))); + + // the results contain all network operations performed this debounce, but we only want to send back the ones related to + // the requested names + var pveVnetNames = vnetNames.Select(name => _nameService.ToPveName(name)).ToArray(); + return results + .Where(r => pveVnetNames.Contains(r.Vnet.Alias)) + .Select(r => r.Vnet); + } + + public async Task> DeleteVnetsByTerm(string term) + { + var vnets = await _vnetsApi.GetVnets(); + var matchingVnetDeleteOps = vnets + .Where(v => v.Alias.Contains(term)) + .Select(v => new PveVnetOperation + ( + v.Alias, + PveVnetOperationType.Delete + )); + + var results = await this.DebounceVnetOperations(matchingVnetDeleteOps); + return results + .Where(r => r.NetName.Contains(term)) + .Select(r => r.Vnet); + } + + public Task> GetVnets() + => _vnetsApi.GetVnets(); + + public bool IsReserved(string networkName) + => _reservedVnetIds.ContainsKey(networkName); + + public async Task> Provision(IEnumerable vnetNames) + { + _logger.LogDebug($"Deploying vnets: {string.Join(",", vnetNames)}"); + var requestedVnetNames = vnetNames + .Where(n => !string.IsNullOrWhiteSpace(n)) + .Distinct(); + + if (!requestedVnetNames.Any()) + { + _logger.LogDebug($"No vnet names passed. Cancelling vnet deploy."); + return Array.Empty(); + } + + // create the nets + var results = await DebounceVnetOperations(requestedVnetNames.Select(name => new PveVnetOperation(name, PveVnetOperationType.Create))); + + // the results contain all network operations performed this debounce, but we only want to send back the ones related to + // the requested names + var pveVnetNames = requestedVnetNames.Select(name => _nameService.ToPveName(name)).ToArray(); + return results + .Where(r => pveVnetNames.Contains(r.Vnet.Alias)) + .Select(r => r.Vnet); + } + + public string ResolvePveNetName(string topoName) + => IsReserved(topoName) ? topoName : _nameService.ToPveName(topoName); + + public async Task Initialize() + { + _logger.LogDebug($"initializing nets"); + + var vnets = await _vnetsApi.GetVnets(); + + foreach (var vnet in vnets) + { + _logger.LogDebug($"Adding to recent vnet cache: {vnet.Alias}"); + AddToRecentCache(vnet); + } + } + + public async Task Clean(ConcurrentDictionary vmCache, string tag = null) + { + _logger.LogDebug($"cleaning nets [{tag}]"); + + var vnets = await _vnetsApi.GetVnets(); + var vnetsToDelete = new List(); + + if (!string.IsNullOrEmpty(tag)) + { + vnets = vnets.Where(x => _nameService.FromPveName(x.Alias).Tag() == tag); + } + + // exclude non-tagged + vnets = vnets.Where(x => _nameService.FromPveName(x.Alias).Contains('#')); + + // find portgroups with no associated vm's + foreach (var vnet in vnets) + { + string id = _nameService.FromPveName(vnet.Alias).Tag(); + + // if vm's still exist, skip + if (!vmCache.Values.Any(v => _nameService.FromPveName(v.Name).Tag() == id)) + { + vnetsToDelete.Add(vnet.Alias); + } + } + + await DeleteVnets(vnetsToDelete, force: true); + } + + private void AddToRecentCache(PveVnet vnet) + { + _recentVnetCache.Set( + vnet.Alias, + vnet.Vnet, + new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(_recentExpirationMinutes) + }); + } + + private void Reserve(IEnumerable vlans) + { + var vlanNames = vlans.Select(v => v.Name).Distinct().ToArray(); + + if (vlanNames.Length != vlans.Count()) + throw new InvalidOperationException($"Can't reserve virtual networks with duplicate names: {string.Join(",", vlans.Select(v => v.Name))}"); + + foreach (var vlan in vlans) + { + _reservedVnetIds.Add(vlan.Name, vlan.Id); + } + } + + private async Task> DebounceVnetOperations(IEnumerable requestedOperations) + { + var debouncedOperations = await _vnetOpsPool.Value.AddRange(requestedOperations, CancellationToken.None); + + // PveVnetOperation implements object comparison, so we can use .Distinct to ensure we don't ever + // try a duplicate op (at least not in the same debounce) + debouncedOperations.Items = debouncedOperations.Items.Distinct(); + + try + { + await _deploySemaphore.Value.WaitAsync(CancellationToken.None); + + // check the cache to see if this debounce batch has already been created. + // if so, just bail out and return what we already have + _logger.LogDebug($"Looking up id {debouncedOperations.Id}"); + if (_recentVnetOpsCache.TryGetValue>(debouncedOperations.Id, out var cachedOperations)) + { + return cachedOperations.Where(o => requestedOperations.Any(req => req.Equals(o))); + } + _logger.LogDebug($"Cache miss {debouncedOperations.Id}"); + + var results = new List(); + var vnetsToCreate = debouncedOperations.Items.Where(op => op.Type == PveVnetOperationType.Create).ToArray(); + var vnetsToDelete = debouncedOperations.Items.Where(op => op.Type == PveVnetOperationType.Delete).ToArray(); + + if (vnetsToCreate.Any()) + { + var vnetNamesToCreate = vnetsToCreate.Select(v => _nameService.ToPveName(v.NetworkName)); + var deployedVnets = await _vnetsApi.CreateVnets(vnetsToCreate.Select(n => new CreatePveVnet + { + Alias = _nameService.ToPveName(n.NetworkName), + Zone = _hypervisorOptions.SDNZone, + Tag = _reservedVnetIds.TryGetValue(n.NetworkName, out var reservedId) ? reservedId : default(int?) + })); + + // Add created vnets to recent cache + foreach (var vnet in deployedVnets) + { + AddToRecentCache(vnet); + } + + results.AddRange(deployedVnets.Select(v => new PveVnetOperationResult + { + NetName = _nameService.FromPveName(v.Alias), + Vnet = v, + Type = PveVnetOperationType.Create + })); + } + + if (vnetsToDelete.Any()) + { + // Only delete vnets that haven't been created recently to avoid accidentally deleting + // a vnet that is in use. Also, don't delete a vnet if we created it in this batch + var vnetNamesToDelete = vnetsToDelete + .Select(n => _nameService.ToPveName(n.NetworkName)) + .Where(x => + !vnetsToCreate.Any(y => y.NetworkName == x) && + !_recentVnetCache.TryGetValue(x, out _)); + + var deletedVnets = await _vnetsApi.DeleteVnets(vnetNamesToDelete); + + foreach (var deletedVnet in deletedVnets) + { + results.Add + ( + new PveVnetOperationResult + { + NetName = _nameService.FromPveName(deletedVnet.Alias), + Vnet = deletedVnet, + Type = PveVnetOperationType.Delete + }); + } + } + + _logger.LogDebug($"Batch {debouncedOperations.Id} results: {string.Join(",", results)}"); + + if (results.Any(x => + x.Type == PveVnetOperationType.Create) || + _lastReload.AddMinutes(_lastReloadMaxMinutes) < DateTimeOffset.UtcNow) + { + // because we're allowing creates/deletes in the same debounce pool and trying to minimize reload calls, + // we manually reload proxmox's vnets at the end of the batch + // skip reload if only delete operations - we'll catch them on the next reload + await _vnetsApi.ReloadVnets(); + _lastReload = DateTimeOffset.UtcNow; + + // cache the id of the debounce batch we just handled (so later callers won't try to recreate/redelete the vnets) + _recentVnetOpsCache + .GetOrCreate> + ( + debouncedOperations.Id, + entry => + { + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(_cacheDurationMs); + return results; + } + ); + + _logger.LogDebug($"Cached id {debouncedOperations.Id}"); + } + + return results; + } + finally + { + _deploySemaphore.Value.Release(); + } + } + } +} diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxVnetsClient.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxVnetsClient.cs new file mode 100644 index 0000000..07fc08f --- /dev/null +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxVnetsClient.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Corsinvest.ProxmoxVE.Api; +using Microsoft.Extensions.Logging; +using TopoMojo.Hypervisor.Proxmox.Models; + +namespace TopoMojo.Hypervisor.Proxmox +{ + public interface IProxmoxVnetsClient + { + Task> CreateVnets(IEnumerable createVnets); + Task> DeleteVnets(IEnumerable names); + Task> DeleteVnetsByTerm(string term); + Task> GetVnets(); + Task ReloadVnets(); + } + + public class ProxmoxVnetsClient : IProxmoxVnetsClient + { + private readonly ILogger _logger; + private readonly PveClient _pveClient; + private readonly Random _random; + private readonly string _sdnZone; + + public ProxmoxVnetsClient + ( + HypervisorServiceConfiguration hypervisorOptions, + ILogger logger, + Random random + ) + { + _logger = logger; + + int port = 443; + string host = hypervisorOptions.Url; + if (Uri.TryCreate(hypervisorOptions.Url, UriKind.RelativeOrAbsolute, out Uri result) && result.IsAbsoluteUri) + { + host = result.Host; + port = result.Port; + } + hypervisorOptions.Host = host; + + _pveClient = new PveClient(host, port) + { + ApiToken = hypervisorOptions.AccessToken + }; + _random = random; + _sdnZone = hypervisorOptions.SDNZone; + } + + public async Task> CreateVnets(IEnumerable createVnets) + { + var existingNets = await this.GetVnets(); + var deployedNets = new List(); + + foreach (var createVnet in createVnets) + { + if (existingNets.Any(n => n.Alias == createVnet.Alias)) + { + _logger.LogDebug($"Skipped creating vnet {createVnet} - it already exists."); + continue; + } + + var newVnetTag = createVnet.Tag; + + if (newVnetTag == null) + { + do + { + // VXLAN range should be 1 - 16777215, but some devices may reserve 1-4096 + newVnetTag = _random.Next(4097, 16777215); + } + while (existingNets.Any(n => n.Tag == newVnetTag)); + } + + var vnetId = this.GetRandomVnetId(); + + // check for existence of alias = vnetname--gamespaceid + var createTask = await _pveClient.Cluster.Sdn.Vnets.Create + ( + vnet: vnetId, + tag: newVnetTag, + zone: createVnet.Zone, + alias: createVnet.Alias + ); + + if (createTask.IsSuccessStatusCode) + { + + deployedNets.Add + ( + new PveVnet + { + Alias = createVnet.Alias, + Tag = newVnetTag.GetValueOrDefault(), + Type = string.Empty, + Vnet = vnetId, + Zone = createVnet.Zone + } + ); + } + } + + return deployedNets; + } + + public async Task> DeleteVnets(IEnumerable aliases) + { + _logger.LogDebug($"Deleting vnets: {string.Join(",", aliases)}"); + var vnets = await this.GetVnets(); + var deletedPveNets = new List(); + + foreach (var alias in aliases) + { + var vnet = vnets.SingleOrDefault(v => v.Alias == alias); + + if (vnet != null) + { + var deleteTask = await _pveClient.Cluster.Sdn.Vnets[vnet.Vnet].Delete(); + await _pveClient.WaitForTaskToFinish(deleteTask); + + if (deleteTask.IsSuccessStatusCode) + { + deletedPveNets.Add(vnet); + } + } + else + { + _logger.LogDebug($"Vnet delete requested for {alias}, but the network doesn't exist."); + } + } + + return deletedPveNets; + } + + public async Task> DeleteVnetsByTerm(string term) + { + var vnets = await this.GetVnets(); + var deletedPveNets = new List(); + + foreach (var vnet in vnets.Where(x => x.Alias.Contains(term))) + { + var deleteTask = await _pveClient.Cluster.Sdn.Vnets[vnet.Vnet].Delete(); + await _pveClient.WaitForTaskToFinish(deleteTask); + + if (deleteTask.IsSuccessStatusCode) + { + deletedPveNets.Add(vnet); + } + } + + return deletedPveNets; + } + + public async Task> GetVnets() + { + var task = await _pveClient.Cluster.Sdn.Vnets.Index(); + await _pveClient.WaitForTaskToFinish(task); + + if (!task.IsSuccessStatusCode) + throw new Exception($"Failed to load virtual networks from Proxmox. Status code: {task.StatusCode}"); + + return task.ToModel().Where(x => x.Zone == _sdnZone); + } + + public async Task ReloadVnets() + { + var reloadTask = await _pveClient.Cluster.Sdn.Reload(); + await _pveClient.WaitForTaskToFinish(reloadTask); + } + + private string GetRandomVnetId() + { + var builder = new StringBuilder(); + var _chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".ToCharArray(); + + for (int i = 0; i <= 7; i++) + { + builder.Append(_chars[_random.Next(_chars.Length)]); + } + + return builder.ToString(); + } + } +} diff --git a/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj b/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj index aba4b08..3e1b487 100755 --- a/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj +++ b/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj @@ -7,11 +7,12 @@ TopoMojo.vSphere disable - + + + - - + \ No newline at end of file diff --git a/src/TopoMojo.Hypervisor/Vm.cs b/src/TopoMojo.Hypervisor/Vm.cs index f92a84e..6374f95 100644 --- a/src/TopoMojo.Hypervisor/Vm.cs +++ b/src/TopoMojo.Hypervisor/Vm.cs @@ -16,9 +16,11 @@ public class Vm public string Stats { get; set; } public string Status { get; set; } public string GroupName { get; set; } + public string[] Tags { get; set; } public VmPowerState State { get; set; } public VmQuestion Question { get; set; } public VmTask Task { get; set; } + public HypervisorType HypervisorType { get; set; } = HypervisorType.Vsphere; } public enum VmPowerState { Off, Running, Suspended} @@ -79,6 +81,13 @@ public class VmConsole public string IsolationId { get; set; } public string Name { get; set; } public string Url { get; set; } + public string Ticket { get; set; } public bool IsRunning { get; set; } } + + public enum HypervisorType + { + Vsphere, + Proxmox + } } diff --git a/src/TopoMojo.Hypervisor/VmTemplate.cs b/src/TopoMojo.Hypervisor/VmTemplate.cs index 1b718e9..db5a745 100644 --- a/src/TopoMojo.Hypervisor/VmTemplate.cs +++ b/src/TopoMojo.Hypervisor/VmTemplate.cs @@ -17,13 +17,15 @@ public class VmTemplate public string Floppy { get; set; } public string Version { get; set; } public string IsolationTag { get; set; } - public bool HostAffinity {get; set; } - public bool UseUplinkSwitch {get; set; } + public bool HostAffinity { get; set; } + public bool UseUplinkSwitch { get; set; } public int Ram { get; set; } public int VideoRam { get; set; } public int Adapters { get; set; } public int Delay { get; set; } public bool AutoStart { get; set; } = true; + public string Template { get; set; } + public string ParentTemplate { get; set; } public VmNet[] Eth { get; set; } public VmDisk[] Disks { get; set; } public VmKeyValue[] GuestSettings { get; set; }