From 75c39f076f6b51797d78269e7c42dee53e0aacdf Mon Sep 17 00:00:00 2001 From: Jeff Mattson Date: Thu, 7 Nov 2024 20:01:05 -0500 Subject: [PATCH] support batch deployments --- src/TopoMojo.Api/AppSettings.cs | 1 + .../Features/Gamespace/GamespaceService.cs | 47 ++++++------ src/TopoMojo.Api/appsettings.conf | 1 + src/TopoMojo.Hypervisor/DeploymentContext.cs | 11 +++ src/TopoMojo.Hypervisor/IHypervisorService.cs | 2 +- .../Proxmox/ProxmoxHypervisorService.cs | 32 ++++++++ .../TopoMojo.Hypervisor.csproj | 5 +- .../vMock/MockHypervisorService.cs | 32 ++++++++ .../vSphere/INetworkManager.cs | 2 + .../vSphere/NetworkManager.cs | 74 +++++++++++-------- .../vSphere/NsxNetworkManager.cs | 47 ++++++++++++ src/TopoMojo.Hypervisor/vSphere/VimClient.cs | 5 ++ .../vSphere/vSphereHypervisorService.cs | 52 ++++++++++++- 13 files changed, 253 insertions(+), 58 deletions(-) create mode 100644 src/TopoMojo.Hypervisor/DeploymentContext.cs diff --git a/src/TopoMojo.Api/AppSettings.cs b/src/TopoMojo.Api/AppSettings.cs index dd9dda0..ea0c077 100644 --- a/src/TopoMojo.Api/AppSettings.cs +++ b/src/TopoMojo.Api/AppSettings.cs @@ -155,6 +155,7 @@ public class CoreOptions public int ReplicaLimit { get; set; } = 5; public bool AllowUnprivilegedVmReconfigure { get; set; } public bool AllowPrivilegedNetworkIsolationExemption { get; set; } + public bool WaitForDeployment { get; set; } = true; public string DefaultUserScope { get; set; } = "everyone"; public string GameEngineIsoFolder { get; set; } = "static"; public string ConsoleHost { get; set; } diff --git a/src/TopoMojo.Api/Features/Gamespace/GamespaceService.cs b/src/TopoMojo.Api/Features/Gamespace/GamespaceService.cs index aabcc39..7bd0a74 100644 --- a/src/TopoMojo.Api/Features/Gamespace/GamespaceService.cs +++ b/src/TopoMojo.Api/Features/Gamespace/GamespaceService.cs @@ -166,6 +166,9 @@ public async Task LoadChallengeProgress(string gamespaceI { var gamespaceEntity = await _store.Retrieve(gamespaceId); var spec = JsonSerializer.Deserialize(gamespaceEntity.Challenge, jsonOptions); + if (spec.Challenge is null) + return new(); + var mappedVariant = Mapper.Map(spec.Challenge).FilterSections(); // only include available question sets in the output viewmodel @@ -543,34 +546,28 @@ private async Task Deploy(TopoMojo.Api.Data.Gamespace gamespace, bool sudo = fal } } - foreach (var template in templates) - { - tasks.Add( - _pod.Deploy( - template + await _pod.Deploy(new DeploymentContext( + gamespace.Id, + gamespace.Workspace.HostAffinity, + sudo, + templates.Select(t => t .ToVirtualTemplate(gamespace.Id) - .SetHostAffinity(gamespace.Workspace.HostAffinity), - sudo - ) - ); - } - - await Task.WhenAll(tasks.ToArray()); - - if (gamespace.Workspace.HostAffinity) - { - var vms = tasks.Select(t => t.Result).ToArray(); + .SetHostAffinity(gamespace.Workspace.HostAffinity) + ).ToArray() + ), _options.WaitForDeployment); - await _pod.SetAffinity(gamespace.Id, vms, true); - - foreach (var vm in vms) - vm.State = VmPowerState.Running; - } - - if (gamespace.StartTime.Year <= 1) + for (int i = 0; i < 18 ; i++) { - gamespace.StartTime = DateTimeOffset.UtcNow; - await _store.Update(gamespace); + await Task.Delay(5000); + var existing = await _pod.Find(gamespace.Id); + if (existing.Length == templates.Count) { + if (gamespace.StartTime.Year <= 1) + { + gamespace.StartTime = DateTimeOffset.UtcNow; + await _store.Update(gamespace); + } + break; + } } } diff --git a/src/TopoMojo.Api/appsettings.conf b/src/TopoMojo.Api/appsettings.conf index 3143b60..2b34f37 100644 --- a/src/TopoMojo.Api/appsettings.conf +++ b/src/TopoMojo.Api/appsettings.conf @@ -83,6 +83,7 @@ # Core__LaunchUrl = /lp # Core__AllowUnprivilegedVmReconfigure = false # Core__AllowPrivilegedNetworkIsolationExemption = false +# Core__WaitForDeployment = true ## Cleanup tasks delete resources after periods with no activity # Core__Expirations__DryRun = true diff --git a/src/TopoMojo.Hypervisor/DeploymentContext.cs b/src/TopoMojo.Hypervisor/DeploymentContext.cs new file mode 100644 index 0000000..0c914da --- /dev/null +++ b/src/TopoMojo.Hypervisor/DeploymentContext.cs @@ -0,0 +1,11 @@ + +namespace TopoMojo.Hypervisor +{ + + public record DeploymentContext( + string Id, + bool Affinity, + bool Privileged, + VmTemplate[] Templates + ); +} diff --git a/src/TopoMojo.Hypervisor/IHypervisorService.cs b/src/TopoMojo.Hypervisor/IHypervisorService.cs index 8df7074..ef0deeb 100755 --- a/src/TopoMojo.Hypervisor/IHypervisorService.cs +++ b/src/TopoMojo.Hypervisor/IHypervisorService.cs @@ -19,6 +19,7 @@ public interface IHypervisorService Task ChangeState(VmOperation op); Task ChangeConfiguration(string id, VmKeyValue change, bool privileged = false); Task Deploy(VmTemplate template, bool privileged = false); + Task Deploy(DeploymentContext ctx, bool wait = false); Task SetAffinity(string isolationTag, Vm[] vms, bool start); Task Refresh(VmTemplate template); Task Find(string searchText); @@ -32,7 +33,6 @@ public interface IHypervisorService Task GetVmNetOptions(string key); string Version { get; } Task ReloadHost(string host); - HypervisorServiceConfiguration Options { get; } } diff --git a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs index b1996a9..7561d00 100644 --- a/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs +++ b/src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs @@ -39,6 +39,8 @@ Random random nameService, vlanManager, random); + + _ = Task.Run(() => DeploymentHandler()); } private readonly HypervisorServiceConfiguration _options; @@ -53,6 +55,8 @@ Random random private readonly IProxmoxVlanManager _vlanManager; public HypervisorServiceConfiguration Options { get { return _options; } } + private BlockingCollection DeploymentCollection = []; + public async Task Deploy(VmTemplate template, bool privileged = false) { @@ -490,5 +494,33 @@ public Task ReloadHost(string host) { throw new NotImplementedException(); } + + private Task DeploymentHandler() + { + foreach(var ctx in DeploymentCollection.GetConsumingEnumerable()) + _ = DeployBatch(ctx); + + return Task.CompletedTask; + } + + private async Task DeployBatch(DeploymentContext ctx) + { + var tasks = new List>(); + var existing = (await Find(ctx.Id)).Select(vm => vm.Name); + var missing = ctx.Templates.Where(t => existing.Contains(t.Name).Equals(false)); + + foreach (var template in missing) + tasks.Add(Deploy(template, ctx.Privileged)); + + await Task.WhenAll(tasks.ToArray()); + } + + public async Task Deploy(DeploymentContext ctx, bool wait = false) + { + if (wait) + await DeployBatch(ctx); + else + DeploymentCollection.Add(ctx); + } } } diff --git a/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj b/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj index 5c22c91..4bfb1fb 100755 --- a/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj +++ b/src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj @@ -1,18 +1,19 @@  - netstandard2.0 + net8.0 portable TopoMojo.vSphere TopoMojo.vSphere disable - + + diff --git a/src/TopoMojo.Hypervisor/vMock/MockHypervisorService.cs b/src/TopoMojo.Hypervisor/vMock/MockHypervisorService.cs index 7449f96..40b7416 100644 --- a/src/TopoMojo.Hypervisor/vMock/MockHypervisorService.cs +++ b/src/TopoMojo.Hypervisor/vMock/MockHypervisorService.cs @@ -2,6 +2,7 @@ // Released under a 3 Clause BSD-style license. See LICENSE.md in the project root for license information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; @@ -28,6 +29,7 @@ ILoggerFactory mill _rand = new Random(); NormalizeOptions(_optPod); + _ = Task.Run(() => DeploymentHandler()); } private readonly HypervisorServiceConfiguration _optPod; @@ -36,6 +38,7 @@ ILoggerFactory mill private Random _rand; private Dictionary _vms; private Dictionary _tasks; + private BlockingCollection DeploymentCollection = []; public HypervisorServiceConfiguration Options { get { return _optPod; } } @@ -527,6 +530,35 @@ private void NormalizeOptions(HypervisorServiceConfiguration options) if (!regex.IsMatch(options.IsoStore)) options.IsoStore += "/"; } + + private Task DeploymentHandler() + { + foreach (var ctx in DeploymentCollection.GetConsumingEnumerable()) + _ = DeployBatch(ctx); + + return Task.CompletedTask; + } + + private async Task DeployBatch(DeploymentContext ctx) + { + _logger.LogInformation("Deploying Batch {id}", ctx.Id); + var tasks = new List>(); + var existing = (await Find(ctx.Id)).Select(vm => vm.Name); + var missing = ctx.Templates.Where(t => existing.Contains(t.Name).Equals(false)); + + foreach (var template in missing) + tasks.Add(Deploy(template, ctx.Privileged)); + + await Task.WhenAll(tasks.ToArray()); + } + + public async Task Deploy(DeploymentContext ctx, bool wait = false) + { + if (wait) + await DeployBatch(ctx); + else + DeploymentCollection.Add(ctx); + } } public class MockDisk diff --git a/src/TopoMojo.Hypervisor/vSphere/INetworkManager.cs b/src/TopoMojo.Hypervisor/vSphere/INetworkManager.cs index 744ab27..baab0c0 100755 --- a/src/TopoMojo.Hypervisor/vSphere/INetworkManager.cs +++ b/src/TopoMojo.Hypervisor/vSphere/INetworkManager.cs @@ -13,6 +13,7 @@ public interface INetworkManager Task AddSwitch(string sw); Task RemoveSwitch(string sw); Task AddPortGroup(string sw, VmNet eth); + Task AddPortGroups(string sw, VmNet[] eths); Task RemovePortgroup(string pgReference); Task GetVmNetworks(ManagedObjectReference managedObjectReference); Task LoadPortGroups(); @@ -20,6 +21,7 @@ public interface INetworkManager Task Initialize(); Task Provision(VmTemplate template); + Task ProvisionAll(VmNet[] template, bool useUplinkSwitch); Task Unprovision(ManagedObjectReference vmMOR); Task Clean(string tag = null); string Resolve(string net); diff --git a/src/TopoMojo.Hypervisor/vSphere/NetworkManager.cs b/src/TopoMojo.Hypervisor/vSphere/NetworkManager.cs index 31d99f6..6af2968 100644 --- a/src/TopoMojo.Hypervisor/vSphere/NetworkManager.cs +++ b/src/TopoMojo.Hypervisor/vSphere/NetworkManager.cs @@ -84,48 +84,55 @@ public async Task Provision(VmTemplate template) { await Task.Delay(0); + ProvisionAll(template.Eth, template.UseUplinkSwitch).Wait(); + + foreach (var eth in template.Eth) + { + eth.Key = _pgAllocation[eth.Net].Key; + _pgAllocation[eth.Net].Counter += 1; + } + } + + public async Task ProvisionAll(VmNet[] nets, bool useUplinkSwitch) + { + await Task.Delay(0); + lock (_pgAllocation) { string sw = _client.UplinkSwitch; - if (_client.dvs == null && _client.net != null && !template.UseUplinkSwitch) + if (_client.dvs == null && _client.net != null && !useUplinkSwitch) { - sw = template.IsolationTag.ToSwitchName(); - if (!_swAllocation.ContainsKey(sw)) + sw = nets[0].Net.Tag().ToSwitchName(); + if (_swAllocation.TryAdd(sw, 0)) { AddSwitch(sw).Wait(); - _swAllocation.Add(sw, 0); } } - foreach (VmNet eth in template.Eth) - { - if (!_pgAllocation.ContainsKey(eth.Net)) - { - var pg = AddPortGroup(sw, eth).Result; - pg.Timestamp = DateTimeOffset.UtcNow; - pg.Counter = 1; - - _pgAllocation.Add(pg.Net, pg); - - _vlanManager.Activate(new Vlan[] { - new Vlan { - Id = pg.VlanId, - Name = pg.Net, - OnUplink = sw == _client.UplinkSwitch - } - }); + var manifest = nets + .Where(e => _pgAllocation.ContainsKey(e.Net).Equals(false)) + .Distinct() + .ToArray() + ; - if (_swAllocation.ContainsKey(sw)) - _swAllocation[sw] += 1; + var pgs = AddPortGroups(sw, manifest).Result; - } - else - { - _pgAllocation[eth.Net].Counter += 1; - } + _vlanManager.Activate( + pgs.Select(p => new Vlan { + Id = p.VlanId, + Name = p.Net, + OnUplink = sw == _client.UplinkSwitch + }).ToArray() + ); - eth.Key = _pgAllocation[eth.Net].Key; + foreach (var pg in pgs) + { + pg.Timestamp = DateTimeOffset.UtcNow; + _pgAllocation.Add(pg.Net, pg); } + + if (_swAllocation.ContainsKey(sw)) + _swAllocation[sw] += pgs.Length; } } @@ -223,6 +230,15 @@ private Dictionary GetKeyMap() public abstract Task GetVmNetworks(ManagedObjectReference managedObjectReference); public abstract Task LoadPortGroups(); public abstract Task AddPortGroup(string sw, VmNet eth); + public virtual async Task AddPortGroups(string sw, VmNet[] eths) + { + List pgs = []; + foreach (var eth in eths) + pgs.Add( + await AddPortGroup(sw, eth) + ); + return [.. pgs]; + } public abstract Task RemovePortgroup(string pgReference); public abstract Task AddSwitch(string sw); public abstract Task RemoveSwitch(string sw); diff --git a/src/TopoMojo.Hypervisor/vSphere/NsxNetworkManager.cs b/src/TopoMojo.Hypervisor/vSphere/NsxNetworkManager.cs index b968c43..a1042c2 100644 --- a/src/TopoMojo.Hypervisor/vSphere/NsxNetworkManager.cs +++ b/src/TopoMojo.Hypervisor/vSphere/NsxNetworkManager.cs @@ -169,6 +169,53 @@ public override async Task AddPortGroup(string sw, VmNet et } + public override async Task AddPortGroups(string sw, VmNet[] eths) + { + await InitClient(); + + string tag = eths[0].Net.Tag(); + var names = new List(); + + foreach (var eth in eths) + { + string url = $"{_apiUrl}/{_apiSegments}/{eth.Net.Replace("#","%23")}"; + + var response = await _sddc.PutAsync( + url, + new StringContent( + "{\"advanced_config\": { \"connectivity\": \"OFF\" } }", + Encoding.UTF8, + "application/json" + ) + ); + + if (response.IsSuccessStatusCode) + names.Add(eth.Net); + else + _logger.LogDebug("Failed to add SDDC PortGroup {net}", eth.Net); + } + + int count = 10; + bool complete = false; + PortGroupAllocation[] pgas = []; + do + { + await Task.Delay(1500); + + pgas = (await LoadPortGroups()) + .Where(p => names.Contains(p.Net)) + .ToArray() + ; + complete = pgas.Length == names.Count; + count -= 1; + } while (count > 0 && !complete); + + if (!complete) + throw new Exception($"Failed to create net(s) for {tag}"); + + return pgas; + } + public override Task AddSwitch(string sw) { return Task.FromResult(0); diff --git a/src/TopoMojo.Hypervisor/vSphere/VimClient.cs b/src/TopoMojo.Hypervisor/vSphere/VimClient.cs index f05dd64..aba1767 100644 --- a/src/TopoMojo.Hypervisor/vSphere/VimClient.cs +++ b/src/TopoMojo.Hypervisor/vSphere/VimClient.cs @@ -1386,6 +1386,11 @@ private async Task MonitorTasks() } // _logger.LogDebug("taskMonitor ended."); } + + internal async Task PreDeployNets(VmNet[] eths, bool useUplinkSwitch) + { + await _netman.ProvisionAll(eths, useUplinkSwitch); + } } } diff --git a/src/TopoMojo.Hypervisor/vSphere/vSphereHypervisorService.cs b/src/TopoMojo.Hypervisor/vSphere/vSphereHypervisorService.cs index c582436..2ef0a76 100644 --- a/src/TopoMojo.Hypervisor/vSphere/vSphereHypervisorService.cs +++ b/src/TopoMojo.Hypervisor/vSphere/vSphereHypervisorService.cs @@ -30,6 +30,7 @@ ILoggerFactory mill _vlanman = new VlanManager(_options.Vlan); NormalizeOptions(_options); + _ = Task.Run(() => DeploymentHandler()); } private readonly HypervisorServiceConfiguration _options; @@ -41,7 +42,7 @@ ILoggerFactory mill private DateTimeOffset _lastCacheUpdate = DateTimeOffset.MinValue; private Dictionary _affinityMap; private ConcurrentDictionary _vmCache; - + private BlockingCollection DeploymentCollection = new BlockingCollection(); public HypervisorServiceConfiguration Options { get {return _options;}} public async Task ReloadHost(string hostname) @@ -93,6 +94,7 @@ public async Task Refresh(VmTemplate template) return vm; } + public async Task Deploy(VmTemplate template, bool privileged = false) { @@ -698,6 +700,54 @@ private void NormalizeOptions(HypervisorServiceConfiguration options) if (!regex.IsMatch(options.IsoStore)) options.IsoStore += "/"; } + + private Task DeploymentHandler() + { + foreach(var ctx in DeploymentCollection.GetConsumingEnumerable()) + _ = DeployBatch(ctx); + + return Task.CompletedTask; + } + + private async Task DeployBatch(DeploymentContext ctx) + { + var existing = (await Find(ctx.Id)).Select(vm => vm.Name); + var missing = ctx.Templates + .Where(t => existing.Contains(t.Name).Equals(false)) + .ToArray() + ; + if (missing.Length == 0) + return; + + if (_hostCache.Count == 1 && _hostCache.First().Value.Options.IsNsxNetwork) + { + var eths = ctx.Templates.SelectMany(t => t.Eth).ToArray(); + foreach (var eth in eths) + eth.Net += $"#{ctx.Id}"; + await _hostCache.First().Value.PreDeployNets(eths, false); + } + + var tasks = missing.Select(t => Deploy(t, ctx.Privileged)).ToArray(); + await Task.WhenAll(tasks); + + if (ctx.Affinity) + { + var vms = tasks.Select(t => t.Result).ToArray(); + + await SetAffinity(ctx.Id, vms, true); + + foreach (var vm in vms) + vm.State = VmPowerState.Running; + } + } + + public async Task Deploy(DeploymentContext ctx, bool wait = false) + { + if (wait) + await DeployBatch(ctx); + else + DeploymentCollection.Add(ctx); + } } }