support batch deployments

src/TopoMojo.Api/AppSettings.cs

@@ -155,6 +155,7 @@ public class CoreOptions
         public int ReplicaLimit { get; set; } = 5;
         public bool AllowUnprivilegedVmReconfigure { get; set; }
         public bool AllowPrivilegedNetworkIsolationExemption { get; set; }
+        public bool WaitForDeployment { get; set; } = true;
         public string DefaultUserScope { get; set; } = "everyone";
         public string GameEngineIsoFolder { get; set; } = "static";
         public string ConsoleHost { get; set; }

src/TopoMojo.Api/Features/Gamespace/GamespaceService.cs

@@ -166,6 +166,9 @@ public async Task<ChallengeProgressView> LoadChallengeProgress(string gamespaceI
         {
             var gamespaceEntity = await _store.Retrieve(gamespaceId);
             var spec = JsonSerializer.Deserialize<ChallengeSpec>(gamespaceEntity.Challenge, jsonOptions);
+            if (spec.Challenge is null)
+                return new();
+
             var mappedVariant = Mapper.Map<VariantView>(spec.Challenge).FilterSections();
 
             // only include available question sets in the output viewmodel
@@ -543,34 +546,28 @@ private async Task Deploy(TopoMojo.Api.Data.Gamespace gamespace, bool sudo = fal
                 }
             }
 
-            foreach (var template in templates)
-            {
-                tasks.Add(
-                    _pod.Deploy(
-                        template
+            await _pod.Deploy(new DeploymentContext(
+                gamespace.Id,
+                gamespace.Workspace.HostAffinity,
+                sudo,
+                templates.Select(t => t
                         .ToVirtualTemplate(gamespace.Id)
-                        .SetHostAffinity(gamespace.Workspace.HostAffinity),
-                        sudo
-                    )
-                );
-            }
-
-            await Task.WhenAll(tasks.ToArray());
-
-            if (gamespace.Workspace.HostAffinity)
-            {
-                var vms = tasks.Select(t => t.Result).ToArray();
+                        .SetHostAffinity(gamespace.Workspace.HostAffinity)
+                ).ToArray()
+            ), _options.WaitForDeployment);
 
-                await _pod.SetAffinity(gamespace.Id, vms, true);
-
-                foreach (var vm in vms)
-                    vm.State = VmPowerState.Running;
-            }
-
-            if (gamespace.StartTime.Year <= 1)
+            for (int i = 0; i < 18 ; i++)
             {
-                gamespace.StartTime = DateTimeOffset.UtcNow;
-                await _store.Update(gamespace);
+                await Task.Delay(5000);
+                var existing = await _pod.Find(gamespace.Id);
+                if (existing.Length == templates.Count) {
+                    if (gamespace.StartTime.Year <= 1)
+                    {
+                        gamespace.StartTime = DateTimeOffset.UtcNow;
+                        await _store.Update(gamespace);
+                    }
+                    break;
+                }
             }
         }
 

src/TopoMojo.Api/appsettings.conf

@@ -83,6 +83,7 @@
 # Core__LaunchUrl = /lp
 # Core__AllowUnprivilegedVmReconfigure = false
 # Core__AllowPrivilegedNetworkIsolationExemption = false
+# Core__WaitForDeployment = true
 
 ## Cleanup tasks delete resources after periods with no activity
 # Core__Expirations__DryRun = true

src/TopoMojo.Hypervisor/DeploymentContext.cs

@@ -0,0 +1,11 @@
+
+namespace TopoMojo.Hypervisor
+{
+
+    public record DeploymentContext(
+        string Id,
+        bool Affinity,
+        bool Privileged,
+        VmTemplate[] Templates
+    );
+}

src/TopoMojo.Hypervisor/IHypervisorService.cs

@@ -19,6 +19,7 @@ public interface IHypervisorService
         Task<Vm> ChangeState(VmOperation op);
         Task<Vm> ChangeConfiguration(string id, VmKeyValue change, bool privileged = false);
         Task<Vm> Deploy(VmTemplate template, bool privileged = false);
+        Task Deploy(DeploymentContext ctx, bool wait = false);
         Task SetAffinity(string isolationTag, Vm[] vms, bool start);
         Task<Vm> Refresh(VmTemplate template);
         Task<Vm[]> Find(string searchText);
@@ -32,7 +33,6 @@ public interface IHypervisorService
         Task<VmOptions> GetVmNetOptions(string key);
         string Version { get; }
         Task ReloadHost(string host);
-
         HypervisorServiceConfiguration Options { get; }
     }
 

src/TopoMojo.Hypervisor/Proxmox/ProxmoxHypervisorService.cs

@@ -39,6 +39,8 @@ Random random
                 nameService,
                 vlanManager,
                 random);
+
+            _ = Task.Run(() => DeploymentHandler());
         }
 
         private readonly HypervisorServiceConfiguration _options;
@@ -53,6 +55,8 @@ Random random
         private readonly IProxmoxVlanManager _vlanManager;
 
         public HypervisorServiceConfiguration Options { get { return _options; } }
+        private BlockingCollection<DeploymentContext> DeploymentCollection = [];
+
 
         public async Task<Vm> Deploy(VmTemplate template, bool privileged = false)
         {
@@ -490,5 +494,33 @@ public Task ReloadHost(string host)
         {
             throw new NotImplementedException();
         }
+
+        private Task DeploymentHandler()
+        {
+            foreach(var ctx in DeploymentCollection.GetConsumingEnumerable())
+                _ = DeployBatch(ctx);
+
+            return Task.CompletedTask;
+        }
+
+        private async Task DeployBatch(DeploymentContext ctx)
+        {
+            var tasks = new List<Task<Vm>>();
+            var existing = (await Find(ctx.Id)).Select(vm => vm.Name);
+            var missing = ctx.Templates.Where(t => existing.Contains(t.Name).Equals(false));
+
+            foreach (var template in missing)
+                tasks.Add(Deploy(template, ctx.Privileged));
+
+            await Task.WhenAll(tasks.ToArray());
+        }
+
+        public async Task Deploy(DeploymentContext ctx, bool wait = false)
+        {
+            if (wait)
+                await DeployBatch(ctx);
+            else
+                DeploymentCollection.Add(ctx);
+        }
     }
 }

src/TopoMojo.Hypervisor/TopoMojo.Hypervisor.csproj

@@ -1,18 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <DebugType>portable</DebugType>
     <AssemblyName>TopoMojo.vSphere</AssemblyName>
     <PackageId>TopoMojo.vSphere</PackageId>
     <Nullable>disable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
     <PackageReference Include="VimClient" Version="7.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0" />
     <PackageReference Include="Corsinvest.ProxmoxVE.Api" Version="8.2.4" />
     <PackageReference Include="Corsinvest.ProxmoxVE.Api.Extension" Version="8.2.4" />
   </ItemGroup>
+
 </Project>

src/TopoMojo.Hypervisor/vMock/MockHypervisorService.cs

@@ -2,6 +2,7 @@
 // Released under a 3 Clause BSD-style license. See LICENSE.md in the project root for license information.
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
@@ -28,6 +29,7 @@ ILoggerFactory mill
             _rand = new Random();
 
             NormalizeOptions(_optPod);
+            _ = Task.Run(() => DeploymentHandler());
         }
 
         private readonly HypervisorServiceConfiguration _optPod;
@@ -36,6 +38,7 @@ ILoggerFactory mill
         private Random _rand;
         private Dictionary<string, Vm> _vms;
         private Dictionary<string, VmTask> _tasks;
+        private BlockingCollection<DeploymentContext> DeploymentCollection = [];
 
         public HypervisorServiceConfiguration Options { get { return _optPod; } }
 
@@ -527,6 +530,35 @@ private void NormalizeOptions(HypervisorServiceConfiguration options)
             if (!regex.IsMatch(options.IsoStore))
                 options.IsoStore += "/";
         }
+
+        private Task DeploymentHandler()
+        {
+            foreach (var ctx in DeploymentCollection.GetConsumingEnumerable())
+                _ = DeployBatch(ctx);
+
+            return Task.CompletedTask;
+        }
+
+        private async Task DeployBatch(DeploymentContext ctx)
+        {
+            _logger.LogInformation("Deploying Batch {id}", ctx.Id);
+            var tasks = new List<Task<Vm>>();
+            var existing = (await Find(ctx.Id)).Select(vm => vm.Name);
+            var missing = ctx.Templates.Where(t => existing.Contains(t.Name).Equals(false));
+
+            foreach (var template in missing)
+                tasks.Add(Deploy(template, ctx.Privileged));
+
+            await Task.WhenAll(tasks.ToArray());
+        }
+
+        public async Task Deploy(DeploymentContext ctx, bool wait = false)
+        {
+            if (wait)
+                await DeployBatch(ctx);
+            else
+                DeploymentCollection.Add(ctx);
+        }
     }
 
     public class MockDisk

src/TopoMojo.Hypervisor/vSphere/INetworkManager.cs

@@ -13,13 +13,15 @@ public interface INetworkManager
         Task AddSwitch(string sw);
         Task RemoveSwitch(string sw);
         Task<PortGroupAllocation> AddPortGroup(string sw, VmNet eth);
+        Task<PortGroupAllocation[]> AddPortGroups(string sw, VmNet[] eths);
         Task<bool> RemovePortgroup(string pgReference);
         Task<VmNetwork[]> GetVmNetworks(ManagedObjectReference managedObjectReference);
         Task<PortGroupAllocation[]> LoadPortGroups();
         void UpdateEthernetCardBacking(VirtualEthernetCard card, string portgroupName);
 
         Task Initialize();
         Task Provision(VmTemplate template);
+        Task ProvisionAll(VmNet[] template, bool useUplinkSwitch);
         Task Unprovision(ManagedObjectReference vmMOR);
         Task Clean(string tag = null);
         string Resolve(string net);

src/TopoMojo.Hypervisor/vSphere/NetworkManager.cs

@@ -84,48 +84,55 @@ public async Task Provision(VmTemplate template)
         {
             await Task.Delay(0);
 
+            ProvisionAll(template.Eth, template.UseUplinkSwitch).Wait();
+
+            foreach (var eth in template.Eth)
+            {
+                eth.Key = _pgAllocation[eth.Net].Key;
+                _pgAllocation[eth.Net].Counter += 1;
+            }
+        }
+
+        public async Task ProvisionAll(VmNet[] nets, bool useUplinkSwitch)
+        {
+            await Task.Delay(0);
+
             lock (_pgAllocation)
             {
                 string sw = _client.UplinkSwitch;
-                if (_client.dvs == null && _client.net != null && !template.UseUplinkSwitch)
+                if (_client.dvs == null && _client.net != null && !useUplinkSwitch)
                 {
-                    sw = template.IsolationTag.ToSwitchName();
-                    if (!_swAllocation.ContainsKey(sw))
+                    sw = nets[0].Net.Tag().ToSwitchName();
+                    if (_swAllocation.TryAdd(sw, 0))
                     {
                         AddSwitch(sw).Wait();
-                        _swAllocation.Add(sw, 0);
                     }
                 }
 
-                foreach (VmNet eth in template.Eth)
-                {
-                    if (!_pgAllocation.ContainsKey(eth.Net))
-                    {
-                        var pg = AddPortGroup(sw, eth).Result;
-                        pg.Timestamp = DateTimeOffset.UtcNow;
-                        pg.Counter = 1;
-
-                        _pgAllocation.Add(pg.Net, pg);
-
-                        _vlanManager.Activate(new Vlan[] {
-                            new Vlan {
-                                Id = pg.VlanId,
-                                Name = pg.Net,
-                                OnUplink = sw == _client.UplinkSwitch
-                            }
-                        });
+                var manifest = nets
+                    .Where(e => _pgAllocation.ContainsKey(e.Net).Equals(false))
+                    .Distinct()
+                    .ToArray()
+                ;
 
-                        if (_swAllocation.ContainsKey(sw))
-                            _swAllocation[sw] += 1;
+                var pgs = AddPortGroups(sw, manifest).Result;
 
-                    }
-                    else
-                    {
-                        _pgAllocation[eth.Net].Counter += 1;
-                    }
+                _vlanManager.Activate(
+                    pgs.Select(p => new Vlan {
+                        Id = p.VlanId,
+                        Name = p.Net,
+                        OnUplink = sw == _client.UplinkSwitch
+                    }).ToArray()
+                );
 
-                    eth.Key = _pgAllocation[eth.Net].Key;
+                foreach (var pg in pgs)
+                {
+                    pg.Timestamp = DateTimeOffset.UtcNow;
+                    _pgAllocation.Add(pg.Net, pg);
                 }
+
+                if (_swAllocation.ContainsKey(sw))
+                    _swAllocation[sw] += pgs.Length;
             }
         }
 
@@ -223,6 +230,15 @@ private Dictionary<string, PortGroupAllocation> GetKeyMap()
         public abstract Task<VmNetwork[]> GetVmNetworks(ManagedObjectReference managedObjectReference);
         public abstract Task<PortGroupAllocation[]> LoadPortGroups();
         public abstract Task<PortGroupAllocation> AddPortGroup(string sw, VmNet eth);
+        public virtual async Task<PortGroupAllocation[]> AddPortGroups(string sw, VmNet[] eths)
+        {
+            List<PortGroupAllocation> pgs = [];
+            foreach (var eth in eths)
+                pgs.Add(
+                    await AddPortGroup(sw, eth)
+                );
+            return [.. pgs];
+        }
         public abstract Task<bool> RemovePortgroup(string pgReference);
         public abstract Task AddSwitch(string sw);
         public abstract Task RemoveSwitch(string sw);