Node Ops test hooks

src/ApiService/ApiService/TestHooks.cs

@@ -6,7 +6,7 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 
 namespace Microsoft.OneFuzz.Service;
-
+#if DEBUG
 public record FunctionInfo(string Name, string ResourceGroup, string? SlotName);
 public class TestHooks {
 
@@ -107,3 +107,4 @@ public async Task<HttpResponseData> GetMonitorSettings([HttpTrigger(Authorizatio
     }
 
 }
+#endif

src/ApiService/ApiService/TestHooks/ExtensionsTestHooks.cs

@@ -25,17 +25,7 @@ public async Task<HttpResponseData> GenericExtensions([HttpTrigger(Authorization
             _log.Info("Get Generic extensions");
 
             var query = UriExtension.GetQueryComponents(req.Url);
-
-            Os os;
-            if (query["os"].ToLowerInvariant() == "windows") {
-                os = Os.Windows;
-            } else if (query["os"].ToLowerInvariant() == "linux") {
-                os = Os.Linux;
-            } else {
-                var err = req.CreateResponse(HttpStatusCode.BadRequest);
-                await err.WriteAsJsonAsync(new { error = $"unsupported os {query["os"]}" });
-                return err;
-            }
+            Os os = Enum.Parse<Os>(query["os"]);
 
             var ext = await (_extensions as Extensions)!.GenericExtensions(query["region"], os);
             var resp = req.CreateResponse(HttpStatusCode.OK);

src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs

@@ -0,0 +1,278 @@
+using System.Net;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.OneFuzz.Service;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+
+
+
+#if DEBUG
+namespace ApiService.TestHooks {
+
+    record MarkTasks(Node node, Error? error);
+
+    public class NodeOperationsTestHooks {
+        private readonly ILogTracer _log;
+        private readonly IConfigOperations _configOps;
+        private readonly INodeOperations _nodeOps;
+
+        public NodeOperationsTestHooks(ILogTracer log, IConfigOperations configOps, INodeOperations nodeOps) {
+            _log = log.WithTag("TestHooks", nameof(NodeOperationsTestHooks));
+            _configOps = configOps;
+            _nodeOps = nodeOps;
+        }
+
+        [Function("GetByMachineIdTestHook")]
+        public async Task<HttpResponseData> GetByMachineId([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/nodeOperations/getByMachineId")] HttpRequestData req) {
+            _log.Info("Get node by machine id");
+
+            var query = UriExtension.GetQueryComponents(req.Url);
+            var machineId = query["machineId"];
+
+            var node = await _nodeOps.GetByMachineId(Guid.Parse(machineId));
+
+            var msg = JsonSerializer.Serialize(node, EntityConverter.GetJsonSerializerOptions());
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteStringAsync(msg);
+            return resp;
+        }
+
+        [Function("CanProcessNewWorkTestHook")]
+        public async Task<HttpResponseData> CanProcessNewWork([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/canProcessNewWork")] HttpRequestData req) {
+            _log.Info("Can process new work");
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = await _nodeOps.CanProcessNewWork(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+
+        [Function("IsOutdatedTestHook")]
+        public async Task<HttpResponseData> IsOutdated([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/isOutdated")] HttpRequestData req) {
+            _log.Info("Is outdated");
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.IsOutdated(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+
+        [Function("IsTooOldTestHook")]
+        public async Task<HttpResponseData> IsTooOld([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/isTooOld")] HttpRequestData req) {
+            _log.Info("Is too old");
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.IsTooOld(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+        [Function("CouldShrinkScalesetTestHook")]
+        public async Task<HttpResponseData> CouldShrinkScaleset([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/couldShrinkScaleset")] HttpRequestData req) {
+            _log.Info("Could shrink scaleset");
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.CouldShrinkScaleset(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+
+        [Function("SetHaltTestHook")]
+        public async Task<HttpResponseData> SetHalt([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/setHalt")] HttpRequestData req) {
+            _log.Info("set halt");
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.SetHalt(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+        [Function("SetStateTestHook")]
+        public async Task<HttpResponseData> SetState([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/setState")] HttpRequestData req) {
+            _log.Info("set state");
+
+            var query = UriExtension.GetQueryComponents(req.Url);
+            var state = Enum.Parse<NodeState>(query["state"]);
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.SetState(node!, state);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+
+        [Function("ToReimageTestHook")]
+        public async Task<HttpResponseData> ToReimage([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/toReimage")] HttpRequestData req) {
+            _log.Info("to reimage");
+
+            var query = UriExtension.GetQueryComponents(req.Url);
+            var done = UriExtension.GetBoolValue("done", query, false);
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.ToReimage(node!, done);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+        [Function("SendStopIfFreeTestHook")]
+        public async Task<HttpResponseData> SendStopIfFree([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/sendStopIfFree")] HttpRequestData req) {
+            _log.Info("send stop if free");
+
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.SendStopIfFree(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+
+        [Function("SearchStatesTestHook")]
+        public async Task<HttpResponseData> SearchStates([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/searchStates")] HttpRequestData req) {
+            _log.Info("search states");
+
+            var query = UriExtension.GetQueryComponents(req.Url);
+
+            Guid? poolId = default;
+            if (query.ContainsKey("poolId")) {
+                poolId = Guid.Parse(query["poolId"]);
+            }
+
+            Guid? scaleSetId = default;
+            if (query.ContainsKey("scaleSetId")) {
+                scaleSetId = Guid.Parse(query["scaleSetId"]);
+            }
+
+            List<NodeState>? states = default;
+            if (query.ContainsKey("states")) {
+                states = query["states"].Split('-').Select(s => Enum.Parse<NodeState>(s)).ToList();
+            }
+
+            string? poolName = default;
+            if (query.ContainsKey("poolName")) {
+                poolName = query["poolName"];
+            }
+
+            var excludeUpdateScheduled = UriExtension.GetBoolValue("excludeUpdateScheduled", query, false);
+            int? numResults = default;
+            if (query.ContainsKey("numResults")) {
+                numResults = int.Parse(query["numResults"]);
+            }
+            var r = _nodeOps.SearchStates(poolId, scaleSetId, states, poolName, excludeUpdateScheduled, numResults);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+
+            await resp.WriteStringAsync(JsonSerializer.Serialize(await r.ToListAsync(), EntityConverter.GetJsonSerializerOptions()));
+            return resp;
+        }
+
+
+        [Function("DeleteNodeTestHook")]
+        public async Task<HttpResponseData> DeleteNode([HttpTrigger(AuthorizationLevel.Anonymous, "delete", Route = "testhooks/nodeOperations/node")] HttpRequestData req) {
+            _log.Info("delete node");
+            var s = await req.ReadAsStringAsync();
+            var node = JsonSerializer.Deserialize<Node>(s!, EntityConverter.GetJsonSerializerOptions());
+
+            var r = _nodeOps.Delete(node!);
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+
+        [Function("ReimageLongLivedNodesTestHook")]
+        public async Task<HttpResponseData> ReimageLongLivedNodes([HttpTrigger(AuthorizationLevel.Anonymous, "patch", Route = "testhooks/nodeOperations/reimageLongLivedNodes")] HttpRequestData req) {
+            _log.Info("reimage long lived nodes");
+            var query = UriExtension.GetQueryComponents(req.Url);
+
+            var r = _nodeOps.ReimageLongLivedNodes(Guid.Parse(query["scaleSetId"]));
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(r);
+            return resp;
+        }
+
+        [Function("CreateTestHook")]
+        public async Task<HttpResponseData> CreateNode([HttpTrigger(AuthorizationLevel.Anonymous, "put", Route = "testhooks/nodeOperations/node")] HttpRequestData req) {
+
+            _log.Info("create node");
+
+            var query = UriExtension.GetQueryComponents(req.Url);
+
+            Guid poolId = Guid.Parse(query["poolId"]);
+            string poolName = query["poolName"];
+            Guid machineId = Guid.Parse(query["machineId"]);
+
+            Guid? scaleSetId = default;
+            if (query.ContainsKey("scaleSetId")) {
+                scaleSetId = Guid.Parse(query["scaleSetId"]);
+            }
+
+            string version = query["version"];
+
+            bool isNew = UriExtension.GetBoolValue("isNew", query, false);
+
+            var node = await _nodeOps.Create(poolId, poolName, machineId, scaleSetId, version, isNew);
+
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteAsJsonAsync(JsonSerializer.Serialize(node, EntityConverter.GetJsonSerializerOptions()));
+            return resp;
+        }
+
+        [Function("GetDeadNodesTestHook")]
+        public async Task<HttpResponseData> GetDeadNodes([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "testhooks/nodeOperations/getDeadNodes")] HttpRequestData req) {
+
+            _log.Info("get dead nodes");
+
+            var query = UriExtension.GetQueryComponents(req.Url);
+
+            Guid scaleSetId = Guid.Parse(query["scaleSetId"]);
+            TimeSpan timeSpan = TimeSpan.Parse(query["timeSpan"]);
+
+            var nodes = await (_nodeOps.GetDeadNodes(scaleSetId, timeSpan).ToListAsync());
+            var json = JsonSerializer.Serialize(nodes, EntityConverter.GetJsonSerializerOptions());
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            await resp.WriteStringAsync(json);
+            return resp;
+        }
+
+
+        [Function("MarkTasksStoppedEarly")]
+        public async Task<HttpResponseData> MarkTasksStoppedEarly([HttpTrigger(AuthorizationLevel.Anonymous, "patch", Route = "testhooks/nodeOperations/markTasksStoppedEarly")] HttpRequestData req) {
+            _log.Info("mark tasks stopped early");
+
+            var s = await req.ReadAsStringAsync();
+            var markTasks = JsonSerializer.Deserialize<MarkTasks>(s!, EntityConverter.GetJsonSerializerOptions());
+            await _nodeOps.MarkTasksStoppedEarly(markTasks.node, markTasks.error);
+
+            var resp = req.CreateResponse(HttpStatusCode.OK);
+            return resp;
+        }
+    }
+}
+#endif

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -21,7 +21,7 @@ public interface INodeOperations : IStatefulOrm<Node, NodeState> {
     Async.Task SendStopIfFree(Node node);
     IAsyncEnumerable<Node> SearchStates(Guid? poolId = default,
         Guid? scaleSetId = default,
-        IList<NodeState>? states = default,
+        IEnumerable<NodeState>? states = default,
         string? poolName = default,
         bool excludeUpdateScheduled = false,
         int? numResults = default);
@@ -200,7 +200,8 @@ public IAsyncEnumerable<Node> GetDeadNodes(Guid scaleSetId, TimeSpan expirationP
         var minDate = DateTimeOffset.UtcNow - expirationPeriod;
 
         var filter = $"heartbeat lt datetime'{minDate.ToString("o")}' or Timestamp lt datetime'{minDate.ToString("o")}'";
-        return QueryAsync(Query.And(filter, $"scaleset_id eq ${scaleSetId}"));
+        var query = Query.And(filter, $"scaleset_id eq '{scaleSetId}'");
+        return QueryAsync(query);
     }
 
 
@@ -344,12 +345,17 @@ public static string SearchStatesQuery(
     public IAsyncEnumerable<Node> SearchStates(
         Guid? poolId = default,
         Guid? scaleSetId = default,
-        IList<NodeState>? states = default,
+        IEnumerable<NodeState>? states = default,
         string? poolName = default,
         bool excludeUpdateScheduled = false,
         int? numResults = default) {
         var query = NodeOperations.SearchStatesQuery(_context.ServiceConfiguration.OneFuzzVersion, poolId, scaleSetId, states, poolName, excludeUpdateScheduled, numResults);
-        return QueryAsync(query);
+
+        if (numResults is null) {
+            return QueryAsync(query);
+        } else {
+            return QueryAsync(query).TakeWhile((_, i) => i < numResults);
+        }
     }
 
     public async Async.Task MarkTasksStoppedEarly(Node node, Error? error = null) {

src/ApiService/Tests/OrmModelsTest.cs

@@ -906,6 +906,11 @@ public bool EventMessage(EventMessage e) {
             return Test(e);
         }
 
+        [Property]
+        public bool Error(Error e) {
+            return Test(e);
+        }
+
         /*
         //Sample function on how repro a failing test run, using Replay
         //functionality of FsCheck. Feel free to