Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Fix logic of marking task as failed #3083

Merged
merged 7 commits into from
May 4, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions src/ApiService/ApiService/Functions/AgentCanSchedule.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
_log.Warning($"Unable to find {canScheduleRequest.MachineId:Tag:MachineId}");
return await _context.RequestHandling.NotOk(
req,
new Error(
ErrorCode.UNABLE_TO_FIND,
new string[] {
"unable to find node"
}),
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
canScheduleRequest.MachineId.ToString());
}

Expand Down
49 changes: 20 additions & 29 deletions src/ApiService/ApiService/Functions/AgentEvents.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
NodeStateUpdate updateEvent => await OnStateUpdate(envelope.MachineId, updateEvent),
WorkerEvent workerEvent => await OnWorkerEvent(envelope.MachineId, workerEvent),
NodeEvent nodeEvent => await OnNodeEvent(envelope.MachineId, nodeEvent),
_ => new Error(ErrorCode.INVALID_REQUEST, new string[] { $"invalid node event: {envelope.Event.GetType().Name}" }),
_ => Error.Create(ErrorCode.INVALID_REQUEST, $"invalid node event: {envelope.Event.GetType().Name}"),
};

if (error is Error e) {
Expand Down Expand Up @@ -114,17 +114,17 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
} else if (ev.State == NodeState.SettingUp) {
if (ev.Data is NodeSettingUpEventData settingUpData) {
if (!settingUpData.Tasks.Any()) {
return new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
$"setup without tasks. machine_id: {machineId}",
});
return Error.Create(ErrorCode.INVALID_REQUEST,
$"setup without tasks. machine_id: {machineId}"
);
}

foreach (var taskId in settingUpData.Tasks) {
var task = await _context.TaskOperations.GetByTaskId(taskId);
if (task is null) {
return new Error(
return Error.Create(
ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find task: {taskId}" });
$"unable to find task: {taskId}");
}

_log.Info($"node starting task. {machineId:Tag:MachineId} {task.JobId:Tag:JobId} {task.TaskId:Tag:TaskId}");
Expand Down Expand Up @@ -154,7 +154,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (ev.Data is NodeDoneEventData doneData) {
if (doneData.Error is not null) {
var errorText = EntityConverter.ToJsonString(doneData);
error = new Error(ErrorCode.TASK_FAILED, Errors: new string[] { errorText });
error = Error.Create(ErrorCode.TASK_FAILED, errorText);
_log.Error($"node 'done' {machineId:Tag:MachineId} - {errorText:Tag:Error}");
}
}
Expand All @@ -178,9 +178,9 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
return await OnWorkerEventRunning(machineId, ev.Running);
}

return new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { "WorkerEvent should have either 'done' or 'running' set" });
return Error.Create(
ErrorCode.INVALID_REQUEST,
"WorkerEvent should have either 'done' or 'running' set");
}

private async Async.Task<Error?> OnWorkerEventRunning(Guid machineId, WorkerRunningEvent running) {
Expand All @@ -189,15 +189,11 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
_context.NodeOperations.GetByMachineId(machineId));

if (task is null) {
return new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find task: {running.TaskId}" });
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find task: {running.TaskId}");
}

if (node is null) {
return new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find node: {machineId}" });
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find node: {machineId}");
}

if (!node.State.ReadyForReset()) {
Expand Down Expand Up @@ -240,15 +236,11 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
_context.NodeOperations.GetByMachineId(machineId));

if (task is null) {
return new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find task: {done.TaskId}" });
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find task: {done.TaskId}");
}

if (node is null) {
return new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find node: {machineId}" });
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find node: {machineId}");
}

// trim stdout/stderr if too long
Expand All @@ -272,13 +264,12 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
} else {
await _context.TaskOperations.MarkFailed(
task,
new Error(
Code: ErrorCode.TASK_FAILED,
Errors: new string[] {
$"task failed. exit_status:{done.ExitStatus}",
done.Stdout,
done.Stderr,
}));
Error.Create(
ErrorCode.TASK_FAILED,
$"task failed. exit_status:{done.ExitStatus}",
done.Stdout,
done.Stderr
));

// keep node if any keep options are set
if ((task.Config.Debug?.Contains(TaskDebugFlag.KeepNodeOnFailure) == true)
Expand Down
33 changes: 15 additions & 18 deletions src/ApiService/ApiService/Functions/AgentRegistration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,29 +40,29 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
if (machineId == Guid.Empty) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Error.Create(
ErrorCode.INVALID_REQUEST,
new string[] { "'machine_id' query parameter must be provided" }),
"'machine_id' query parameter must be provided"),
"agent registration");
}

var agentNode = await _context.NodeOperations.GetByMachineId(machineId);
if (agentNode is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Error.Create(
ErrorCode.INVALID_REQUEST,
new string[] { $"unable to find a registration associated with machine_id '{machineId}'" }),
$"unable to find a registration associated with machine_id '{machineId}'"),
"agent registration");
}

var pool = await _context.PoolOperations.GetByName(agentNode.PoolName);
if (!pool.IsOk) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Error.Create(
ErrorCode.INVALID_REQUEST,
new string[] { "unable to find a pool associated with the provided machine_id" }),
"unable to find a pool associated with the provided machine_id"),
"agent registration");
}

Expand Down Expand Up @@ -101,18 +101,16 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (machineId == Guid.Empty) {
return await _context.RequestHandling.NotOk(
req,
new Error(
ErrorCode.INVALID_REQUEST,
new string[] { "'machine_id' query parameter must be provided" }),
Error.Create(
ErrorCode.INVALID_REQUEST, "'machine_id' query parameter must be provided"),
"agent registration");
}

if (poolName is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
ErrorCode.INVALID_REQUEST,
new string[] { "'pool_name' query parameter must be provided" }),
Error.Create(
ErrorCode.INVALID_REQUEST, "'pool_name' query parameter must be provided"),
"agent registration");
}

Expand All @@ -124,9 +122,8 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (!poolResult.IsOk) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new[] { $"unable to find pool '{poolName}'" }),
Error.Create(
ErrorCode.INVALID_REQUEST, $"unable to find pool '{poolName}'"),
"agent registration");
}

Expand All @@ -140,9 +137,9 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (os != null && pool.Os != os) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new[] { $"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'" }),
Error.Create(
ErrorCode.INVALID_REQUEST,
$"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'"),
"agent registration");
}

Expand Down
12 changes: 6 additions & 6 deletions src/ApiService/ApiService/Functions/Containers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
if (container is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new[] { "invalid container" }),
Error.Create(
ErrorCode.INVALID_REQUEST,
"invalid container"),
context: get.Name.String);
}

Expand Down Expand Up @@ -101,9 +101,9 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (sas is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new[] { "invalid container" }),
Error.Create(
ErrorCode.INVALID_REQUEST,
"invalid container"),
context: post.Name.String);
}

Expand Down
8 changes: 4 additions & 4 deletions src/ApiService/ApiService/Functions/Download.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,19 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
if (queryContainer is null || !Container.TryParse(queryContainer, out var container)) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Error.Create(
ErrorCode.INVALID_REQUEST,
new string[] { "'container' query parameter must be provided and valid" }),
"'container' query parameter must be provided and valid"),
"download");
}

var filename = query["filename"];
if (filename is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Error.Create(
ErrorCode.INVALID_REQUEST,
new string[] { "'filename' query parameter must be provided" }),
"'filename' query parameter must be provided"),
"download");
}

Expand Down
20 changes: 8 additions & 12 deletions src/ApiService/ApiService/Functions/Jobs.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,7 @@ private async Task<HttpResponseData> Post(HttpRequestData req) {
if (containerSas is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.UNABLE_TO_CREATE_CONTAINER,
Errors: new string[] { "unable to create logs container " }),
Error.Create(ErrorCode.UNABLE_TO_CREATE_CONTAINER, "unable to create logs container "),
"logs");
}

Expand All @@ -73,9 +71,9 @@ private async Task<HttpResponseData> Post(HttpRequestData req) {
_logTracer.WithTag("HttpRequest", "POST").WithHttpStatus(r.ErrorV).Error($"failed to insert job {job.JobId:Tag:JobId}");
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.UNABLE_TO_CREATE,
Errors: new string[] { "unable to create job " }
Error.Create(
ErrorCode.UNABLE_TO_CREATE,
"unable to create job"
),
"job");
}
Expand All @@ -95,9 +93,9 @@ private async Task<HttpResponseData> Delete(HttpRequestData req) {
if (job is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_JOB,
Errors: new string[] { "no such job" }),
Error.Create(
ErrorCode.INVALID_JOB,
"no such job"),
context: jobId.ToString());
}

Expand All @@ -124,9 +122,7 @@ private async Task<HttpResponseData> Get(HttpRequestData req) {
if (job is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_JOB,
Errors: new string[] { "no such job" }),
Error.Create(ErrorCode.INVALID_JOB, "no such job"),
context: jobId.ToString());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ await notifications.Select(notification => notification.NotificationId).ToListAs
_log.Info($"Migrated notification: {notification.NotificationId} to jinja");
} else {
failedNotificationIds.Add(notification.NotificationId);
_log.Error(new Error(ErrorCode.UNABLE_TO_UPDATE, new[] { r.ErrorV.Reason, r.ErrorV.Status.ToString() }));
_log.Error(Error.Create(ErrorCode.UNABLE_TO_UPDATE, r.ErrorV.Reason, r.ErrorV.Status.ToString()));
}
} catch (Exception ex) {
failedNotificationIds.Add(notification.NotificationId);
Expand Down
16 changes: 4 additions & 12 deletions src/ApiService/ApiService/Functions/Node.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
if (node is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.UNABLE_TO_FIND,
Errors: new string[] { "unable to find node" }),
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
context: machineId.ToString());
}

Expand Down Expand Up @@ -94,9 +92,7 @@ private async Async.Task<HttpResponseData> Patch(HttpRequestData req) {
if (node is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.UNABLE_TO_FIND,
Errors: new string[] { "unable to find node" }),
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
context: patch.MachineId.ToString());
}

Expand Down Expand Up @@ -130,9 +126,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (node is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.UNABLE_TO_FIND,
Errors: new string[] { "unable to find node" }),
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
context: post.MachineId.ToString());
}

Expand Down Expand Up @@ -166,9 +160,7 @@ private async Async.Task<HttpResponseData> Delete(HttpRequestData req) {
if (node is null) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.UNABLE_TO_FIND,
new string[] { "unable to find node" }),
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
context: delete.MachineId.ToString());
}

Expand Down
2 changes: 1 addition & 1 deletion src/ApiService/ApiService/Functions/NodeAddSshKey.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
if (node == null) {
return await _context.RequestHandling.NotOk(
req,
new Error(ErrorCode.UNABLE_TO_FIND, new[] { "unable to find node" }),
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
$"{request.OkV.MachineId}");
}

Expand Down
6 changes: 3 additions & 3 deletions src/ApiService/ApiService/Functions/Notifications.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,18 @@ private async Async.Task<HttpResponseData> Delete(HttpRequestData req) {
var entries = await _context.NotificationOperations.SearchByPartitionKeys(new[] { $"{request.OkV.NotificationId}" }).ToListAsync();

if (entries.Count == 0) {
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find notification" }), context: "notification delete");
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find notification"), context: "notification delete");
}

if (entries.Count > 1) {
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "error identifying Notification" }), context: "notification delete");
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "error identifying Notification"), context: "notification delete");
}

var result = await _context.NotificationOperations.Delete(entries[0]);

if (!result.IsOk) {
var (status, error) = result.ErrorV;
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.UNABLE_TO_UPDATE, new[] { error }), "notification delete");
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.UNABLE_TO_UPDATE, error), "notification delete");
}

var response = req.CreateResponse(HttpStatusCode.OK);
Expand Down
Loading