From 9b502735f31c228f83587ed187bf751ef49d511a Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Mon, 29 Jan 2024 14:22:49 -0800 Subject: [PATCH 01/16] upload diagnostic logs to results --- src/Runner.Common/JobServerQueue.cs | 14 ++++++++++++ src/Runner.Common/ResultsServer.cs | 14 ++++++++++++ src/Runner.Worker/ExecutionContext.cs | 2 ++ src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 25 ++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 04450e01161..12394cc4efc 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -551,6 +551,10 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) { await UploadSummaryFile(file); } + if (string.Equals(file.Type, CoreAttachmentType.DiagnosticLog, StringComparison.OrdinalIgnoreCase)) + { + await UploadResultsDiagnosticLogFile(file); + } else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase)) { if (file.RecordId != _jobTimelineRecordId) @@ -922,6 +926,16 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) await UploadResultsFile(file, summaryHandler); } + private async Task UploadDianosticLogsFile(ResultsUploadFileInfo file) { + Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}"); + ResultsFileUploadHandler diagnosticLogsHandler = async (file) => + { + await _resultsServer.CreateResultsDiagnosticLogsAsync(file.PlanId, file.JobId, file.Path, CancellationToken.None); + }; + + await UploadResultsFile(file, diagnosticLogsHandler); + } + private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file) { Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs index f3bf9910cfe..8a1b35948c3 100644 --- a/src/Runner.Common/ResultsServer.cs +++ b/src/Runner.Common/ResultsServer.cs @@ -35,6 +35,8 @@ Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool fin Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); + + Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken); } public sealed class ResultServer : RunnerService, IResultsServer @@ -141,6 +143,18 @@ public Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName throw new InvalidOperationException("Results client is not initialized."); } + public Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file, + CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadResultsDiagnosticLogsAsync(planId, jobId, file, + cancellationToken: cancellationToken); + } + + throw new InvalidOperationException("Results client is not initialized."); + } + public ValueTask DisposeAsync() { CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index cbd935b9744..a1de7ed7fdf 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -968,6 +968,8 @@ public void QueueAttachFile(string type, string name, string filePath) } _jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false); + + _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, type, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); } public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 9a7eb990b4e..c8c31c8f349 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -81,6 +81,19 @@ private async Task GetStepLogUploadUrlAsync(string return await GetResultsSignedURLResponse(getStepLogsSignedBlobURLEndpoint, cancellationToken, request); } + private async Task GetSignedDiagnosticLogsURL(string planid, string jobid, cancellationtoken cancellationtoken) + { + var request = new GetSignedDiagnosticLogsURLRequest() + { + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + }; + + var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetDiagnosticLogsSignedBlobURL); + + return await GetSignedDiagnosticLogsURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); + } + private async Task GetJobLogUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) { var request = new GetSignedJobLogsURLRequest() @@ -421,6 +434,18 @@ public async Task UploadResultsJobLogAsync(string planId, string jobId, string f } } + public async Task UploadResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken) + { + // Get the upload url + var uploadUrlResponse = await GetDiagnosticLogsUploadUrlAsync(planId, jobId, cancellationToken); + if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null) + { + throw new Exception("Failed to get diagnostic logs upload url"); + } + + await UploadLogFile(file, true, true, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + } + private Step ConvertTimelineRecordToStep(TimelineRecord r) { return new Step() From 440e8aa6a8638c140ac32058be26744a3a4b026f Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Mon, 29 Jan 2024 16:12:43 -0800 Subject: [PATCH 02/16] correct casing --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index c8c31c8f349..1ae5a956b27 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -81,7 +81,7 @@ private async Task GetStepLogUploadUrlAsync(string return await GetResultsSignedURLResponse(getStepLogsSignedBlobURLEndpoint, cancellationToken, request); } - private async Task GetSignedDiagnosticLogsURL(string planid, string jobid, cancellationtoken cancellationtoken) + private async Task GetSignedDiagnosticLogsURL(string planid, string jobid, CancellationToken cancellationtoken) { var request = new GetSignedDiagnosticLogsURLRequest() { From 6811e1b933d3987ebc907d5d4d6d60f61f0fe30d Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Mon, 29 Jan 2024 16:13:54 -0800 Subject: [PATCH 03/16] correct typo --- src/Runner.Common/JobServerQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 12394cc4efc..4b1b470d7a0 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -926,7 +926,7 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) await UploadResultsFile(file, summaryHandler); } - private async Task UploadDianosticLogsFile(ResultsUploadFileInfo file) { + private async Task UploadDiagnosticLogsFile(ResultsUploadFileInfo file) { Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}"); ResultsFileUploadHandler diagnosticLogsHandler = async (file) => { From 8aa0cb53d3bacde0574b1a1438a4f73229fa25b8 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Mon, 29 Jan 2024 16:39:58 -0800 Subject: [PATCH 04/16] update contract --- src/Sdk/WebApi/WebApi/Contracts.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index 95279e8ec30..6bef26d8160 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -89,6 +89,28 @@ public class GetSignedStepLogsURLResponse public long SoftSizeLimit; } + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedDiagnosticLogsURLRequest + { + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string WorkflowRunBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedDiagnosticLogsURLResponse + { + [DataMember] + public string DiagnosticLogsURL; + [DataMember] + public string BlobStorageType; + [DataMember] + public long SoftSizeLimit; + } + [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] public class JobLogsMetadataCreate From 479b701f0d82022211cb37541bf06d1dde2e84fa Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 09:57:07 -0800 Subject: [PATCH 05/16] update constant and define private method --- src/Sdk/WebApi/WebApi/Contracts.cs | 2 -- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 13 +++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index 6bef26d8160..7dac0fd2ab7 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -107,8 +107,6 @@ public class GetSignedDiagnosticLogsURLResponse public string DiagnosticLogsURL; [DataMember] public string BlobStorageType; - [DataMember] - public long SoftSizeLimit; } [DataContract] diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 1ae5a956b27..ae4f5dfa0c7 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -94,6 +94,19 @@ private async Task GetSignedDiagnosticLogsUR return await GetSignedDiagnosticLogsURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); } + private async TaskGetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) + { + var request = new GetSignedDiagnosticLogsURLRequest() + { + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + }; + + var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetDiagnosticLogsSignedBlobURL); + + return await GetResultsSignedURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); + } + private async Task GetJobLogUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) { var request = new GetSignedJobLogsURLRequest() From ddfae641d7dcc9d6e4e2f91a630e6e5135856083 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 09:59:21 -0800 Subject: [PATCH 06/16] fix access --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index ae4f5dfa0c7..25078606418 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -456,7 +456,7 @@ public async Task UploadResultsDiagnosticLogsAsync(string planId, string jobId, throw new Exception("Failed to get diagnostic logs upload url"); } - await UploadLogFile(file, true, true, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + await UploadLogFile(file, true, true, uploadUrlResponse.DiagnosticLogsURL, uploadUrlResponse.BlobStorageType, cancellationToken); } private Step ConvertTimelineRecordToStep(TimelineRecord r) From 647049203cd62e077d82a0cdac94962ad171ea9d Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 10:11:43 -0800 Subject: [PATCH 07/16] fix reference to logs url --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 25078606418..d86e2ff76e2 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -451,7 +451,7 @@ public async Task UploadResultsDiagnosticLogsAsync(string planId, string jobId, { // Get the upload url var uploadUrlResponse = await GetDiagnosticLogsUploadUrlAsync(planId, jobId, cancellationToken); - if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null) + if (uploadUrlResponse == null || uploadUrlResponse.DiagnosticLogsURL == null) { throw new Exception("Failed to get diagnostic logs upload url"); } From d30b4c3a455f7e9310e6eed0241479d084d3d0f7 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 10:20:22 -0800 Subject: [PATCH 08/16] update --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index d86e2ff76e2..e8f15b043d2 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -549,6 +549,7 @@ public static class Constants public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata"; public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL"; public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata"; + public static readonly string GetDiagnosticLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetDiagnosticLogsSignedBlobURL"; public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/"; public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate"; From 4fc9ef702ae02aea306daaeef7085f0682180fd7 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 10:33:46 -0800 Subject: [PATCH 09/16] use results func --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index e8f15b043d2..40f3e6ffbce 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -81,7 +81,7 @@ private async Task GetStepLogUploadUrlAsync(string return await GetResultsSignedURLResponse(getStepLogsSignedBlobURLEndpoint, cancellationToken, request); } - private async Task GetSignedDiagnosticLogsURL(string planid, string jobid, CancellationToken cancellationtoken) + private async Task GetSignedDiagnosticLogsURL(string planId, string jobId, CancellationToken cancellationToken) { var request = new GetSignedDiagnosticLogsURLRequest() { @@ -91,7 +91,7 @@ private async Task GetSignedDiagnosticLogsUR var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetDiagnosticLogsSignedBlobURL); - return await GetSignedDiagnosticLogsURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); + return await GetResultsSignedURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); } private async TaskGetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) From 41c44a2c0309a99fd056c4b30f8c99b5532ca3a7 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 10:38:06 -0800 Subject: [PATCH 10/16] fix method naming --- src/Runner.Common/JobServerQueue.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 4b1b470d7a0..804eab5905c 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -553,7 +553,7 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) } if (string.Equals(file.Type, CoreAttachmentType.DiagnosticLog, StringComparison.OrdinalIgnoreCase)) { - await UploadResultsDiagnosticLogFile(file); + await UploadResultsDiagnosticLogsFile(file); } else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase)) { @@ -926,7 +926,7 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) await UploadResultsFile(file, summaryHandler); } - private async Task UploadDiagnosticLogsFile(ResultsUploadFileInfo file) { + private async Task UploadResultsDiagnosticLogsFile(ResultsUploadFileInfo file) { Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}"); ResultsFileUploadHandler diagnosticLogsHandler = async (file) => { From 93c8b07eb61cd372446100282e7110e9fd433aae Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 11:54:59 -0800 Subject: [PATCH 11/16] rename to match rpc --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 40f3e6ffbce..c66ee0de59e 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -89,7 +89,7 @@ private async Task GetSignedDiagnosticLogsUR WorkflowRunBackendId = planId, }; - var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetDiagnosticLogsSignedBlobURL); + var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobDiagLogsSignedBlobURL); return await GetResultsSignedURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); } @@ -102,7 +102,7 @@ private async TaskGetDiagnosticLogsUploadUrl WorkflowRunBackendId = planId, }; - var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetDiagnosticLogsSignedBlobURL); + var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobDiagLogsSignedBlobURL); return await GetResultsSignedURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); } @@ -549,7 +549,7 @@ public static class Constants public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata"; public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL"; public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata"; - public static readonly string GetDiagnosticLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetDiagnosticLogsSignedBlobURL"; + public static readonly string GetJobDiagLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobDiagLogsSignedBlobURL"; public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/"; public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate"; From 7a28c2137626d2a0dd2613c667f59d62cbc8f607 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 12:14:59 -0800 Subject: [PATCH 12/16] change API contract --- src/Sdk/WebApi/WebApi/Contracts.cs | 2 +- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index 7dac0fd2ab7..0018062ea58 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -104,7 +104,7 @@ public class GetSignedDiagnosticLogsURLRequest public class GetSignedDiagnosticLogsURLResponse { [DataMember] - public string DiagnosticLogsURL; + public string DiagLogsURL; [DataMember] public string BlobStorageType; } diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index c66ee0de59e..3066ff2add9 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -451,12 +451,12 @@ public async Task UploadResultsDiagnosticLogsAsync(string planId, string jobId, { // Get the upload url var uploadUrlResponse = await GetDiagnosticLogsUploadUrlAsync(planId, jobId, cancellationToken); - if (uploadUrlResponse == null || uploadUrlResponse.DiagnosticLogsURL == null) + if (uploadUrlResponse == null || uploadUrlResponse.DiagLogsURL == null) { throw new Exception("Failed to get diagnostic logs upload url"); } - await UploadLogFile(file, true, true, uploadUrlResponse.DiagnosticLogsURL, uploadUrlResponse.BlobStorageType, cancellationToken); + await UploadLogFile(file, true, true, uploadUrlResponse.DiagLogsURL, uploadUrlResponse.BlobStorageType, cancellationToken); } private Step ConvertTimelineRecordToStep(TimelineRecord r) From 7bb10d8cb333b6464d4dd0812618b19cf49af4e8 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Tue, 30 Jan 2024 13:22:36 -0800 Subject: [PATCH 13/16] fix lint issue --- src/Runner.Common/JobServerQueue.cs | 3 ++- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 804eab5905c..64342310dcc 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -926,7 +926,8 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) await UploadResultsFile(file, summaryHandler); } - private async Task UploadResultsDiagnosticLogsFile(ResultsUploadFileInfo file) { + private async Task UploadResultsDiagnosticLogsFile(ResultsUploadFileInfo file) + { Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}"); ResultsFileUploadHandler diagnosticLogsHandler = async (file) => { diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 3066ff2add9..c61748bdaf0 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -94,7 +94,7 @@ private async Task GetSignedDiagnosticLogsUR return await GetResultsSignedURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); } - private async TaskGetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) + private async Task GetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) { var request = new GetSignedDiagnosticLogsURLRequest() { From c1adf17a6dddf148ac3c01ee488e5e320cfce949 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Wed, 31 Jan 2024 10:34:31 -0800 Subject: [PATCH 14/16] refactor --- src/Runner.Worker/DiagnosticLogManager.cs | 2 ++ src/Runner.Worker/ExecutionContext.cs | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/DiagnosticLogManager.cs b/src/Runner.Worker/DiagnosticLogManager.cs index 7259bab26bd..261689b5f3f 100644 --- a/src/Runner.Worker/DiagnosticLogManager.cs +++ b/src/Runner.Worker/DiagnosticLogManager.cs @@ -108,6 +108,8 @@ public void UploadDiagnosticLogs(IExecutionContext executionContext, parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath); + parentContext.QueueDiagnosticLogFile(name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath); + executionContext.Debug("Diagnostic file upload complete."); } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index a1de7ed7fdf..f6b1d17ec2b 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -90,6 +90,7 @@ public interface IExecutionContext : IRunnerService long Write(string tag, string message); void QueueAttachFile(string type, string name, string filePath); void QueueSummaryFile(string name, string filePath, Guid stepRecordId); + void QueueDiagnosticLogFile(string name, string filePath); // timeline record update methods void Start(string currentOperation = null); @@ -968,8 +969,6 @@ public void QueueAttachFile(string type, string name, string filePath) } _jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false); - - _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, type, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); } public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) @@ -984,6 +983,19 @@ public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); } + public void QueueDiagnosticLogFile(string name, string filePath) + { + ArgUtil.NotNullOrEmpty(name, nameof(name)); + ArgUtil.NotNullOrEmpty(filePath, nameof(filePath)); + + + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Can't upload diagnostic log file: {filePath}. File does not exist."); + } + _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, CoreAttachmentType.DiagnosticLog, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); + } + // Add OnMatcherChanged public void Add(OnMatcherChanged handler) { From 4895f44e009e29c826c17fb100473a468f1fc8a5 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Wed, 31 Jan 2024 18:21:25 -0800 Subject: [PATCH 15/16] remove unused method --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index c61748bdaf0..53338dd15a2 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -81,19 +81,6 @@ private async Task GetStepLogUploadUrlAsync(string return await GetResultsSignedURLResponse(getStepLogsSignedBlobURLEndpoint, cancellationToken, request); } - private async Task GetSignedDiagnosticLogsURL(string planId, string jobId, CancellationToken cancellationToken) - { - var request = new GetSignedDiagnosticLogsURLRequest() - { - WorkflowJobRunBackendId = jobId, - WorkflowRunBackendId = planId, - }; - - var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobDiagLogsSignedBlobURL); - - return await GetResultsSignedURLResponse(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request); - } - private async Task GetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) { var request = new GetSignedDiagnosticLogsURLRequest() From cce36c968c42ee1dbd768e3c636fb24bf9bb6110 Mon Sep 17 00:00:00 2001 From: Jonathan Tamsut Date: Thu, 1 Feb 2024 10:17:36 -0800 Subject: [PATCH 16/16] create new log type --- src/Runner.Common/JobServerQueue.cs | 2 +- src/Runner.Worker/ExecutionContext.cs | 3 +-- src/Sdk/DTWebApi/WebApi/TaskAttachment.cs | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 64342310dcc..c1425b80721 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -551,7 +551,7 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) { await UploadSummaryFile(file); } - if (string.Equals(file.Type, CoreAttachmentType.DiagnosticLog, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(file.Type, CoreAttachmentType.ResultsDiagnosticLog, StringComparison.OrdinalIgnoreCase)) { await UploadResultsDiagnosticLogsFile(file); } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index f6b1d17ec2b..77c145d1ddc 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -988,12 +988,11 @@ public void QueueDiagnosticLogFile(string name, string filePath) ArgUtil.NotNullOrEmpty(name, nameof(name)); ArgUtil.NotNullOrEmpty(filePath, nameof(filePath)); - if (!File.Exists(filePath)) { throw new FileNotFoundException($"Can't upload diagnostic log file: {filePath}. File does not exist."); } - _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, CoreAttachmentType.DiagnosticLog, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); + _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, CoreAttachmentType.ResultsDiagnosticLog, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); } // Add OnMatcherChanged diff --git a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs index 572b1f6426b..e1aa07417de 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs @@ -101,6 +101,7 @@ public class CoreAttachmentType public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment"; public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog"; public static readonly String ResultsLog = "Results.Core.Log"; + public static readonly String ResultsDiagnosticLog = "Results.Core.DiagnosticLog"; } [GenerateAllConstants]