Merge branch 'main' into chkeita/watson

src/ApiService/ApiService/Functions/ValidateScriban.cs

@@ -103,6 +103,7 @@ public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymou
                 null,
                 null,
                 null,
+                null,
                 null
             );
 

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -302,17 +302,27 @@ public record EventCrashReported(
     Container Container,
     [property: JsonPropertyName("filename")] String FileName,
     TaskConfig? TaskConfig
-) : BaseEvent();
-
+) : BaseEvent(), ITruncatable<BaseEvent> {
+    public BaseEvent Truncate(int maxLength) {
+        return this with {
+            Report = Report.Truncate(maxLength)
+        };
+    }
+}
 
 [EventType(EventType.RegressionReported)]
 public record EventRegressionReported(
     RegressionReport RegressionReport,
     Container Container,
     [property: JsonPropertyName("filename")] String FileName,
     TaskConfig? TaskConfig
-) : BaseEvent();
-
+) : BaseEvent(), ITruncatable<BaseEvent> {
+    public BaseEvent Truncate(int maxLength) {
+        return this with {
+            RegressionReport = RegressionReport.Truncate(maxLength)
+        };
+    }
+}
 
 [EventType(EventType.FileAdded)]
 public record EventFileAdded(

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -457,8 +457,31 @@ public record Report(
     string? MinimizedStackFunctionLinesSha256,
     string? ToolName,
     string? ToolVersion,
-    string? OnefuzzVersion
-) : IReport;
+    string? OnefuzzVersion,
+    Uri? ReportUrl
+) : IReport, ITruncatable<Report> {
+    public Report Truncate(int maxLength) {
+        return this with {
+            Executable = Executable[..maxLength],
+            CrashType = CrashType[..Math.Min(maxLength, CrashType.Length)],
+            CrashSite = CrashSite[..Math.Min(maxLength, CrashSite.Length)],
+            CallStack = TruncateUtils.TruncateList(CallStack, maxLength),
+            CallStackSha256 = CallStackSha256[..Math.Min(maxLength, CallStackSha256.Length)],
+            InputSha256 = InputSha256[..Math.Min(maxLength, InputSha256.Length)],
+            AsanLog = AsanLog?[..Math.Min(maxLength, AsanLog.Length)],
+            ScarinessDescription = ScarinessDescription?[..Math.Min(maxLength, ScarinessDescription.Length)],
+            MinimizedStack = MinimizedStack != null ? TruncateUtils.TruncateList(MinimizedStack, maxLength) : MinimizedStack,
+            MinimizedStackSha256 = MinimizedStackSha256?[..Math.Min(maxLength, MinimizedStackSha256.Length)],
+            MinimizedStackFunctionNames = MinimizedStackFunctionNames != null ? TruncateUtils.TruncateList(MinimizedStackFunctionNames, maxLength) : MinimizedStackFunctionNames,
+            MinimizedStackFunctionNamesSha256 = MinimizedStackFunctionNamesSha256?[..Math.Min(maxLength, MinimizedStackFunctionNamesSha256.Length)],
+            MinimizedStackFunctionLines = MinimizedStackFunctionLines != null ? TruncateUtils.TruncateList(MinimizedStackFunctionLines, maxLength) : MinimizedStackFunctionLines,
+            MinimizedStackFunctionLinesSha256 = MinimizedStackFunctionLinesSha256?[..Math.Min(maxLength, MinimizedStackFunctionLinesSha256.Length)],
+            ToolName = ToolName?[..Math.Min(maxLength, ToolName.Length)],
+            ToolVersion = ToolVersion?[..Math.Min(maxLength, ToolVersion.Length)],
+            OnefuzzVersion = OnefuzzVersion?[..Math.Min(maxLength, OnefuzzVersion.Length)],
+        };
+    }
+}
 
 public record NoReproReport(
     string InputSha,
@@ -468,18 +491,40 @@ public record NoReproReport(
     Guid JobId,
     long Tries,
     string? Error
-);
+) : ITruncatable<NoReproReport> {
+    public NoReproReport Truncate(int maxLength) {
+        return this with {
+            Executable = Executable?[..maxLength],
+            Error = Error?[..maxLength]
+        };
+    }
+}
 
 public record CrashTestResult(
     Report? CrashReport,
     NoReproReport? NoReproReport
-);
+) : ITruncatable<CrashTestResult> {
+    public CrashTestResult Truncate(int maxLength) {
+        return new CrashTestResult(
+            CrashReport?.Truncate(maxLength),
+            NoReproReport?.Truncate(maxLength)
+        );
+    }
+}
 
 public record RegressionReport(
     CrashTestResult CrashTestResult,
-    CrashTestResult? OriginalCrashTestResult
-) : IReport;
-
+    CrashTestResult? OriginalCrashTestResult,
+    Uri? ReportUrl
+) : IReport, ITruncatable<RegressionReport> {
+    public RegressionReport Truncate(int maxLength) {
+        return new RegressionReport(
+            CrashTestResult.Truncate(maxLength),
+            OriginalCrashTestResult?.Truncate(maxLength),
+            ReportUrl
+        );
+    }
+}
 
 [JsonConverter(typeof(NotificationTemplateConverter))]
 #pragma warning disable CA1715
@@ -968,3 +1013,7 @@ public record TemplateRenderContext(
     string ReportFilename,
     string ReproCmd
 );
+
+public interface ITruncatable<T> {
+    public T Truncate(int maxLength);
+}

src/ApiService/ApiService/onefuzzlib/Reports.cs

@@ -44,21 +44,27 @@ public Reports(ILogTracer log, IContainers containers) {
             return null;
         }
 
-        return ParseReportOrRegression(blob.ToString(), filePath, expectReports);
+        var reportUrl = await _containers.GetFileUrl(container, fileName, StorageType.Corpus);
+
+        return ParseReportOrRegression(blob.ToString(), filePath, reportUrl, expectReports);
     }
 
-    private IReport? ParseReportOrRegression(string content, string? filePath, bool expectReports = false) {
+    private IReport? ParseReportOrRegression(string content, string? filePath, Uri? reportUrl, bool expectReports = false) {
         var regressionReport = JsonSerializer.Deserialize<RegressionReport>(content, EntityConverter.GetJsonSerializerOptions());
         if (regressionReport == null || regressionReport.CrashTestResult == null) {
             var report = JsonSerializer.Deserialize<Report>(content, EntityConverter.GetJsonSerializerOptions());
             if (expectReports && report == null) {
                 _log.Error($"unable to parse report ({filePath:Tag:FilePath}) as a report or regression");
                 return null;
             }
-            return report;
+            return report != null ? report with { ReportUrl = reportUrl } : report;
         }
-        return regressionReport;
+        return regressionReport != null ? regressionReport with { ReportUrl = reportUrl } : regressionReport;
     }
 }
 
-public interface IReport { };
+public interface IReport {
+    Uri? ReportUrl {
+        init;
+    }
+};

src/ApiService/ApiService/onefuzzlib/Utils.cs

@@ -39,3 +39,10 @@ public static async IAsyncEnumerable<List<TSource>> Chunk<TSource>(this IAsyncEn
         }
     }
 }
+
+public static class TruncateUtils {
+    public static List<string> TruncateList(List<string> data, int maxLength) {
+        int currentLength = 0;
+        return data.TakeWhile(curr => (currentLength += curr.Length) <= maxLength).ToList();
+    }
+}

src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs

@@ -34,6 +34,8 @@ async public Async.Task SendEvent(EventMessage eventMessage) {
     }
 
     async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) {
+        (string, string)[] tags = { ("WebhookId", webhook.WebhookId.ToString()), ("EventId", eventMessage.EventId.ToString()) };
+
         var message = new WebhookMessageLog(
              EventId: eventMessage.EventId,
              EventType: eventMessage.EventType,
@@ -46,8 +48,18 @@ async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) {
 
         var r = await _context.WebhookMessageLogOperations.Replace(message);
         if (!r.IsOk) {
-            _logTracer.WithHttpStatus(r.ErrorV).Error($"Failed to replace webhook message log {webhook.WebhookId:Tag:WebhookId} - {eventMessage.EventId:Tag:EventId}");
+            if (r.ErrorV.Reason.Contains("The entity is larger than the maximum allowed size") && eventMessage.Event is ITruncatable<BaseEvent> truncatableEvent) {
+                _logTracer.WithTags(tags).Warning($"The WebhookMessageLog was too long. Truncating event data and trying again.");
+                var truncatedEventMessage = message with {
+                    Event = truncatableEvent.Truncate(1000)
+                };
+                r = await _context.WebhookMessageLogOperations.Replace(truncatedEventMessage);
+            }
+            if (!r.IsOk) {
+                _logTracer.WithHttpStatus(r.ErrorV).WithTags(tags).Error($"Failed to replace webhook message log {webhook.WebhookId:Tag:WebhookId} - {eventMessage.EventId:Tag:EventId}");
+            }
         }
+
         await _context.WebhookMessageLogOperations.QueueWebhook(message);
     }
 
@@ -57,7 +69,7 @@ public async Async.Task<bool> Send(WebhookMessageLog messageLog) {
             throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found");
         }
 
-        var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat);
+        var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event!, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat);
 
         var headers = new Dictionary<string, string> { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } };
 

src/ApiService/ApiService/onefuzzlib/orm/Orm.cs

@@ -79,15 +79,19 @@ public async IAsyncEnumerable<T> QueryAsync(string? filter = null) {
         }
 
         public async Task<ResultVoid<(HttpStatusCode Status, string Reason)>> Replace(T entity) {
-            var tableClient = await GetTableClient(typeof(T).Name);
-            var tableEntity = _entityConverter.ToTableEntity(entity);
-            var response = await tableClient.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace);
-            if (response.IsError) {
-                return ResultVoid<(HttpStatusCode, string)>.Error(((HttpStatusCode)response.Status, response.ReasonPhrase));
-            } else {
-                // update ETag on success
-                entity.ETag = response.Headers.ETag;
-                return ResultVoid<(HttpStatusCode, string)>.Ok();
+            try {
+                var tableClient = await GetTableClient(typeof(T).Name);
+                var tableEntity = _entityConverter.ToTableEntity(entity);
+                var response = await tableClient.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace);
+                if (response.IsError) {
+                    return ResultVoid<(HttpStatusCode, string)>.Error(((HttpStatusCode)response.Status, response.ReasonPhrase));
+                } else {
+                    // update ETag on success
+                    entity.ETag = response.Headers.ETag;
+                    return ResultVoid<(HttpStatusCode, string)>.Ok();
+                }
+            } catch (RequestFailedException ex) {
+                return ResultVoid<(HttpStatusCode, string)>.Error(((HttpStatusCode)ex.Status, ex.Message));
             }
         }
 

src/ApiService/IntegrationTests/ReproVmssTests.cs

@@ -190,6 +190,7 @@ public async Async.Task CannotCreateVMForMissingReport() {
                 null,
                 null,
                 null,
+                null,
                 null
                 );
 

src/ApiService/Tests/OrmModelsTest.cs

@@ -300,7 +300,7 @@ public static Gen<WebhookMessageEventGrid> WebhookMessageEventGrid() {
         }
 
         public static Gen<Report> Report() {
-            return Arb.Generate<Tuple<string, BlobRef, List<string>, Guid, int>>().Select(
+            return Arb.Generate<Tuple<string, BlobRef, List<string>, Guid, int, Uri?>>().Select(
                 arg =>
                     new Report(
                         InputUrl: arg.Item1,
@@ -324,8 +324,8 @@ public static Gen<Report> Report() {
                         MinimizedStackFunctionLinesSha256: arg.Item1,
                         ToolName: arg.Item1,
                         ToolVersion: arg.Item1,
-                        OnefuzzVersion: arg.Item1
-
+                        OnefuzzVersion: arg.Item1,
+                        ReportUrl: arg.Item6
 
                     )
             );
@@ -357,11 +357,12 @@ public static Gen<CrashTestResult> CrashTestResult() {
         }
 
         public static Gen<RegressionReport> RegressionReport() {
-            return Arb.Generate<Tuple<CrashTestResult, CrashTestResult?>>().Select(
+            return Arb.Generate<Tuple<CrashTestResult, CrashTestResult?, Uri?>>().Select(
                 arg =>
                     new RegressionReport(
                         arg.Item1,
-                        arg.Item2
+                        arg.Item2,
+                        arg.Item3
                     )
             );
         }

src/ApiService/Tests/TemplateTests.cs

@@ -190,6 +190,7 @@ private static Report GetReport() {
             null,
             null,
             null,
+            null,
             null
         );
     }

src/ApiService/Tests/TruncationTests.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using FluentAssertions;
+using Microsoft.OneFuzz.Service;
+using Xunit;
+
+namespace Tests;
+
+public class TruncationTests {
+    [Fact]
+    public static void ReportIsTruncatable() {
+        var report = GenerateReport();
+
+        var truncatedReport = report.Truncate(5);
+
+        truncatedReport.Executable.Should().Be("SOMES");
+        truncatedReport.CallStack.Count.Should().Be(0);
+    }
+
+    [Fact]
+    public static void TestListTruncation() {
+        var testList = new List<string> {
+            "1", "2", "3", "456"
+        };
+
+        var truncatedList = TruncateUtils.TruncateList(testList, 3);
+        truncatedList.Count.Should().Be(3);
+        truncatedList.Should().BeEquivalentTo(new[] { "1", "2", "3" });
+    }
+
+    [Fact]
+    public static void TestNestedTruncation() {
+        var eventCrashReported = new EventCrashReported(
+            GenerateReport(),
+            Container.Parse("123"),
+            "abc",
+            null
+        );
+
+        var truncatedEvent = eventCrashReported.Truncate(3) as EventCrashReported;
+        truncatedEvent.Should().NotBeNull();
+        truncatedEvent?.Report.Executable.Should().Be("SOM");
+        truncatedEvent?.Report.CallStack.Count.Should().Be(0);
+    }
+
+    private static Report GenerateReport() {
+        return new Report(
+            null,
+            null,
+            "SOMESUPRTLONGSTRINGSOMESUPRTLONGSTRINGSOMESUPRTLONGSTRINGSOMESUPRTLONGSTRING",
+            "abc",
+            "abc",
+            new List<string> { "SOMESUPRTLONGSTRINGSOMESUPRTLONGSTRING" },
+            "abc",
+            "abc",
+            null,
+            Guid.Empty,
+            Guid.Empty,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            new Uri("http://example.com")
+        );
+    }
+}