Merge remote-tracking branch 'upstream/release/13.0' into v13/feature…

…/webhook-all-the-things

src/Umbraco.Core/Configuration/Models/WebhookSettings.cs

@@ -8,6 +8,9 @@ public class WebhookSettings
     private const bool StaticEnabled = true;
     private const int StaticMaximumRetries = 5;
     internal const string StaticPeriod = "00:00:10";
+    private const bool StaticEnableLoggingCleanup = true;
+    private const int StaticKeepLogsForDays = 30;
+
 
     /// <summary>
     ///     Gets or sets a value indicating whether webhooks are enabled.
@@ -38,4 +41,26 @@ public class WebhookSettings
     /// </summary>
     [DefaultValue(StaticPeriod)]
     public TimeSpan Period { get; set; } = TimeSpan.Parse(StaticPeriod);
+
+    /// <summary>
+    ///     Gets or sets a value indicating whether cleanup of webhook logs are enabled.
+    /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         By default, cleanup is enabled.
+    ///     </para>
+    /// </remarks>
+    [DefaultValue(StaticEnableLoggingCleanup)]
+    public bool EnableLoggingCleanup { get; set; } = StaticEnableLoggingCleanup;
+
+    /// <summary>
+    ///     Gets or sets a value indicating number of days to keep logs for.
+    /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///         By default, logs are kept for 30 days.
+    ///     </para>
+    /// </remarks>
+    [DefaultValue(StaticKeepLogsForDays)]
+    public int KeepLogsForDays { get; set; } = StaticKeepLogsForDays;
 }

src/Umbraco.Core/Persistence/Constants-Locks.cs

@@ -72,8 +72,13 @@ public static class Locks
         public const int ScheduledPublishing = -341;
 
         /// <summary>
-        ///     ScheduledPublishing job.
+        ///    All Webhook requests.
         /// </summary>
         public const int WebhookRequest = -342;
+
+        /// <summary>
+        ///     All webhook logs.
+        /// </summary>
+        public const int WebhookLogs = -343;
     }
 }

src/Umbraco.Core/Persistence/Repositories/IWebhookLogRepository.cs

@@ -1,5 +1,4 @@
 using Umbraco.Cms.Core.Models;
-using Umbraco.Cms.Core.Webhooks;
 
 namespace Umbraco.Cms.Core.Persistence.Repositories;
 
@@ -8,4 +7,8 @@ public interface IWebhookLogRepository
     Task CreateAsync(WebhookLog log);
 
     Task<PagedModel<WebhookLog>> GetPagedAsync(int skip, int take);
+
+    Task<IEnumerable<WebhookLog>> GetOlderThanDate(DateTime date);
+
+    Task DeleteByIds(int[] ids);
 }

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/WebhookFiring.cs

@@ -1,4 +1,5 @@
-using System.Text;
+using System.Net.Mime;
+using System.Text;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core;
@@ -87,12 +88,11 @@ await Task.WhenAll(requests.Select(request =>
         }));
     }
 
-    private async Task<HttpResponseMessage?> SendRequestAsync(Webhook webhook, string eventName, object? payload, int retryCount, CancellationToken cancellationToken)
+    private async Task<HttpResponseMessage?> SendRequestAsync(Webhook webhook, string eventName, string? serializedObject, int retryCount, CancellationToken cancellationToken)
     {
         using var httpClient = new HttpClient();
 
-        var serializedObject = _jsonSerializer.Serialize(payload);
-        var stringContent = new StringContent(serializedObject, Encoding.UTF8, "application/json");
+        var stringContent = new StringContent(serializedObject ?? string.Empty, Encoding.UTF8, MediaTypeNames.Application.Json);
         stringContent.Headers.TryAddWithoutValidation("Umb-Webhook-Event", eventName);
 
         foreach (KeyValuePair<string, string> header in webhook.Headers)

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/WebhookLoggingCleanup.cs

@@ -0,0 +1,69 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.Persistence.Repositories;
+using Umbraco.Cms.Core.Scoping;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+
+/// <summary>
+/// Daily background job that removes all webhook log data older than x days as defined by <see cref="WebhookSettings.KeepLogsForDays"/>
+/// </summary>
+public class WebhookLoggingCleanup : IRecurringBackgroundJob
+{
+    private readonly ILogger<WebhookLoggingCleanup> _logger;
+    private readonly WebhookSettings _webhookSettings;
+    private readonly IWebhookLogRepository _webhookLogRepository;
+    private readonly ICoreScopeProvider _coreScopeProvider;
+
+    public WebhookLoggingCleanup(ILogger<WebhookLoggingCleanup> logger, IOptionsMonitor<WebhookSettings> webhookSettings, IWebhookLogRepository webhookLogRepository, ICoreScopeProvider coreScopeProvider)
+    {
+        _logger = logger;
+        _webhookSettings = webhookSettings.CurrentValue;
+        _webhookLogRepository = webhookLogRepository;
+        _coreScopeProvider = coreScopeProvider;
+    }
+
+    /// <inheritdoc />
+    // No-op event as the period never changes on this job
+    public event EventHandler PeriodChanged
+    {
+        add { } remove { }
+    }
+
+    /// <inheritdoc />
+    public TimeSpan Period => TimeSpan.FromDays(1);
+
+    /// <inheritdoc />
+    public TimeSpan Delay { get; } = TimeSpan.FromSeconds(20);
+
+    /// <inheritdoc />
+    public async Task RunJobAsync()
+    {
+        if (_webhookSettings.EnableLoggingCleanup is false)
+        {
+            _logger.LogInformation("WebhookLoggingCleanup task will not run as it has been globally disabled via configuration");
+            return;
+        }
+
+        IEnumerable<WebhookLog> webhookLogs;
+        using (ICoreScope scope = _coreScopeProvider.CreateCoreScope())
+        {
+            scope.ReadLock(Constants.Locks.WebhookLogs);
+            webhookLogs = await _webhookLogRepository.GetOlderThanDate(DateTime.Now - TimeSpan.FromDays(_webhookSettings.KeepLogsForDays));
+            scope.Complete();
+        }
+
+        foreach (IEnumerable<WebhookLog> group in webhookLogs.InGroupsOf(Constants.Sql.MaxParameterCount))
+        {
+            using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+            scope.WriteLock(Constants.Locks.WebhookLogs);
+            await _webhookLogRepository.DeleteByIds(group.Select(x => x.Id).ToArray());
+
+            scope.Complete();
+        }
+    }
+}

src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs

@@ -1006,6 +1006,7 @@ private void CreateLockData()
         _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.ScheduledPublishing, Name = "ScheduledPublishing" });
         _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.MainDom, Name = "MainDom" });
         _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.WebhookRequest, Name = "WebhookRequest" });
+        _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.WebhookLogs, Name = "WebhookLogs" });
     }
 
     private void CreateContentTypeData()

src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs

@@ -100,5 +100,6 @@ protected virtual void DefinePlan()
         To<V_13_0_0.RenameEventNameColumn>("{D5139400-E507-4259-A542-C67358F7E329}");
         To<V_13_0_0.AddWebhookRequest>("{4E652F18-9A29-4656-A899-E3F39069C47E}");
         To<V_13_0_0.RenameWebhookIdToKey>("{148714C8-FE0D-4553-B034-439D91468761}");
+        To<V_13_0_0.AddWebhookDatabaseLock>("{23BA95A4-FCCE-49B0-8AA1-45312B103A9B}");
     }
 }

src/Umbraco.Infrastructure/Migrations/Upgrade/V_13_0_0/AddWebhookDatabaseLock.cs

@@ -0,0 +1,30 @@
+using NPoco;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Infrastructure.Persistence;
+using Umbraco.Cms.Infrastructure.Persistence.Dtos;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_13_0_0;
+
+public class AddWebhookDatabaseLock : MigrationBase
+{
+    public AddWebhookDatabaseLock(IMigrationContext context)
+        : base(context)
+    {
+    }
+
+    protected override void Migrate()
+    {
+        Sql<ISqlContext> sql = Database.SqlContext.Sql()
+            .Select<LockDto>()
+            .From<LockDto>()
+            .Where<LockDto>(x => x.Id == Constants.Locks.WebhookLogs);
+
+        LockDto? webhookLogsLock = Database.FirstOrDefault<LockDto>(sql);
+
+        if (webhookLogsLock is null)
+        {
+            Database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.WebhookLogs, Name = "WebhookLogs" });
+        }
+    }
+}

src/Umbraco.Infrastructure/Persistence/Repositories/Implement/WebhookLogRepository.cs

@@ -2,7 +2,6 @@
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Persistence.Repositories;
-using Umbraco.Cms.Core.Webhooks;
 using Umbraco.Cms.Infrastructure.Persistence.Dtos;
 using Umbraco.Cms.Infrastructure.Persistence.Factories;
 using Umbraco.Cms.Infrastructure.Scoping;
@@ -14,31 +13,65 @@ public class WebhookLogRepository : IWebhookLogRepository
 {
     private readonly IScopeAccessor _scopeAccessor;
 
+    private IUmbracoDatabase Database
+    {
+        get
+        {
+            if (_scopeAccessor.AmbientScope is null)
+            {
+                throw new NotSupportedException("Need to be executed in a scope");
+            }
+
+            return _scopeAccessor.AmbientScope.Database;
+        }
+    }
+
     public WebhookLogRepository(IScopeAccessor scopeAccessor) => _scopeAccessor = scopeAccessor;
 
     public async Task CreateAsync(WebhookLog log)
     {
         WebhookLogDto dto = WebhookLogFactory.CreateDto(log);
-        var result = await _scopeAccessor.AmbientScope?.Database.InsertAsync(dto)!;
+        var result = await Database.InsertAsync(dto)!;
         var id = Convert.ToInt32(result);
         log.Id = id;
     }
 
     public async Task<PagedModel<WebhookLog>> GetPagedAsync(int skip, int take)
     {
-        Sql<ISqlContext>? sql = _scopeAccessor.AmbientScope?.Database.SqlContext.Sql()
+        Sql<ISqlContext> sql = Database.SqlContext.Sql()
             .Select<WebhookLogDto>()
             .From<WebhookLogDto>()
             .OrderByDescending<WebhookLogDto>(x => x.Date);
 
         PaginationHelper.ConvertSkipTakeToPaging(skip, take, out var pageNumber, out var pageSize);
 
-        Page<WebhookLogDto>? page = await _scopeAccessor.AmbientScope?.Database.PageAsync<WebhookLogDto>(pageNumber + 1, pageSize, sql)!;
+        Page<WebhookLogDto>? page = await Database.PageAsync<WebhookLogDto>(pageNumber + 1, pageSize, sql)!;
 
         return new PagedModel<WebhookLog>
         {
             Total = page.TotalItems,
             Items = page.Items.Select(WebhookLogFactory.DtoToEntity),
         };
     }
+
+    public async Task<IEnumerable<WebhookLog>> GetOlderThanDate(DateTime date)
+    {
+        Sql<ISqlContext> sql = Database.SqlContext.Sql()
+            .Select<WebhookLogDto>()
+            .From<WebhookLogDto>()
+            .Where<WebhookLogDto>(log => log.Date < date);
+
+        List<WebhookLogDto>? logs = await Database.FetchAsync<WebhookLogDto>(sql);
+
+        return logs.Select(WebhookLogFactory.DtoToEntity);
+    }
+
+    public async Task DeleteByIds(int[] ids)
+    {
+        Sql<ISqlContext> query = Database.SqlContext.Sql()
+            .Delete<WebhookLogDto>()
+            .WhereIn<WebhookLogDto>(x => x.Id, ids);
+
+        await Database.ExecuteAsync(query);
+    }
 }

src/Umbraco.PublishedCache.NuCache/ContentStore.cs

@@ -736,16 +736,22 @@ public void UpdateDataTypesLocked(IEnumerable<int> dataTypeIds, Func<int, IPubli
     {
         EnsureLocked();
 
-        IPublishedContentType?[] contentTypes = _contentTypesById
+        IPublishedContentType[] contentTypes = _contentTypesById
             .Where(kvp =>
                 kvp.Value.Value != null &&
                 kvp.Value.Value.PropertyTypes.Any(p => dataTypeIds.Contains(p.DataType.Id)))
             .Select(kvp => kvp.Value.Value)
             .Select(x => getContentType(x!.Id))
-            .Where(x => x != null) // poof, gone, very unlikely and probably an anomaly
+            .WhereNotNull() // poof, gone, very unlikely and probably an anomaly
             .ToArray();
 
-        var contentTypeIdsA = contentTypes.Select(x => x!.Id).ToArray();
+        // all content types that are affected by this data type update must be updated
+        foreach (IPublishedContentType contentType in contentTypes)
+        {
+            SetContentTypeLocked(contentType);
+        }
+
+        var contentTypeIdsA = contentTypes.Select(x => x.Id).ToArray();
         var contentTypeNodes = new Dictionary<int, List<int>>();
         foreach (var id in contentTypeIdsA)
         {
@@ -761,7 +767,7 @@ public void UpdateDataTypesLocked(IEnumerable<int> dataTypeIds, Func<int, IPubli
             }
         }
 
-        foreach (IPublishedContentType contentType in contentTypes.WhereNotNull())
+        foreach (IPublishedContentType contentType in contentTypes)
         {
             // again, weird situation
             if (contentTypeNodes.ContainsKey(contentType.Id) == false)

src/Umbraco.Web.Common/DependencyInjection/UmbracoBuilderExtensions.cs

@@ -216,6 +216,7 @@ public static IUmbracoBuilder AddRecurringBackgroundJobs(this IUmbracoBuilder bu
         builder.Services.AddRecurringBackgroundJob<InstructionProcessJob>();
         builder.Services.AddRecurringBackgroundJob<TouchServerJob>();
         builder.Services.AddRecurringBackgroundJob<WebhookFiring>();
+        builder.Services.AddRecurringBackgroundJob<WebhookLoggingCleanup>();
         builder.Services.AddRecurringBackgroundJob(provider =>
             new ReportSiteJob(
                 provider.GetRequiredService<ILogger<ReportSiteJob>>(),