Merge branch 'v15/dev' into contrib

Directory.Build.props

@@ -14,7 +14,7 @@
     <NeutralLanguage>en-US</NeutralLanguage>
     <Nullable>enable</Nullable>
     <WarningsAsErrors>nullable</WarningsAsErrors>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <TreatWarningsAsErrors Condition="'$(Configuration)' != 'Debug'">true</TreatWarningsAsErrors>
     <ImplicitUsings>enable</ImplicitUsings>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>

Directory.Packages.props

@@ -52,7 +52,7 @@
     <PackageVersion Include="K4os.Compression.LZ4" Version="1.3.8" />
     <PackageVersion Include="MailKit" Version="4.8.0" />
     <PackageVersion Include="Markdown" Version="2.2.1" />
-    <PackageVersion Include="MessagePack" Version="2.5.172" />
+    <PackageVersion Include="MessagePack" Version="2.5.187" />
     <PackageVersion Include="MiniProfiler.AspNetCore.Mvc" Version="4.3.8" />
     <PackageVersion Include="MiniProfiler.Shared" Version="4.3.8" />
     <PackageVersion Include="ncrontab" Version="3.3.3" />
@@ -86,7 +86,7 @@
     <!-- Dazinator.Extensions.FileProviders brings in a vulnerable version of System.Net.Http -->
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
     <!-- Examine brings in a vulnerable version of System.Security.Cryptography.Xml -->
-    <PackageVersion Include="System.Security.Cryptography.Xml" Version="9.0.0-rc.1.24431.7" />
+    <PackageVersion Include="System.Security.Cryptography.Xml" Version="9.0.0-rc.2.24473.5" />
     <!-- Both Dazinator.Extensions.FileProviders and MiniProfiler.AspNetCore.Mvc bring in a vulnerable version of System.Text.RegularExpressions -->
     <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />
     <!-- Both OpenIddict.AspNetCore, Npoco.SqlServer and Microsoft.EntityFrameworkCore.SqlServer bring in a vulnerable version of Microsoft.IdentityModel.JsonWebTokens -->

build/azure-pipelines.yml

@@ -71,7 +71,7 @@ stages:
       - job: A
         displayName: Build Umbraco CMS
         pool:
-          vmImage: 'ubuntu-latest'
+          vmImage: 'windows-latest'
         steps:
           - checkout: self
             submodules: true
@@ -284,6 +284,7 @@ stages:
     jobs:
       # Integration Tests (SQLite)
       - job:
+        timeoutInMinutes: 180
         displayName: Integration Tests (SQLite)
         strategy:
           matrix:
@@ -513,7 +514,7 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
-          - pwsh: npm run smokeTest --ignore-certificate-errors
+          - pwsh: npm run smokeTestSqlite --ignore-certificate-errors
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest

build/nightly-E2E-test-pipelines.yml

@@ -207,9 +207,9 @@ stages:
 
           # Test
           - ${{ if eq(parameters.runSmokeTests, true) }}:
-              pwsh: npm run smokeTest --ignore-certificate-errors
+              pwsh: npm run smokeTestSqlite --ignore-certificate-errors
             ${{ else }}:
-              pwsh: npm run test --ignore-certificate-errors
+              pwsh: npm run testSqlite --ignore-certificate-errors
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest

src/Umbraco.Cms.Api.Management/Controllers/DocumentType/Item/ItemDocumentTypeItemController.cs

@@ -32,7 +32,7 @@ public async Task<IActionResult> Item(
             return Ok(Enumerable.Empty<DocumentTypeItemResponseModel>());
         }
 
-        IEnumerable<IContentType> contentTypes = _contentTypeService.GetAll(ids);
+        IEnumerable<IContentType> contentTypes = _contentTypeService.GetMany(ids);
         List<DocumentTypeItemResponseModel> responseModels = _mapper.MapEnumerable<IContentType, DocumentTypeItemResponseModel>(contentTypes);
         return await Task.FromResult(Ok(responseModels));
     }

src/Umbraco.Cms.Api.Management/Controllers/DocumentType/Item/SearchDocumentTypeItemController.cs

@@ -35,7 +35,7 @@ public async Task<IActionResult> Search(CancellationToken cancellationToken, str
             return await Task.FromResult(Ok(new PagedModel<DocumentTypeItemResponseModel> { Total = searchResult.Total }));
         }
 
-        IEnumerable<IContentType> contentTypes = _contentTypeService.GetAll(searchResult.Items.Select(item => item.Key).ToArray().EmptyNull());
+        IEnumerable<IContentType> contentTypes = _contentTypeService.GetMany(searchResult.Items.Select(item => item.Key).ToArray().EmptyNull());
         var result = new PagedModel<DocumentTypeItemResponseModel>
         {
             Items = _mapper.MapEnumerable<IContentType, DocumentTypeItemResponseModel>(contentTypes),

src/Umbraco.Cms.Api.Management/Controllers/DocumentType/Tree/DocumentTypeTreeControllerBase.cs

@@ -29,7 +29,7 @@ public DocumentTypeTreeControllerBase(IEntityService entityService, IContentType
     protected override DocumentTypeTreeItemResponseModel[] MapTreeItemViewModels(Guid? parentKey, IEntitySlim[] entities)
     {
         var contentTypes = _contentTypeService
-            .GetAll(entities.Select(entity => entity.Id).ToArray())
+            .GetMany(entities.Select(entity => entity.Id).ToArray())
             .ToDictionary(contentType => contentType.Id);
 
         return entities.Select(entity =>

src/Umbraco.Cms.Api.Management/Controllers/MediaType/Item/ItemMediaTypeItemController.cs

@@ -32,7 +32,7 @@ public async Task<IActionResult> Item(
             return Ok(Enumerable.Empty<MediaTypeItemResponseModel>());
         }
 
-        IEnumerable<IMediaType> mediaTypes = _mediaTypeService.GetAll(ids);
+        IEnumerable<IMediaType> mediaTypes = _mediaTypeService.GetMany(ids);
         List<MediaTypeItemResponseModel> responseModels = _mapper.MapEnumerable<IMediaType, MediaTypeItemResponseModel>(mediaTypes);
         return await Task.FromResult(Ok(responseModels));
     }

src/Umbraco.Cms.Api.Management/Controllers/MediaType/Item/SearchMediaTypeItemController.cs

@@ -35,7 +35,7 @@ public async Task<IActionResult> Search(CancellationToken cancellationToken, str
             return await Task.FromResult(Ok(new PagedModel<MediaTypeItemResponseModel> { Total = searchResult.Total }));
         }
 
-        IEnumerable<IMediaType> mediaTypes = _mediaTypeService.GetAll(searchResult.Items.Select(item => item.Key).ToArray().EmptyNull());
+        IEnumerable<IMediaType> mediaTypes = _mediaTypeService.GetMany(searchResult.Items.Select(item => item.Key).ToArray().EmptyNull());
         var result = new PagedModel<MediaTypeItemResponseModel>
         {
             Items = _mapper.MapEnumerable<IMediaType, MediaTypeItemResponseModel>(mediaTypes),

src/Umbraco.Cms.Api.Management/Controllers/MediaType/Tree/MediaTypeTreeControllerBase.cs

@@ -30,7 +30,7 @@ public MediaTypeTreeControllerBase(IEntityService entityService, IMediaTypeServi
     protected override MediaTypeTreeItemResponseModel[] MapTreeItemViewModels(Guid? parentKey, IEntitySlim[] entities)
     {
         var mediaTypes = _mediaTypeService
-            .GetAll(entities.Select(entity => entity.Id).ToArray())
+            .GetMany(entities.Select(entity => entity.Id).ToArray())
             .ToDictionary(contentType => contentType.Id);
 
         return entities.Select(entity =>

src/Umbraco.Cms.Api.Management/Controllers/MemberType/Item/ItemMemberTypeItemController.cs

@@ -32,7 +32,7 @@ public async Task<IActionResult> Item(
             return Ok(Enumerable.Empty<MemberTypeItemResponseModel>());
         }
 
-        IEnumerable<IMemberType> memberTypes = _memberTypeService.GetAll(ids);
+        IEnumerable<IMemberType> memberTypes = _memberTypeService.GetMany(ids);
         List<MemberTypeItemResponseModel> responseModels = _mapper.MapEnumerable<IMemberType, MemberTypeItemResponseModel>(memberTypes);
         return Ok(responseModels);
     }

src/Umbraco.Cms.Api.Management/Controllers/MemberType/Item/SearchMemberTypeItemController.cs

@@ -34,7 +34,7 @@ public async Task<IActionResult> Search(CancellationToken cancellationToken, str
             return await Task.FromResult(Ok(new PagedModel<MemberTypeItemResponseModel> { Total = searchResult.Total }));
         }
 
-        IEnumerable<IMemberType> memberTypes = _memberTypeService.GetAll(searchResult.Items.Select(item => item.Key).ToArray());
+        IEnumerable<IMemberType> memberTypes = _memberTypeService.GetMany(searchResult.Items.Select(item => item.Key).ToArray());
         var result = new PagedModel<MemberTypeItemResponseModel>
         {
             Items = _mapper.MapEnumerable<IMemberType, MemberTypeItemResponseModel>(memberTypes),

src/Umbraco.Cms.Api.Management/Controllers/MemberType/Tree/MemberTypeTreeControllerBase.cs

@@ -27,7 +27,7 @@ public MemberTypeTreeControllerBase(IEntityService entityService, IMemberTypeSer
     protected override MemberTypeTreeItemResponseModel[] MapTreeItemViewModels(Guid? parentKey, IEntitySlim[] entities)
     {
         var memberTypes = _memberTypeService
-            .GetAll(entities.Select(entity => entity.Id).ToArray())
+            .GetMany(entities.Select(entity => entity.Id).ToArray())
             .ToDictionary(contentType => contentType.Id);
 
         return entities.Select(entity =>

src/Umbraco.Cms.Api.Management/Controllers/Template/Query/ExecuteTemplateQueryController.cs

@@ -60,7 +60,7 @@ public async Task<ActionResult<TemplateQueryResultResponseModel>> Execute(
         timer.Stop();
 
         var contentTypeIconsByKey = _contentTypeService
-            .GetAll(results.Select(content => content.ContentType.Key).Distinct())
+            .GetMany(results.Select(content => content.ContentType.Key).Distinct())
             .ToDictionary(contentType => contentType.Key, contentType => contentType.Icon);
 
         return await Task.FromResult(Ok(new TemplateQueryResultResponseModel

src/Umbraco.Cms.Api.Management/Factories/DataTypeReferencePresentationFactory.cs

@@ -26,9 +26,9 @@ public IEnumerable<DataTypeReferenceResponseModel> CreateDataTypeReferenceViewMo
     {
         var getContentTypesByObjectType = new Dictionary<string, Func<IEnumerable<Guid>, IEnumerable<IContentTypeBase>>>
         {
-            { UmbracoObjectTypes.DocumentType.GetUdiType(), keys => _contentTypeService.GetAll(keys) },
-            { UmbracoObjectTypes.MediaType.GetUdiType(), keys => _mediaTypeService.GetAll(keys) },
-            { UmbracoObjectTypes.MemberType.GetUdiType(), keys => _memberTypeService.GetAll(keys) }
+            { UmbracoObjectTypes.DocumentType.GetUdiType(), keys => _contentTypeService.GetMany(keys) },
+            { UmbracoObjectTypes.MediaType.GetUdiType(), keys => _mediaTypeService.GetMany(keys) },
+            { UmbracoObjectTypes.MemberType.GetUdiType(), keys => _memberTypeService.GetMany(keys) }
         };
 
         foreach (IGrouping<string, KeyValuePair<Udi, IEnumerable<string>>> usagesByEntityType in dataTypeUsages.GroupBy(u => u.Key.EntityType))

src/Umbraco.Cms.Persistence.EFCore/Scoping/AmbientEFCoreScopeStack.cs

@@ -5,36 +5,45 @@ namespace Umbraco.Cms.Persistence.EFCore.Scoping;
 
 public class AmbientEFCoreScopeStack<TDbContext> : IAmbientEFCoreScopeStack<TDbContext> where TDbContext : DbContext
 {
-
+    private static Lock _lock = new();
     private static AsyncLocal<ConcurrentStack<IEfCoreScope<TDbContext>>> _stack = new();
 
     public IEfCoreScope<TDbContext>? AmbientScope
     {
         get
         {
-            if (_stack.Value?.TryPeek(out IEfCoreScope<TDbContext>? ambientScope) ?? false)
+            lock (_lock)
             {
-                return ambientScope;
-            }
+                if (_stack.Value?.TryPeek(out IEfCoreScope<TDbContext>? ambientScope) ?? false)
+                {
+                    return ambientScope;
+                }
 
-            return null;
+                return null;
+            }
         }
     }
 
     public IEfCoreScope<TDbContext> Pop()
     {
-        if (_stack.Value?.TryPop(out IEfCoreScope<TDbContext>? ambientScope) ?? false)
+        lock (_lock)
         {
-            return ambientScope;
-        }
+            if (_stack.Value?.TryPop(out IEfCoreScope<TDbContext>? ambientScope) ?? false)
+            {
+                return ambientScope;
+            }
 
-        throw new InvalidOperationException("No AmbientScope was found.");
+            throw new InvalidOperationException("No AmbientScope was found.");
+        }
     }
 
     public void Push(IEfCoreScope<TDbContext> scope)
     {
-        _stack.Value ??= new ConcurrentStack<IEfCoreScope<TDbContext>>();
+        lock (_lock)
+        {
+            _stack.Value ??= new ConcurrentStack<IEfCoreScope<TDbContext>>();
 
-        _stack.Value.Push(scope);
+            _stack.Value.Push(scope);
+        }
     }
 }

src/Umbraco.Cms.StaticAssets/Umbraco.Cms.StaticAssets.csproj

@@ -4,6 +4,7 @@
     <Description>Contains the static assets needed to run Umbraco CMS.</Description>
     <AddRazorSupportForMvc>true</AddRazorSupportForMvc>
     <StaticWebAssetBasePath>/</StaticWebAssetBasePath>
+    <CompressionEnabled>false</CompressionEnabled> <!-- Disable compression. E.g. for umbraco backoffice files. These files should be precompressed by node and not let dotnet handle it -->
   </PropertyGroup>
 
   <PropertyGroup>
@@ -76,4 +77,4 @@
     </ItemGroup>
     <RemoveDir Directories="@(LoginDirectories)" />
   </Target>
-</Project>
+</Project>

src/Umbraco.Core/Cache/Refreshers/Implement/ContentCacheRefresher.cs

@@ -1,5 +1,3 @@
-using Microsoft.Extensions.DependencyInjection;
-using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Notifications;
@@ -22,6 +20,7 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
     private readonly IDocumentNavigationQueryService _documentNavigationQueryService;
     private readonly IDocumentNavigationManagementService _documentNavigationManagementService;
     private readonly IContentService _contentService;
+    private readonly IPublishStatusManagementService _publishStatusManagementService;
     private readonly IIdKeyMap _idKeyMap;
 
     public ContentCacheRefresher(
@@ -35,7 +34,8 @@ public ContentCacheRefresher(
         IDomainCacheService domainCacheService,
         IDocumentNavigationQueryService documentNavigationQueryService,
         IDocumentNavigationManagementService documentNavigationManagementService,
-        IContentService contentService)
+        IContentService contentService,
+        IPublishStatusManagementService publishStatusManagementService)
         : base(appCaches, serializer, eventAggregator, factory)
     {
         _idKeyMap = idKeyMap;
@@ -45,6 +45,7 @@ public ContentCacheRefresher(
         _documentNavigationQueryService = documentNavigationQueryService;
         _documentNavigationManagementService = documentNavigationManagementService;
         _contentService = contentService;
+        _publishStatusManagementService = publishStatusManagementService;
     }
 
     #region Indirect
@@ -109,6 +110,7 @@ public override void Refresh(JsonPayload[] payloads)
             HandleRouting(payload);
 
             HandleNavigation(payload);
+            HandlePublishedAsync(payload, CancellationToken.None).GetAwaiter().GetResult();
             _idKeyMap.ClearCache(payload.Id);
             if (payload.Key.HasValue)
             {
@@ -143,6 +145,13 @@ public override void Refresh(JsonPayload[] payloads)
 
     private void HandleNavigation(JsonPayload payload)
     {
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
+        {
+            _documentNavigationManagementService.RebuildAsync();
+            _documentNavigationManagementService.RebuildBinAsync();
+        }
+
         if (payload.Key is null)
         {
             return;
@@ -154,15 +163,9 @@ private void HandleNavigation(JsonPayload payload)
             _documentNavigationManagementService.RemoveFromBin(payload.Key.Value);
         }
 
-        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
-        {
-            _documentNavigationManagementService.RebuildAsync();
-            _documentNavigationManagementService.RebuildBinAsync();
-        }
-
         if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshNode))
         {
-            IContent? content = _contentService.GetById(payload.Id);
+            IContent? content = _contentService.GetById(payload.Key.Value);
 
             if (content is null)
             {
@@ -174,7 +177,7 @@ private void HandleNavigation(JsonPayload payload)
 
         if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
         {
-            IContent? content = _contentService.GetById(payload.Id);
+            IContent? content = _contentService.GetById(payload.Key.Value);
 
             if (content is null)
             {
@@ -194,7 +197,7 @@ private void HandleNavigationForSingleContent(IContent content)
         // First creation
         if (ExistsInNavigation(content.Key) is false && ExistsInNavigationBin(content.Key) is false)
         {
-            _documentNavigationManagementService.Add(content.Key, GetParentKey(content));
+            _documentNavigationManagementService.Add(content.Key, GetParentKey(content), content.SortOrder);
             if (content.Trashed)
             {
                 // If created as trashed, move to bin
@@ -210,14 +213,20 @@ private void HandleNavigationForSingleContent(IContent content)
             }
             else
             {
+                if (_documentNavigationQueryService.TryGetParentKey(content.Key, out Guid? oldParentKey) is false)
+                {
+                    return;
+                }
+
                 // It must have been saved. Check if parent is different
-                if (_documentNavigationQueryService.TryGetParentKey(content.Key, out var oldParentKey))
+                Guid? newParentKey = GetParentKey(content);
+                if (oldParentKey != newParentKey)
+                {
+                    _documentNavigationManagementService.Move(content.Key, newParentKey);
+                }
+                else
                 {
-                    Guid? newParentKey = GetParentKey(content);
-                    if (oldParentKey != newParentKey)
-                    {
-                        _documentNavigationManagementService.Move(content.Key, newParentKey);
-                    }
+                    _documentNavigationManagementService.UpdateSortOrder(content.Key, content.SortOrder);
                 }
             }
         }
@@ -237,6 +246,32 @@ private void HandleNavigationForSingleContent(IContent content)
 
     private bool ExistsInNavigationBin(Guid contentKey) => _documentNavigationQueryService.TryGetParentKeyInBin(contentKey, out _);
 
+    private async Task HandlePublishedAsync(JsonPayload payload, CancellationToken cancellationToken)
+    {
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
+        {
+            await _publishStatusManagementService.InitializeAsync(cancellationToken);
+        }
+
+        if (payload.Key.HasValue is false)
+        {
+            return;
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove))
+        {
+            await _publishStatusManagementService.RemoveAsync(payload.Key.Value, cancellationToken);
+        }
+        else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshNode))
+        {
+            await _publishStatusManagementService.AddOrUpdateStatusAsync(payload.Key.Value, cancellationToken);
+        }
+        else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
+        {
+            await _publishStatusManagementService.AddOrUpdateStatusWithDescendantsAsync(payload.Key.Value, cancellationToken);
+        }
+    }
     private void HandleRouting(JsonPayload payload)
     {
         if(payload.ChangeTypes.HasType(TreeChangeTypes.Remove))