Merge branch 'Release/faster-mapping-of-opti-content' into Release/El…

…astic8OptimizedMapping

src/Epinova.ElasticSearch.Core.EPiServer.Commerce/Controllers/ElasticAdminCommerceController.cs

@@ -7,6 +7,7 @@
 using Epinova.ElasticSearch.Core.EPiServer.Controllers;
 using Epinova.ElasticSearch.Core.Models;
 using Epinova.ElasticSearch.Core.Settings;
+using EPiServer.Commerce.Catalog.ContentTypes;
 using EPiServer.Core;
 using EPiServer.DataAbstraction;
 using EPiServer.Scheduler;
@@ -19,7 +20,7 @@ public class ElasticAdminCommerceController : ElasticAdminController
         private readonly IElasticSearchSettings _settings;
         private readonly ReferenceConverter _referenceConverter;
 
-        public ElasticAdminCommerceController(IContentIndexService contentIndexService, ILanguageBranchRepository languageBranchRepository, ICoreIndexer coreIndexer, IElasticSearchSettings settings, IHttpClientHelper httpClientHelper, IServerInfoService serverInfoService, IScheduledJobRepository scheduledJobRepository, IScheduledJobExecutor scheduledJobExecutor, ReferenceConverter referenceConverter)  : base(contentIndexService, languageBranchRepository, coreIndexer, settings, httpClientHelper, serverInfoService, scheduledJobRepository, scheduledJobExecutor)
+        public ElasticAdminCommerceController(IContentIndexService contentIndexService, IContentTypeRepository contentTypeRepository, ILanguageBranchRepository languageBranchRepository, ICoreIndexer coreIndexer, IElasticSearchSettings settings, IHttpClientHelper httpClientHelper, IServerInfoService serverInfoService, IScheduledJobRepository scheduledJobRepository, IScheduledJobExecutor scheduledJobExecutor, ReferenceConverter referenceConverter)  : base(contentIndexService, contentTypeRepository, languageBranchRepository, coreIndexer, settings, httpClientHelper, serverInfoService, scheduledJobRepository, scheduledJobExecutor)
         {
             _settings = settings;
             _referenceConverter = referenceConverter;
@@ -52,33 +53,26 @@ public override ActionResult AddNewIndexWithMappings()
             {
                 var commerceIndexName = _settings.GetCommerceIndexName(new CultureInfo(lang.Key));
                 CreateIndex(indexType, commerceIndexName);
-                
-                ContentReference commerceRoot = _referenceConverter.GetRootLink();
-                UpdateMappingForTypes(commerceRoot, indexType, commerceIndexName, lang.Key);
+
+                List<Type> commerceTypes = ListCommerceContentTypes();
+                UpdateMappingForTypes(indexType, commerceIndexName, lang.Key, commerceTypes);
             }
 
             return RedirectToAction("Index");
-        }
 
-        public override ActionResult RunIndexJob()
-        {
-            return base.RunIndexJob();
-        }
-
-        public override ActionResult DeleteIndex(string indexName)
-        {
-            return base.DeleteIndex(indexName);
+            List<Type> ListCommerceContentTypes()
+            {
+                List<Type> types = ListOptimizelyTypes();
+                types.RemoveAll(t => !t.IsSubclassOf(typeof(CatalogContentBase)));
+                return types;
+            }
         }
 
+
         [HttpPost]
         public override ActionResult DeleteAll()
         {
             return base.DeleteAll();
         }
-
-        public override ActionResult ChangeTokenizer(string indexName, string tokenizer)
-        {
-            return base.ChangeTokenizer(indexName, tokenizer);
-        }
     }
 }

src/Epinova.ElasticSearch.Core.EPiServer/Contracts/IContentIndexService.cs

@@ -1,14 +1,11 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using EPiServer.Core;
 using EPiServer.DataAbstraction;
 
 namespace Epinova.ElasticSearch.Core.EPiServer.Contracts
 {
     public interface IContentIndexService
     {
-        Type[] ListContainedTypes(List<IContent> contentList);
-        List<IContent> ListContentFromRoot(int bulkSize, ContentReference rootLink, List<LanguageBranch> languages);
         IEnumerable<IContent> ListContent(List<ContentReference> contentReferences, List<LanguageBranch> languages);
     }
 }

src/Epinova.ElasticSearch.Core.EPiServer/Controllers/ElasticAdminController.cs

@@ -2,14 +2,13 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Web.Mvc;
-using System.Web.Routing;
 using Epinova.ElasticSearch.Core.Admin;
 using Epinova.ElasticSearch.Core.Contracts;
 using Epinova.ElasticSearch.Core.EPiServer.Contracts;
 using Epinova.ElasticSearch.Core.EPiServer.Controllers.Abstractions;
 using Epinova.ElasticSearch.Core.EPiServer.Models.ViewModels;
+using Epinova.ElasticSearch.Core.Extensions;
 using Epinova.ElasticSearch.Core.Models;
 using Epinova.ElasticSearch.Core.Models.Admin;
 using Epinova.ElasticSearch.Core.Settings;
@@ -24,6 +23,7 @@ namespace Epinova.ElasticSearch.Core.EPiServer.Controllers
     public class ElasticAdminController : ElasticSearchControllerBase
     {
         private readonly IContentIndexService _contentIndexService;
+        private readonly IContentTypeRepository _contentTypeRepository;
         private readonly ICoreIndexer _coreIndexer;
         private readonly IElasticSearchSettings _settings;
         private readonly Health _healthHelper;
@@ -34,6 +34,7 @@ public class ElasticAdminController : ElasticSearchControllerBase
 
         public ElasticAdminController(
             IContentIndexService contentIndexService,
+            IContentTypeRepository contentTypeRepository,
             ILanguageBranchRepository languageBranchRepository,
             ICoreIndexer coreIndexer,
             IElasticSearchSettings settings,
@@ -44,6 +45,7 @@ public ElasticAdminController(
             : base(serverInfoService, settings, httpClientHelper, languageBranchRepository)
         {
             _contentIndexService = contentIndexService;
+            _contentTypeRepository = contentTypeRepository;
             _coreIndexer = coreIndexer;
             _settings = settings;
             _healthHelper = new Health(settings, httpClientHelper);
@@ -130,7 +132,10 @@ private void CreateIndexAndMappings(string indexName, Type indexType, string lan
             if(IsCustomType(indexType))
                 _coreIndexer.UpdateMapping(indexType, indexType, indexName, lang, optIn: false);
             else
-                UpdateMappingForTypes(ContentReference.RootPage, indexType, indexName, lang);
+            {
+                List<Type> allTypes = ListAllTypes(lang);
+                UpdateMappingForTypes(indexType, indexName, lang, allTypes);
+            }
 
             index.WaitForStatus();
         }
@@ -192,15 +197,28 @@ protected Index CreateIndex(Type indexType, string indexName)
             return index;
         }
 
+        private List<Type> ListCmsContentTypes()
+        {
+            List<Type> types = ListOptimizelyTypes();
+            types.RemoveAll(t => t.Namespace == null || t.Namespace.StartsWith("EPiServer.Commerce.", StringComparison.OrdinalIgnoreCase));
+            types.RemoveAll(t => !(t.IsSubclassOf(typeof(BlockData)) || t.IsSubclassOf(typeof(PageData)) || t.IsSubclassOf(typeof(BasicContent))));
+            return types;
+        }
 
-        protected void UpdateMappingForTypes(ContentReference rootLink, Type indexType, string indexName, string languageKey)
+        private List<Type> ListOptimizelyMediaTypes()
         {
-            List<IContent> allContents = languageKey.Equals(Constants.InvariantCultureIndexNamePostfix) ?
-                _contentIndexService.ListContentFromRoot(_settings.BulkSize, rootLink, new List<LanguageBranch>())
-                : _contentIndexService.ListContentFromRoot(_settings.BulkSize, rootLink, new List<LanguageBranch> { new LanguageBranch(languageKey) });
-            Type[] types = _contentIndexService.ListContainedTypes(allContents);
+            List<Type> types = ListOptimizelyTypes();
+            types.RemoveAll(t => !t.IsSubclassOf(typeof(MediaData)));
+            return types;
+        }
+
+        protected List<Type> ListOptimizelyTypes() => _contentTypeRepository.List().Select(ct => ct.ModelType).Where(t => t != null && !t.IsExcludedType()).ToList();
 
-            foreach(Type type in types)
+        protected List<Type> ListAllTypes(string languageKey) => languageKey.Equals(Constants.InvariantCultureIndexNamePostfix) ? ListOptimizelyMediaTypes() : ListCmsContentTypes();
+
+        protected void UpdateMappingForTypes(Type indexType, string indexName, string languageKey, List<Type> allTypes)
+        {
+            foreach(Type type in allTypes)
             {
                 _coreIndexer.UpdateMapping(type, indexType, indexName, languageKey, optIn: false);
             }

src/Epinova.ElasticSearch.Core.EPiServer/Plugin/IndexEPiServerContent.cs

@@ -104,7 +104,7 @@ public override string Execute()
                     if(IsStopped)
                         return "Aborted by user";
 
-                    List<IContent> contents = GetDescendentContents(contentReferences.Take(_settings.BulkSize).ToList(), languages);
+                    List<IContent> contents = _contentIndexService.ListContent(contentReferences.Take(_settings.BulkSize).ToList(), languages.ToList()).ToList();
 
                     contents.RemoveAll(_indexer.SkipIndexing);
                     contents.RemoveAll(_indexer.IsExcludedType);
@@ -194,11 +194,6 @@ protected virtual List<ContentReference> GetContentReferences()
             return _contentLoader.GetDescendents(ContentReference.RootPage).ToList();
         }
 
-        protected virtual List<IContent> GetDescendentContents(List<ContentReference> contentReferences, IEnumerable<LanguageBranch> languages)
-        {
-            return _contentIndexService.ListContent(contentReferences, languages.ToList()).ToList();
-        }
-
         private bool IndicesExists(IEnumerable<LanguageBranch> languages)
         {
             foreach(var language in languages.Select(l => l.Culture))

src/Epinova.ElasticSearch.Core.EPiServer/Services/ContentIndexService.cs

@@ -14,51 +14,10 @@ namespace Epinova.ElasticSearch.Core.EPiServer.Services
     public class ContentIndexService : IContentIndexService
     {
         private readonly IContentLoader _contentLoader;
-        private readonly IIndexer _indexer;
-
-        public ContentIndexService(IContentLoader contentLoader, IIndexer indexer)
+
+        public ContentIndexService(IContentLoader contentLoader)
         {
             _contentLoader = contentLoader;
-            _indexer = indexer;
-        }
-
-        public Type[] ListContainedTypes(List<IContent> contentList)
-        {
-            Type[] uniqueTypes = contentList.Select(content =>
-                {
-                    var type = content.GetType();
-                    return type.Name.EndsWith("Proxy") ? type.BaseType : type;
-                })
-                .Distinct()
-                .ToArray();
-
-            return uniqueTypes;
-        }
-
-        public List<IContent> ListContentFromRoot(int bulkSize, ContentReference rootLink, List<LanguageBranch> languages)
-        {
-            List<ContentReference> contentReferences = _contentLoader.GetDescendents(rootLink).ToList();
-
-            List<IContent> contentList = new List<IContent>();
-
-            while(contentReferences.Count > 0)
-            {
-                List<IContent> bulkContents = ListContent(contentReferences.Take(bulkSize).ToList(), languages).ToList();
-
-                bulkContents.RemoveAll(_indexer.SkipIndexing);
-                bulkContents.RemoveAll(_indexer.IsExcludedType);
-                List<IContent> contents = bulkContents.Where(b => _indexer.IsExcludedType(b)).ToList();
-                if(languages.Any())
-                    bulkContents.RemoveAll(c => c is MediaData);
-                else
-                    bulkContents.RemoveAll(c => !(c is MediaData));
-
-                contentList.AddRange(bulkContents);
-                var removeCount = contentReferences.Count >= bulkSize ? bulkSize : contentReferences.Count;
-                contentReferences.RemoveRange(0, removeCount);
-            }
-
-            return contentList;
         }
 
         public IEnumerable<IContent> ListContent(List<ContentReference> contentReferences, List<LanguageBranch> languages)

tests/Core.Episerver.Tests/Controllers/ElasticAdminControllerTests.cs

@@ -30,8 +30,11 @@ public ElasticAdminControllerTests(ServiceLocatorFixture fixture)
                     new LanguageBranch(new CultureInfo("no"))
                 });
 
+            var contentTypeRepositoryMock = new Mock<IContentTypeRepository>();
+
             _controller = new ElasticAdminController(
                 fixture.ServiceLocationMock.ContentIndexServiceMock.Object,
+                contentTypeRepositoryMock.Object,
                 languageBranchRepositoryMock.Object,
                 indexerMock.Object,
                 fixture.ServiceLocationMock.SettingsMock.Object,