Skip to content

Commit

Permalink
Merge pull request #204 from Epinova/Release/Elastic8OptimizedMapping
Browse files Browse the repository at this point in the history 
Release/elastic8 optimized mapping
  • Loading branch information
@otanum
otanum authored Oct 3, 2023
2 parents a996fb9 + cfb386c commit 0e0638b
Show file tree
Hide file tree
Showing 23 changed files with 190 additions and 293 deletions.
30 changes: 12 additions & 18 deletions ...inova.ElasticSearch.Core.EPiServer.Commerce/Controllers/ElasticAdminCommerceController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using Epinova.ElasticSearch.Core.EPiServer.Controllers;
using Epinova.ElasticSearch.Core.Models;
using Epinova.ElasticSearch.Core.Settings;
using EPiServer.Commerce.Catalog.ContentTypes;
using EPiServer.Core;
using EPiServer.DataAbstraction;
using EPiServer.Scheduler;
Expand All @@ -19,7 +20,7 @@ public class ElasticAdminCommerceController : ElasticAdminController
private readonly IElasticSearchSettings _settings;
private readonly ReferenceConverter _referenceConverter;

public ElasticAdminCommerceController(IContentIndexService contentIndexService, ILanguageBranchRepository languageBranchRepository, ICoreIndexer coreIndexer, IElasticSearchSettings settings, IHttpClientHelper httpClientHelper, IServerInfoService serverInfoService, IScheduledJobRepository scheduledJobRepository, IScheduledJobExecutor scheduledJobExecutor, ReferenceConverter referenceConverter) : base(contentIndexService, languageBranchRepository, coreIndexer, settings, httpClientHelper, serverInfoService, scheduledJobRepository, scheduledJobExecutor)
public ElasticAdminCommerceController(IContentIndexService contentIndexService, IContentTypeRepository contentTypeRepository, ILanguageBranchRepository languageBranchRepository, ICoreIndexer coreIndexer, IElasticSearchSettings settings, IHttpClientHelper httpClientHelper, IServerInfoService serverInfoService, IScheduledJobRepository scheduledJobRepository, IScheduledJobExecutor scheduledJobExecutor, ReferenceConverter referenceConverter) : base(contentIndexService, contentTypeRepository, languageBranchRepository, coreIndexer, settings, httpClientHelper, serverInfoService, scheduledJobRepository, scheduledJobExecutor)
{
_settings = settings;
_referenceConverter = referenceConverter;
Expand Down Expand Up @@ -52,33 +53,26 @@ public override ActionResult AddNewIndexWithMappings()
{
var commerceIndexName = _settings.GetCommerceIndexName(new CultureInfo(lang.Key));
CreateIndex(indexType, commerceIndexName);
ContentReference commerceRoot = _referenceConverter.GetRootLink();
UpdateMappingForTypes(commerceRoot, indexType, commerceIndexName, lang.Key);

List<Type> commerceTypes = ListCommerceContentTypes();
UpdateMappingForTypes(indexType, commerceIndexName, lang.Key, commerceTypes);
}

return RedirectToAction("Index");
}

public override ActionResult RunIndexJob()
{
return base.RunIndexJob();
}

public override ActionResult DeleteIndex(string indexName)
{
return base.DeleteIndex(indexName);
List<Type> ListCommerceContentTypes()
{
List<Type> types = ListOptimizelyTypes();
types.RemoveAll(t => !t.IsSubclassOf(typeof(CatalogContentBase)));
return types;
}
}


[HttpPost]
public override ActionResult DeleteAll()
{
return base.DeleteAll();
}

public override ActionResult ChangeTokenizer(string indexName, string tokenizer)
{
return base.ChangeTokenizer(indexName, tokenizer);
}
}
}
5 changes: 1 addition & 4 deletions src/Epinova.ElasticSearch.Core.EPiServer/Contracts/IContentIndexService.cs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using EPiServer.Core;
using EPiServer.DataAbstraction;

namespace Epinova.ElasticSearch.Core.EPiServer.Contracts
{
public interface IContentIndexService
{
Type[] ListContainedTypes(List<IContent> contentList);
List<IContent> ListContentFromRoot(int bulkSize, ContentReference rootLink, List<LanguageBranch> languages);
IEnumerable<IContent> ListContent(List<ContentReference> contentReferences, List<LanguageBranch> languages);
}
}
36 changes: 27 additions & 9 deletions src/Epinova.ElasticSearch.Core.EPiServer/Controllers/ElasticAdminController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Web.Mvc;
using System.Web.Routing;
using Epinova.ElasticSearch.Core.Admin;
using Epinova.ElasticSearch.Core.Contracts;
using Epinova.ElasticSearch.Core.EPiServer.Contracts;
using Epinova.ElasticSearch.Core.EPiServer.Controllers.Abstractions;
using Epinova.ElasticSearch.Core.EPiServer.Models.ViewModels;
using Epinova.ElasticSearch.Core.Extensions;
using Epinova.ElasticSearch.Core.Models;
using Epinova.ElasticSearch.Core.Models.Admin;
using Epinova.ElasticSearch.Core.Settings;
Expand All @@ -24,6 +23,7 @@ namespace Epinova.ElasticSearch.Core.EPiServer.Controllers
public class ElasticAdminController : ElasticSearchControllerBase
{
private readonly IContentIndexService _contentIndexService;
private readonly IContentTypeRepository _contentTypeRepository;
private readonly ICoreIndexer _coreIndexer;
private readonly IElasticSearchSettings _settings;
private readonly Health _healthHelper;
Expand All @@ -34,6 +34,7 @@ public class ElasticAdminController : ElasticSearchControllerBase

public ElasticAdminController(
IContentIndexService contentIndexService,
IContentTypeRepository contentTypeRepository,
ILanguageBranchRepository languageBranchRepository,
ICoreIndexer coreIndexer,
IElasticSearchSettings settings,
Expand All @@ -44,6 +45,7 @@ public ElasticAdminController(
: base(serverInfoService, settings, httpClientHelper, languageBranchRepository)
{
_contentIndexService = contentIndexService;
_contentTypeRepository = contentTypeRepository;
_coreIndexer = coreIndexer;
_settings = settings;
_healthHelper = new Health(settings, httpClientHelper);
Expand Down Expand Up @@ -130,7 +132,10 @@ private void CreateIndexAndMappings(string indexName, Type indexType, string lan
if(IsCustomType(indexType))
_coreIndexer.UpdateMapping(indexType, indexType, indexName, lang, optIn: false);
else
UpdateMappingForTypes(ContentReference.RootPage, indexType, indexName, lang);
{
List<Type> allTypes = ListAllTypes(lang);
UpdateMappingForTypes(indexType, indexName, lang, allTypes);
}

index.WaitForStatus();
}
Expand Down Expand Up @@ -192,15 +197,28 @@ protected Index CreateIndex(Type indexType, string indexName)
return index;
}

private List<Type> ListCmsContentTypes()
{
List<Type> types = ListOptimizelyTypes();
types.RemoveAll(t => t.Namespace == null || t.Namespace.StartsWith("EPiServer.Commerce.", StringComparison.OrdinalIgnoreCase));
types.RemoveAll(t => !(t.IsSubclassOf(typeof(BlockData)) || t.IsSubclassOf(typeof(PageData)) || t.IsSubclassOf(typeof(BasicContent))));
return types;
}

protected void UpdateMappingForTypes(ContentReference rootLink, Type indexType, string indexName, string languageKey)
private List<Type> ListOptimizelyMediaTypes()
{
List<IContent> allContents = languageKey.Equals(Constants.InvariantCultureIndexNamePostfix) ?
_contentIndexService.ListContentFromRoot(_settings.BulkSize, rootLink, new List<LanguageBranch>())
: _contentIndexService.ListContentFromRoot(_settings.BulkSize, rootLink, new List<LanguageBranch> { new LanguageBranch(languageKey) });
Type[] types = _contentIndexService.ListContainedTypes(allContents);
List<Type> types = ListOptimizelyTypes();
types.RemoveAll(t => !t.IsSubclassOf(typeof(MediaData)));
return types;
}

protected List<Type> ListOptimizelyTypes() => _contentTypeRepository.List().Select(ct => ct.ModelType).Where(t => t != null && !t.IsExcludedType()).ToList();

foreach(Type type in types)
protected List<Type> ListAllTypes(string languageKey) => languageKey.Equals(Constants.InvariantCultureIndexNamePostfix) ? ListOptimizelyMediaTypes() : ListCmsContentTypes();

protected void UpdateMappingForTypes(Type indexType, string indexName, string languageKey, List<Type> allTypes)
{
foreach(Type type in allTypes)
{
_coreIndexer.UpdateMapping(type, indexType, indexName, languageKey, optIn: false);
}
Expand Down
56 changes: 9 additions & 47 deletions src/Epinova.ElasticSearch.Core.EPiServer/Indexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Epinova.ElasticSearch.Core.EPiServer.Extensions;
using Epinova.ElasticSearch.Core.Extensions;
using Epinova.ElasticSearch.Core.Models;
using Epinova.ElasticSearch.Core.Models.Admin;
using Epinova.ElasticSearch.Core.Models.Bulk;
using Epinova.ElasticSearch.Core.Settings;
using EPiServer;
Expand Down Expand Up @@ -38,16 +39,20 @@ internal void SetContentPathGetter(Func<ContentReference, int[]> func)
=> _getContentPath = func;

private Func<ContentReference, int[]> _getContentPath = ContentExtensions.GetContentPath;
private readonly ServerInfo _serverInfo;


public Indexer(
ICoreIndexer coreIndexer,
IElasticSearchSettings elasticSearchSettings,
IServerInfoService serverInfoService,
ISiteDefinitionRepository siteDefinitionRepository,
IContentLoader contentLoader,
ContentAssetHelper contentAssetHelper)
{
_coreIndexer = coreIndexer;
_elasticSearchSettings = elasticSearchSettings;
_serverInfo = serverInfoService.GetInfo();
_siteDefinitionRepository = siteDefinitionRepository;
_contentLoader = contentLoader;
_contentAssetHelper = contentAssetHelper;
Expand Down Expand Up @@ -76,8 +81,9 @@ public BulkBatchResult BulkUpdate(IEnumerable<IContent> contents, Action<string>

dynamic indexItem = content.AsIndexItem();
string id = content.ContentLink.ToReferenceWithoutVersion().ToString();
bool isSingleType = _serverInfo.Version >= Constants.SingleTypeMappingVersion;

return new BulkOperation(indexName, indexItem, Operation.Index, typeof(IndexItem), id);
return new BulkOperation(indexName, indexItem, isSingleType, Operation.Index, typeof(IndexItem), id);
}
)
.Where(b => b.Data != null)
Expand Down Expand Up @@ -107,9 +113,7 @@ public IndexingStatus UpdateStructure(IContent root, string indexName = null)
{
var language = CultureInfo.CurrentCulture;
if(root is ILocale locale && locale.Language != null && !CultureInfo.InvariantCulture.Equals(locale.Language))
{
language = locale.Language;
}

Logger.Information($"Performing recursive update, starting from {root.ContentLink}, in language {language}");

Expand All @@ -121,9 +125,7 @@ public IndexingStatus UpdateStructure(IContent root, string indexName = null)
var childStatus = Update(content, indexName);

if(childStatus == IndexingStatus.Error && status != IndexingStatus.Error)
{
status = IndexingStatus.PartialError;
}
}

return status;
Expand All @@ -140,14 +142,10 @@ public IndexingStatus Update(IContent content, string indexName = null)
}

if(IsExcludedType(content))
{
return IndexingStatus.ExcludedByConvention;
}

if(IsExcludedByRoot(content))
{
return IndexingStatus.ExcludedByConvention;
}

_coreIndexer.Update(content.ContentLink.ToReferenceWithoutVersion().ToString(), content.AsIndexItem(), indexName, typeof(IndexItem));

Expand All @@ -158,21 +156,15 @@ private bool IsExcludedByRoot(IContent content)
{
// Check options less expensive than DB-lookup first
if(content.ContentLink != null && Indexing.ExcludedRoots.Contains(content.ContentLink.ID))
{
return true;
}

if(content.ParentLink != null && Indexing.ExcludedRoots.Contains(content.ParentLink.ID))
{
return true;
}

var ancestors = _getContentPath(content.ContentLink);

if(ancestors == null)
{
return false;
}

return Indexing.ExcludedRoots.Intersect(ancestors).Any();
}
Expand All @@ -186,35 +178,23 @@ public bool IsExcludedType(IContent content)
public bool SkipIndexing(IContent content)
{
if(content is ContentFolder)
{
return true;
}

if(ContentReference.WasteBasket.CompareToIgnoreWorkID(content.ParentLink))
{
return true;
}

if(ContentReference.WasteBasket.CompareToIgnoreWorkID(content.ContentLink))
{
return true;
}

if(IsFormUpload(content))
{
return true;
}

var deleted = GetEpiserverBoolProperty(content.Property["PageDeleted"]);
if(deleted)
{
return true;
}

if(IsPageWithInvalidLinkType(content))
{
return true;
}

return false;
}
Expand All @@ -223,9 +203,7 @@ public bool ShouldHideFromSearch(IContent content)
{
//This is already called to avoid indexing
if(SkipIndexing(content))
{
return true;
}

// Common property in Epinova template
object value = content.GetType().GetProperty("HideFromSearch")?.GetValue(content);
Expand All @@ -235,57 +213,43 @@ public bool ShouldHideFromSearch(IContent content)
private static bool IsPageWithInvalidLinkType(IContent content)
{
if(!(content is PageData))
{
return false;
}

var linkType = content.GetType().GetProperty("LinkType");
if(linkType == null)
{
return true;
}

var linkTypeValue = linkType.GetValue(content);
if(linkTypeValue == null)
{
return true;
}

var shortcutType = (PageShortcutType)linkTypeValue;
if(shortcutType != PageShortcutType.Normal && shortcutType != PageShortcutType.FetchData)
{
return true;
}

return false;
}

private CultureInfo GetFallbackLanguage()
{
if(!HostingEnvironment.IsHosted)
{
return _fallbackLanguage;
}

ContentReference startPageLink = ContentReference.StartPage;
if(startPageLink == null || startPageLink == ContentReference.EmptyReference)
{
// Fallback to first defined site if StartPage is empty (no context or star-mapping)
var firstSite = _siteDefinitionRepository.List().FirstOrDefault();
if(firstSite != null)
{
if(firstSite != null)
startPageLink = firstSite.StartPage;
}
}

// Try to fetch master language from startpage
if(startPageLink != null && startPageLink != ContentReference.EmptyReference)
{
var contentLoader = ServiceLocator.Current.GetInstance<IContentLoader>();
if(contentLoader.TryGet(startPageLink, out PageData startPage))
{
return startPage.MasterLanguage;
}
}

Logger.Warning("Could not retrieve StartPage. Are you missing a wildcard mapping in CMS Admin -> Manage Websites?");
Expand Down Expand Up @@ -318,9 +282,7 @@ private bool IsFormUpload(IContent content)
}

if(owner == null)
{
return false;
}

return owner.GetType().GetInterfaces()
.Select(i => i.FullName)
Expand All @@ -335,4 +297,4 @@ private string GetIndexname(ContentReference contentLink, string indexName, Cult
return _elasticSearchSettings.GetCommerceIndexName(language);
}
}
}
}
Loading

0 comments on commit 0e0638b

Please sign in to comment.