Umbraco Examine implementation

[bielu]

As author of https://github.com/Novicell/Novicell.Examine.ElasticSearch, I noticed few problem when working with Umbraco, quite often Umbraco modifies logic in one of following files:
https://github.com/umbraco/Umbraco-CMS/blob/v11/contrib/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs
https://github.com/umbraco/Umbraco-CMS/blob/v11/contrib/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs
https://github.com/umbraco/Umbraco-CMS/blob/v11/contrib/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs
which enforce me to check compatibility everytime when Umbraco will release new version, it is one of reasons, why I didnt release version for v9/10/11.
I am looking into implementing new version for 11/12/13, but this issue makes me worried about effort to maintain the potential package.

One of solution which I would Like to see is change for example in https://github.com/umbraco/Umbraco-CMS/blob/v11/contrib/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs
to be similar to:

public abstract class UmbracoExamineIndex :  IUmbracoIndex, IIndexDiagnostics
{
    private readonly UmbracoExamineIndexDiagnostics _diagnostics;
    private readonly ILogger<UmbracoExamineIndex> _logger;
    private readonly IUmbracoIndexProvider _umbracoIndexProvider;
    private readonly IRuntimeState _runtimeState;
    private bool _hasLoggedInitLog;

    protected UmbracoExamineIndex(
        ILoggerFactory loggerFactory,
        string name,
        IUmbracoIndexProvider umbracoIndexProvider,
        IOptionsMonitor<LuceneDirectoryIndexOptions> indexOptions,
        IHostingEnvironment hostingEnvironment,
        IRuntimeState runtimeState)
    {
        _umbracoIndexProvider = umbracoIndexProvider;
        _runtimeState = runtimeState;
        _diagnostics = new UmbracoExamineIndexDiagnostics(this, loggerFactory.CreateLogger<UmbracoExamineIndexDiagnostics>(), hostingEnvironment, indexOptions);
        _logger = loggerFactory.CreateLogger<UmbracoExamineIndex>();
    }

    public Attempt<string> IsHealthy() => _diagnostics.IsHealthy();
    public virtual IReadOnlyDictionary<string, object> Metadata => _diagnostics.Metadata;

    /// <summary>
    ///     When set to true Umbraco will keep the index in sync with Umbraco data automatically
    /// </summary>
    public bool EnableDefaultEventHandler { get; set; } = true;

    public bool PublishedValuesOnly { get; protected set; } = false;

    /// <summary>
    ///     override to check if we can actually initialize.
    /// </summary>
    /// <remarks>
    ///     This check is required since the base examine lib will try to rebuild on startup
    /// </remarks>
    protected void PerformDeleteFromIndex(IEnumerable<string> itemIds, Action<IndexOperationEventArgs>? onComplete)
    {
        if (CanInitialize())
        {
            _umbracoIndexProvider.PerformDeleteFromIndex(itemIds, onComplete);
        }
    }

    protected  void PerformIndexItems(IEnumerable<ValueSet> values, Action<IndexOperationEventArgs> onComplete)
    {
        if (CanInitialize())
        {
            _umbracoIndexProvider.PerformIndexItems(values, onComplete);
        }
    }

    /// <summary>
    ///     Returns true if the Umbraco application is in a state that we can initialize the examine indexes
    /// </summary>
    /// <returns></returns>
    protected bool CanInitialize()
    {
        var canInit = _runtimeState.Level == RuntimeLevel.Run;

        if (!canInit && !_hasLoggedInitLog)
        {
            _hasLoggedInitLog = true;
            _logger.LogWarning("Runtime state is not " + RuntimeLevel.Run + ", no indexing will occur");
        }

        return canInit;
    }

    /// <summary>
    ///     This ensures that the special __Raw_ fields are indexed correctly
    /// </summary>
    /// <param name="docArgs"></param>
    protected void OnDocumentWriting(DocumentWritingEventArgs docArgs)
    {
        Document? d = docArgs.Document;

        foreach (KeyValuePair<string, IReadOnlyList<object>> f in docArgs.ValueSet.Values
                     .Where(x => x.Key.StartsWith(UmbracoExamineFieldNames.RawFieldPrefix)).ToList())
        {
            if (f.Value.Count > 0)
            {
                //remove the original value so we can store it the correct way
                d.RemoveField(f.Key);

                d.Add(new StoredField(f.Key, f.Value[0].ToString()));
            }
        }

        _umbracoIndexProvider.OnDocumentWriting(docArgs);
    }

    protected void OnTransformingIndexValues(IndexingItemEventArgs e)
    {
        _umbracoIndexProvider.OnTransformingIndexValues(e);

        var updatedValues = e.ValueSet.Values.ToDictionary(x => x.Key, x => (IEnumerable<object>)x.Value);

        //ensure special __Path field
        var path = e.ValueSet.GetValue("path");
        if (path != null)
        {
            updatedValues[UmbracoExamineFieldNames.IndexPathFieldName] = path.Yield();
        }

        //icon
        if (e.ValueSet.Values.TryGetValue("icon", out IReadOnlyList<object>? icon) &&
            e.ValueSet.Values.ContainsKey(UmbracoExamineFieldNames.IconFieldName) == false)
        {
            updatedValues[UmbracoExamineFieldNames.IconFieldName] = icon;
        }

        e.SetValues(updatedValues);
    }

    public void IndexItems(IEnumerable<ValueSet> values)
    {
        _umbracoIndexProvider.PerformIndexItems(values);
    }

    public void DeleteFromIndex(IEnumerable<string> itemIds)
    {
        _umbracoIndexProvider.PerformDeleteFromIndex(itemIds);
    }

    public void CreateIndex()
    {

        _umbracoIndexProvider.CreateIndex();
    }

    public bool IndexExists()
    {
        return  _umbracoIndexProvider.IndexExists();
    }

    public string Name { get; }
    public ISearcher Searcher { get; }
    public ReadOnlyFieldDefinitionCollection FieldDefinitions { get; }
    public event EventHandler<IndexOperationEventArgs> IndexOperationComplete;
    public event EventHandler<IndexingItemEventArgs> TransformingIndexValues;
    public event EventHandler<IndexingErrorEventArgs> IndexingError;
    public long GetDocumentCount()
    {
        return  _umbracoIndexProvider.GetDocumentCount();
    }

    public IEnumerable<string> GetFieldNames()
    {
        return  _umbracoIndexProvider.GetFieldNames();
    }
}

example interface for IUmbracoIndexProvider

public interface IUmbracoIndexProvider : IIndex
{
    void OnTransformingIndexValues(IndexingItemEventArgs indexingItemEventArgs);
    void OnDocumentWriting(DocumentWritingEventArgs docArgs);
    void PerformDeleteFromIndex(IEnumerable<string> itemIds, Action<IndexOperationEventArgs> onComplete = null);
    void PerformIndexItems(IEnumerable<ValueSet> values, Action<IndexOperationEventArgs> onComplete = null);
    void CreateIndex();
    bool IndexExists();
    long GetDocumentCount();
    IEnumerable<string> GetFieldNames();
}

and example implementation:

public class ExamineDefaultProvider : LuceneIndex, IUmbracoIndexProvider
{
    public ExamineDefaultProvider(ILoggerFactory loggerFactory, string name, IOptionsMonitor<LuceneDirectoryIndexOptions> indexOptions) : base(loggerFactory, name, indexOptions)
    {
    }

    public void OnTransformingIndexValues(IndexingItemEventArgs indexingItemEventArgs)
    {
        base.OnTransformingIndexValues(indexingItemEventArgs);
    }

    public void OnDocumentWriting(DocumentWritingEventArgs docArgs)
    {
       base.OnDocumentWriting(docArgs);
    }

    public void PerformDeleteFromIndex(IEnumerable<string> itemIds, Action<IndexOperationEventArgs> onComplete = null)
    {
        base.PerformDeleteFromIndex(itemIds,onComplete);
    }

    public void PerformIndexItems(IEnumerable<ValueSet> values, Action<IndexOperationEventArgs> onComplete = null)
    {
      
        base.PerformIndexItems(values,onComplete);
    }
}

Of course example above will not work, as they don't pass correctly indexes as we rely on index name, so we would need 3 implementations for providers, but I think we can work around it easily, I just want see if Umbraco willing to introduce change in v12 or v13 to make it easier to maintain custom Examine providers.

[inetzo]

@UmbracoHQ this sounds like an excellent proposal if you ask me. We, and many of us I suspect, are anxiously waiting for a version of @bielu Elastic package that is suitable for Umbraco 11 and future versions.

[bielu]

@inetzo package would be release only for umbraco version which move that layer of abstraction, so most likely 13 or up, as I think it is too late to introduce breaking change in v12 :)

[bergmania]

To me, this sounds like an good change for v13.

[bielu]

@bergmania I can prepare it, it will be first step in abstracting out examine, but I am not going do any work related to moving out searchers from core.