Fixes tests, adds notes

src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs

@@ -10,7 +10,6 @@ namespace Examine.Lucene.Directories
     public class FileSystemDirectoryFactory : DirectoryFactoryBase
     {
         private readonly DirectoryInfo _baseDir;
-        private readonly IOptionsMonitor<LuceneDirectoryIndexOptions> _indexOptions;
 
         public FileSystemDirectoryFactory(
             DirectoryInfo baseDir,
@@ -19,11 +18,13 @@ public FileSystemDirectoryFactory(
         {
             _baseDir = baseDir;
             LockFactory = lockFactory;
-            _indexOptions = indexOptions;
+            IndexOptions = indexOptions;
         }
 
         public ILockFactory LockFactory { get; }
 
+        protected IOptionsMonitor<LuceneDirectoryIndexOptions> IndexOptions { get; }
+
         protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock)
         {
             var path = Path.Combine(_baseDir.FullName, luceneIndex.Name);
@@ -35,10 +36,10 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force
                 IndexWriter.Unlock(dir);
             }
 
-            var options = _indexOptions.GetNamedOptions(luceneIndex.Name);
+            var options = IndexOptions.GetNamedOptions(luceneIndex.Name);
             if (options.NrtEnabled)
             {
-                return new NRTCachingDirectory(dir, 5.0, 60.0);
+                return new NRTCachingDirectory(dir, options.NrtCacheMaxMergeSizeMB, options.NrtCacheMaxCachedMB);
             }
             else
             {

src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs

@@ -132,8 +132,15 @@ internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlo
             // Start replicating back to main
             _replicator.StartIndexReplicationOnSchedule(1000);
 
-            // TODO: Put this behind IOptions for NRT stuff, but I think this is going to be better
-            directory = new NRTCachingDirectory(localLuceneDir, 5.0, 60.0);
+            var options = IndexOptions.GetNamedOptions(luceneIndex.Name);
+            if (options.NrtEnabled)
+            {
+                directory = new NRTCachingDirectory(localLuceneDir, options.NrtCacheMaxMergeSizeMB, options.NrtCacheMaxCachedMB);
+            }
+            else
+            {
+                directory = localLuceneDir;
+            }
 
             return mainResult;
         }

src/Examine.Lucene/LuceneIndexOptions.cs

@@ -13,6 +13,10 @@ public class LuceneIndexOptions : IndexOptions
 
         public double NrtTargetMinStaleSec { get; set; } = 1.0;
 
+        public double NrtCacheMaxMergeSizeMB { get; set; } = 5.0;
+
+        public double NrtCacheMaxCachedMB { get; set; } = 60.0;
+
         public IndexDeletionPolicy IndexDeletionPolicy { get; set; }
 
         public Analyzer Analyzer { get; set; }

src/Examine.Lucene/Providers/LuceneIndex.cs

@@ -6,21 +6,18 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Examine.Lucene.Directories;
+using Examine.Lucene.Indexing;
 using Lucene.Net.Analysis;
 using Lucene.Net.Analysis.Miscellaneous;
+using Lucene.Net.Analysis.Standard;
 using Lucene.Net.Documents;
 using Lucene.Net.Index;
 using Lucene.Net.Search;
 using Microsoft.Extensions.Logging;
-using Directory = Lucene.Net.Store.Directory;
-using static Lucene.Net.Index.IndexWriter;
 using Microsoft.Extensions.Options;
-using Lucene.Net.Analysis.Standard;
-using Examine.Lucene.Indexing;
-using Examine.Lucene.Directories;
-using static Lucene.Net.Queries.Function.ValueSources.MultiFunction;
-using Lucene.Net.Search.Join;
-using Lucene.Net.Index.Extensions;
+using static Lucene.Net.Index.IndexWriter;
+using Directory = Lucene.Net.Store.Directory;
 
 namespace Examine.Lucene.Providers
 {
@@ -63,14 +60,14 @@ public LuceneIndex(
             IOptionsMonitor<LuceneDirectoryIndexOptions> indexOptions)
            : this(loggerFactory, name, (IOptionsMonitor<LuceneIndexOptions>)indexOptions)
         {
-            LuceneDirectoryIndexOptions directoryOptions = indexOptions.GetNamedOptions(name);
-            
+            var directoryOptions = indexOptions.GetNamedOptions(name);
+
             if (directoryOptions.DirectoryFactory == null)
             {
                 throw new InvalidOperationException($"No {typeof(IDirectoryFactory)} assigned");
             }
 
-            _directory = new Lazy<Directory>(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex));            
+            _directory = new Lazy<Directory>(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex));
         }
 
         //TODO: The problem with this is that the writer would already need to be configured with a PerFieldAnalyzerWrapper
@@ -342,15 +339,12 @@ public void EnsureIndex(bool forceOverwrite)
                                 _logger.LogDebug("Clearing existing index {IndexName}", Name);
                             }
 
-                            if (_writer == null)
-                            {
-                                //This will happen if the writer hasn't been created/initialized yet which
-                                // might occur if a rebuild is triggered before any indexing has been triggered.
-                                //In this case we need to initialize a writer and continue as normal.
-                                //Since we are already inside the writer lock and it is null, we are allowed to 
-                                // make this call with out using GetIndexWriter() to do the initialization.
-                                _writer = CreateIndexWriterInternal();
-                            }
+                            //This will happen if the writer hasn't been created/initialized yet which
+                            // might occur if a rebuild is triggered before any indexing has been triggered.
+                            //In this case we need to initialize a writer and continue as normal.
+                            //Since we are already inside the writer lock and it is null, we are allowed to 
+                            // make this call with out using GetIndexWriter() to do the initialization.
+                            _writer ??= CreateIndexWriterInternal();
 
                             //We're forcing an overwrite, 
                             // this means that we need to cancel all operations currently in place,
@@ -505,7 +499,9 @@ private int PerformDeleteFromIndexInternal(IEnumerable<string> itemIds, Cancella
                 foreach (var id in itemIds)
                 {
                     if (cancellationToken.IsCancellationRequested)
+                    {
                         break;
+                    }
 
                     var op = new IndexOperation(new ValueSet(id), IndexOperationType.Delete);
                     if (ProcessQueueItem(op))
@@ -635,7 +631,9 @@ private bool IndexExistsImpl()
         {
             //if it's been set and it's true, return true
             if (_exists.HasValue && _exists.Value)
+            {
                 return true;
+            }
 
             //if it's not been set or it just doesn't exist, re-read the lucene files
             if (!_exists.HasValue || !_exists.Value)
@@ -686,7 +684,7 @@ private bool DeleteFromIndex(Term indexTerm, bool performCommit = true)
                 return false;
             }
         }
-        
+
         /// <summary>
         /// Collects the data for the fields and adds the document which is then committed into Lucene.Net's index
         /// </summary>
@@ -705,23 +703,23 @@ protected virtual void AddDocument(Document doc, ValueSet valueSet)
             }
 
             //add node id
-            IIndexFieldValueType nodeIdValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.ItemIdFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.Raw));
+            var nodeIdValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.ItemIdFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.Raw));
             nodeIdValueType.AddValue(doc, valueSet.Id);
 
             //add the category
-            IIndexFieldValueType categoryValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.CategoryFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase));
+            var categoryValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.CategoryFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase));
             categoryValueType.AddValue(doc, valueSet.Category);
 
             //add the item type
-            IIndexFieldValueType indexTypeValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.ItemTypeFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase));
+            var indexTypeValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.ItemTypeFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase));
             indexTypeValueType.AddValue(doc, valueSet.ItemType);
 
-            foreach (KeyValuePair<string, IReadOnlyList<object>> field in valueSet.Values)
+            foreach (var field in valueSet.Values)
             {
                 //check if we have a defined one
-                if (FieldDefinitions.TryGetValue(field.Key, out FieldDefinition definedFieldDefinition))
+                if (FieldDefinitions.TryGetValue(field.Key, out var definedFieldDefinition))
                 {
-                    IIndexFieldValueType valueType = FieldValueTypeCollection.GetValueType(
+                    var valueType = FieldValueTypeCollection.GetValueType(
                         definedFieldDefinition.Name,
                         FieldValueTypeCollection.ValueTypeFactories.TryGetFactory(definedFieldDefinition.Type, out var valTypeFactory)
                             ? valTypeFactory
@@ -736,7 +734,7 @@ protected virtual void AddDocument(Document doc, ValueSet valueSet)
                 {
                     //Check for the special field prefix, if this is the case it's indexed as an invariant culture value
 
-                    IIndexFieldValueType valueType = FieldValueTypeCollection.GetValueType(field.Key, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase));
+                    var valueType = FieldValueTypeCollection.GetValueType(field.Key, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase));
                     foreach (var o in field.Value)
                     {
                         valueType.AddValue(doc, o);
@@ -746,7 +744,7 @@ protected virtual void AddDocument(Document doc, ValueSet valueSet)
                 {
                     // wasn't specifically defined, use FullText as the default
 
-                    IIndexFieldValueType valueType = FieldValueTypeCollection.GetValueType(
+                    var valueType = FieldValueTypeCollection.GetValueType(
                         field.Key,
                         FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.FullText));
 
@@ -914,7 +912,7 @@ private bool ProcessQueueItem(IndexOperation item)
         /// <returns></returns>
         private TrackingIndexWriter CreateIndexWriterInternal()
         {
-            Directory dir = GetLuceneDirectory();
+            var dir = GetLuceneDirectory();
 
             // Unfortunatley if the appdomain is taken down this will remain locked, so we can 
             // ensure that it's unlocked here in that case.
@@ -936,7 +934,7 @@ private TrackingIndexWriter CreateIndexWriterInternal()
                 return null;
             }
 
-            IndexWriter writer = CreateIndexWriter(dir);
+            var writer = CreateIndexWriter(dir);
 
             var trackingIndexWriter = new TrackingIndexWriter(writer);
 
@@ -1014,10 +1012,7 @@ public TrackingIndexWriter IndexWriter
                     Monitor.Enter(_writerLocker);
                     try
                     {
-                        if (_writer == null)
-                        {
-                            _writer = CreateIndexWriterInternal();
-                        }
+                        _writer ??= CreateIndexWriterInternal();
                     }
                     finally
                     {
@@ -1045,16 +1040,23 @@ private LuceneSearcher CreateSearcher()
                     continue;
                 }
 
-                name = name.Substring(0, name.LastIndexOf(suffix, StringComparison.Ordinal));
+                name = name[..name.LastIndexOf(suffix, StringComparison.Ordinal)];
             }
 
-            TrackingIndexWriter writer = IndexWriter;
+            var writer = IndexWriter;
 
             // Create an IndexSearcher ReferenceManager to safely share IndexSearcher instances across
             // multiple threads
             var searcherManager = new SearcherManager(
                 writer.IndexWriter,
-                false, // TODO: Apply All Deletes? Will be faster if this is false, https://blog.mikemccandless.com/2011/11/near-real-time-readers-with-lucenes.html
+
+                // TODO: Apply All Deletes? Will be faster if this is false, https://blog.mikemccandless.com/2011/11/near-real-time-readers-with-lucenes.html
+                // BUT ... to do that we would need to fulfill this requirement:
+                // "yet during searching you have some way to ignore the old versions"
+                // Without fulfilling that requirement our Index_Read_And_Write_Ensure_No_Errors_In_Async tests fail when using
+                // non in-memory directories because it will return more results than what is actually in the index.
+                false,
+
                 new SearcherFactory());
 
             searcherManager.AddListener(this);
@@ -1151,7 +1153,7 @@ private void QueueTask(Func<int> op, Action<IndexOperationEventArgs> onComplete,
 
                         // The task is initialized to completed so just continue with
                         // and return the new task so that any new appended tasks are the current
-                        Task t = _asyncTask.ContinueWith(
+                        var t = _asyncTask.ContinueWith(
                             x =>
                             {
                                 var indexedCount = 0;
@@ -1260,9 +1262,9 @@ public ForceThreadingModeIndexProcessor(LuceneIndex index, IndexThreadingMode mo
         public IEnumerable<string> GetFieldNames()
         {
             var writer = IndexWriter;
-            using (DirectoryReader reader = writer.IndexWriter.GetReader(false))
+            using (var reader = writer.IndexWriter.GetReader(false))
             {
-                IEnumerable<string> fieldInfos = MultiFields.GetMergedFieldInfos(reader).Select(x => x.Name);
+                var fieldInfos = MultiFields.GetMergedFieldInfos(reader).Select(x => x.Name);
                 return fieldInfos;
             }
         }

src/Examine.Lucene/PublicAPI.Unshipped.txt

@@ -1,6 +1,12 @@
 Examine.Lucene.Directories.FileSystemDirectoryFactory.FileSystemDirectoryFactory(System.IO.DirectoryInfo baseDir, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Options.IOptionsMonitor<Examine.Lucene.LuceneDirectoryIndexOptions> indexOptions) -> void
-Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo localDir, System.IO.DirectoryInfo mainDir, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, bool tryFixMainIndexIfCorrupt) -> void
+Examine.Lucene.Directories.FileSystemDirectoryFactory.IndexOptions.get -> Microsoft.Extensions.Options.IOptionsMonitor<Examine.Lucene.LuceneDirectoryIndexOptions>
+Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo localDir, System.IO.DirectoryInfo mainDir, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, Microsoft.Extensions.Options.IOptionsMonitor<Examine.Lucene.LuceneDirectoryIndexOptions> indexOptions) -> void
+Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo localDir, System.IO.DirectoryInfo mainDir, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, Microsoft.Extensions.Options.IOptionsMonitor<Examine.Lucene.LuceneDirectoryIndexOptions> indexOptions, bool tryFixMainIndexIfCorrupt) -> void
 Examine.Lucene.Directories.TempEnvFileSystemDirectoryFactory.TempEnvFileSystemDirectoryFactory(Examine.Lucene.Directories.IApplicationIdentifier applicationIdentifier, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Options.IOptionsMonitor<Examine.Lucene.LuceneDirectoryIndexOptions> indexOptions) -> void
+Examine.Lucene.LuceneIndexOptions.NrtCacheMaxCachedMB.get -> double
+Examine.Lucene.LuceneIndexOptions.NrtCacheMaxCachedMB.set -> void
+Examine.Lucene.LuceneIndexOptions.NrtCacheMaxMergeSizeMB.get -> double
+Examine.Lucene.LuceneIndexOptions.NrtCacheMaxMergeSizeMB.set -> void
 Examine.Lucene.LuceneIndexOptions.NrtEnabled.get -> bool
 Examine.Lucene.LuceneIndexOptions.NrtEnabled.set -> void
 Examine.Lucene.LuceneIndexOptions.NrtTargetMaxStaleSec.get -> double

src/Examine.Lucene/Search/LuceneSearchExecutor.cs

@@ -21,7 +21,6 @@ public class LuceneSearchExecutor
         private readonly ISearchContext _searchContext;
         private readonly Query _luceneQuery;
         private readonly ISet<string> _fieldsToLoad;
-        private int? _maxDoc;
 
         internal LuceneSearchExecutor(QueryOptions options, Query query, IEnumerable<SortField> sortField, ISearchContext searchContext, ISet<string> fieldsToLoad)
         {