From 00cb2a4a79b1158beed23cc26c433ceb52e00745 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 20 Aug 2024 17:18:18 -0600 Subject: [PATCH] Updates various places to reduce the need to acquire a searcher/reader which causes overhead for each search, lots of TODOs and WIPs --- .../Directories/FileSystemDirectoryFactory.cs | 3 +- .../SyncedFileSystemDirectoryFactory.cs | 3 +- src/Examine.Lucene/ExamineReplicator.cs | 1 + src/Examine.Lucene/LuceneIndexOptions.cs | 8 +- src/Examine.Lucene/Providers/LuceneIndex.cs | 65 +++++++++--- .../Providers/LuceneSearcher.cs | 14 ++- src/Examine.Lucene/PublicAPI.Unshipped.txt | 4 + src/Examine.Lucene/Search/ISearchContext.cs | 1 + .../Search/LuceneSearchExecutor.cs | 18 ++-- src/Examine.Lucene/Search/SearchContext.cs | 23 ++++- .../Search/SearcherReference.cs | 44 ++++----- .../Search/ConcurrentSearchBenchmarks.cs | 98 +++++++++++++++++-- src/Examine.Test/ExamineBaseTest.cs | 15 ++- 13 files changed, 234 insertions(+), 63 deletions(-) diff --git a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs index 72ae5ca87..c2c258be2 100644 --- a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs @@ -29,7 +29,8 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force IndexWriter.Unlock(dir); } - return dir; + // TODO: Put this behind IOptions for NRT stuff, but I think this is going to be better + return new NRTCachingDirectory(dir, 5.0, 60.0); } } } diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 025411e35..e1400f48b 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -129,7 +129,8 @@ internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlo // Start replicating back to main _replicator.StartIndexReplicationOnSchedule(1000); - directory = localLuceneDir; + // TODO: Put this behind IOptions for NRT stuff, but I think this is going to be better + directory = new NRTCachingDirectory(localLuceneDir, 5.0, 60.0); return mainResult; } diff --git a/src/Examine.Lucene/ExamineReplicator.cs b/src/Examine.Lucene/ExamineReplicator.cs index d3590a839..12a0a552e 100644 --- a/src/Examine.Lucene/ExamineReplicator.cs +++ b/src/Examine.Lucene/ExamineReplicator.cs @@ -149,6 +149,7 @@ protected virtual void Dispose(bool disposing) { _sourceIndex.IndexCommitted -= SourceIndex_IndexCommitted; _localReplicationClient.Dispose(); + _destinationDirectory.Dispose(); } _disposedValue = true; diff --git a/src/Examine.Lucene/LuceneIndexOptions.cs b/src/Examine.Lucene/LuceneIndexOptions.cs index 733a87e71..917819c26 100644 --- a/src/Examine.Lucene/LuceneIndexOptions.cs +++ b/src/Examine.Lucene/LuceneIndexOptions.cs @@ -1,8 +1,5 @@ -using System; using System.Collections.Generic; -using System.Text; using Lucene.Net.Analysis; -using Lucene.Net.Analysis.Standard; using Lucene.Net.Index; namespace Examine.Lucene @@ -10,6 +7,11 @@ namespace Examine.Lucene public class LuceneIndexOptions : IndexOptions { + public bool NrtEnabled { get; set; } = true; + + public double NrtTargetMaxStaleSec { get; set; } = 5.0; + + public double NrtTargetMinStaleSec { get; set; } = 1.0; public IndexDeletionPolicy IndexDeletionPolicy { get; set; } diff --git a/src/Examine.Lucene/Providers/LuceneIndex.cs b/src/Examine.Lucene/Providers/LuceneIndex.cs index af4005344..32c65c0b6 100644 --- a/src/Examine.Lucene/Providers/LuceneIndex.cs +++ b/src/Examine.Lucene/Providers/LuceneIndex.cs @@ -19,6 +19,8 @@ using Examine.Lucene.Indexing; using Examine.Lucene.Directories; using static Lucene.Net.Queries.Function.ValueSources.MultiFunction; +using Lucene.Net.Search.Join; +using Lucene.Net.Index.Extensions; namespace Examine.Lucene.Providers { @@ -154,8 +156,7 @@ internal LuceneIndex( /// public Analyzer DefaultAnalyzer { get; } - public PerFieldAnalyzerWrapper FieldAnalyzer => _fieldAnalyzer - ?? (_fieldAnalyzer = + public PerFieldAnalyzerWrapper FieldAnalyzer => (PerFieldAnalyzerWrapper)(_fieldAnalyzer ??= (DefaultAnalyzer is PerFieldAnalyzerWrapper pfa) ? pfa : _fieldValueTypeCollection.Value.Analyzer); @@ -416,6 +417,11 @@ private void CreateNewIndex(Directory dir) MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name, (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e))) }; + + // TODO: With NRT, we should apparently use this but there is no real implementation of it!? + // https://stackoverflow.com/questions/12271614/lucene-net-indexwriter-setmergedsegmentwarmer + //writerConfig.SetMergedSegmentWarmer(new SimpleMergedSegmentWarmer()) + writer = new IndexWriter(dir, writerConfig); } @@ -1035,21 +1041,43 @@ private LuceneSearcher CreateSearcher() { //trim the "Indexer" / "Index" suffix if it exists if (!name.EndsWith(suffix)) + { continue; + } + name = name.Substring(0, name.LastIndexOf(suffix, StringComparison.Ordinal)); } TrackingIndexWriter writer = IndexWriter; - var searcherManager = new SearcherManager(writer.IndexWriter, true, new SearcherFactory()); + + // Create an IndexSearcher ReferenceManager to safely share IndexSearcher instances across + // multiple threads + var searcherManager = new SearcherManager( + writer.IndexWriter, + false, // TODO: Apply All Deletes? Will be faster if this is false, https://blog.mikemccandless.com/2011/11/near-real-time-readers-with-lucenes.html + new SearcherFactory()); + searcherManager.AddListener(this); - _nrtReopenThread = new ControlledRealTimeReopenThread(writer, searcherManager, 5.0, 1.0) + if (_options.NrtEnabled) { - Name = $"{Name} NRT Reopen Thread", - IsBackground = true - }; + // Create the ControlledRealTimeReopenThread that reopens the index periodically having into + // account the changes made to the index and tracked by the TrackingIndexWriter instance + // The index is refreshed every XX sec when nobody is waiting + // and every XX sec whenever is someone waiting (see search method) + // (see http://lucene.apache.org/core/4_3_0/core/org/apache/lucene/search/NRTManagerReopenThread.html) + _nrtReopenThread = new ControlledRealTimeReopenThread( + writer, + searcherManager, + _options.NrtTargetMaxStaleSec, // when there is nobody waiting + _options.NrtTargetMinStaleSec) // when there is someone waiting + { + Name = $"{Name} NRT Reopen Thread", + IsBackground = true + }; - _nrtReopenThread.Start(); + _nrtReopenThread.Start(); + } // wait for most recent changes when first creating the searcher WaitForChanges(); @@ -1186,10 +1214,17 @@ public void WaitForChanges() { if (_latestGen.HasValue && !_disposedValue && !_cancellationToken.IsCancellationRequested) { - var found = _nrtReopenThread?.WaitForGeneration(_latestGen.Value, 5000); - if (_logger.IsEnabled(LogLevel.Debug)) + if (_options.NrtEnabled) { - _logger.LogDebug("{IndexName} WaitForChanges returned {GenerationFound}", Name, found); + var found = _nrtReopenThread?.WaitForGeneration(_latestGen.Value, 5000); + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("{IndexName} WaitForChanges returned {GenerationFound}", Name, found); + } + } + else + { + // TODO: MaybeRefresh } } } @@ -1307,13 +1342,17 @@ protected virtual void Dispose(bool disposing) { OnIndexingError(new IndexingErrorEventArgs(this, "Error closing the index", "-1", e)); } - - } _cancellationTokenSource.Dispose(); _logOutput?.Close(); + + _fieldAnalyzer?.Dispose(); + if (!object.ReferenceEquals(_fieldAnalyzer, DefaultAnalyzer)) + { + DefaultAnalyzer?.Dispose(); + } } _disposedValue = true; } diff --git a/src/Examine.Lucene/Providers/LuceneSearcher.cs b/src/Examine.Lucene/Providers/LuceneSearcher.cs index c20194a15..9642a0f76 100644 --- a/src/Examine.Lucene/Providers/LuceneSearcher.cs +++ b/src/Examine.Lucene/Providers/LuceneSearcher.cs @@ -12,9 +12,11 @@ namespace Examine.Lucene.Providers /// public class LuceneSearcher : BaseLuceneSearcher, IDisposable { + private readonly object _locker = new object(); private readonly SearcherManager _searcherManager; private readonly FieldValueTypeCollection _fieldValueTypeCollection; private bool _disposedValue; + private volatile ISearchContext _searchContext; /// /// Constructor allowing for creating a NRT instance based on a given writer @@ -31,7 +33,17 @@ public LuceneSearcher(string name, SearcherManager searcherManager, Analyzer ana } public override ISearchContext GetSearchContext() - => new SearchContext(_searcherManager, _fieldValueTypeCollection); + { + var isCurrent = _searcherManager.IsSearcherCurrent(); + if (_searchContext is null || !isCurrent) + { + _searchContext = new SearchContext(_searcherManager, _fieldValueTypeCollection); + } + + return _searchContext; + + //return new SearchContext(_searcherManager, _fieldValueTypeCollection); + } protected virtual void Dispose(bool disposing) { diff --git a/src/Examine.Lucene/PublicAPI.Unshipped.txt b/src/Examine.Lucene/PublicAPI.Unshipped.txt index 234369766..1b2c2a474 100644 --- a/src/Examine.Lucene/PublicAPI.Unshipped.txt +++ b/src/Examine.Lucene/PublicAPI.Unshipped.txt @@ -1,2 +1,6 @@ Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo localDir, System.IO.DirectoryInfo mainDir, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, bool tryFixMainIndexIfCorrupt) -> void +Examine.Lucene.LuceneIndexOptions.NrtTargetMaxStaleSec.get -> double +Examine.Lucene.LuceneIndexOptions.NrtTargetMaxStaleSec.set -> void +Examine.Lucene.LuceneIndexOptions.NrtTargetMinStaleSec.get -> double +Examine.Lucene.LuceneIndexOptions.NrtTargetMinStaleSec.set -> void virtual Examine.Lucene.Providers.LuceneIndex.UpdateLuceneDocument(Lucene.Net.Index.Term term, Lucene.Net.Documents.Document doc) -> long? \ No newline at end of file diff --git a/src/Examine.Lucene/Search/ISearchContext.cs b/src/Examine.Lucene/Search/ISearchContext.cs index 205a843dc..ee89e9b3b 100644 --- a/src/Examine.Lucene/Search/ISearchContext.cs +++ b/src/Examine.Lucene/Search/ISearchContext.cs @@ -8,6 +8,7 @@ public interface ISearchContext ISearcherReference GetSearcher(); string[] SearchableFields { get; } + IIndexFieldValueType GetFieldValueType(string fieldName); } } diff --git a/src/Examine.Lucene/Search/LuceneSearchExecutor.cs b/src/Examine.Lucene/Search/LuceneSearchExecutor.cs index 62ac2735a..edb836198 100644 --- a/src/Examine.Lucene/Search/LuceneSearchExecutor.cs +++ b/src/Examine.Lucene/Search/LuceneSearchExecutor.cs @@ -36,14 +36,16 @@ private int MaxDoc { get { - if (_maxDoc == null) - { - using (ISearcherReference searcher = _searchContext.GetSearcher()) - { - _maxDoc = searcher.IndexSearcher.IndexReader.MaxDoc; - } - } - return _maxDoc.Value; + return 100; + ////if (_maxDoc == null) + ////{ + //// using (ISearcherReference searcher = _searchContext.GetSearcher()) + //// { + //// // TODO: Getting the IndexSearcher here will call .Acquire() on the SearcherManager again + //// _maxDoc = searcher.IndexSearcher.IndexReader.MaxDoc; + //// } + ////} + ////return _maxDoc.Value; } } diff --git a/src/Examine.Lucene/Search/SearchContext.cs b/src/Examine.Lucene/Search/SearchContext.cs index 406a6941f..d17d4d889 100644 --- a/src/Examine.Lucene/Search/SearchContext.cs +++ b/src/Examine.Lucene/Search/SearchContext.cs @@ -12,15 +12,32 @@ public class SearchContext : ISearchContext { private readonly SearcherManager _searcherManager; private readonly FieldValueTypeCollection _fieldValueTypeCollection; + private readonly Lazy _searcherReference; private string[] _searchableFields; public SearchContext(SearcherManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection) { _searcherManager = searcherManager; _fieldValueTypeCollection = fieldValueTypeCollection ?? throw new ArgumentNullException(nameof(fieldValueTypeCollection)); + _searcherReference = new Lazy(() => + { + // TODO: Only if NRT is disabled? + //_searcherManager.MaybeRefresh(); + + return new SearcherReference(_searcherManager); + }); } - public ISearcherReference GetSearcher() => new SearcherReference(_searcherManager); + // TODO: Do we want to create a new searcher every time? I think so, but we shouldn't allocate so much + public ISearcherReference GetSearcher() + { + //return _searcherReference.Value; + + // TODO: Only if NRT is disabled? + //_searcherManager.MaybeRefresh(); + + return new SearcherReference(_searcherManager); + } public string[] SearchableFields { @@ -32,8 +49,10 @@ public string[] SearchableFields // will not release it from the searcher manager. When we are collecting fields, we are essentially // performing a 'search'. We must ensure that the underlying reader has the correct reference counts. IndexSearcher searcher = _searcherManager.Acquire(); + //var searcher = GetSearcher().IndexSearcher; + try - { + { var fields = MultiFields.GetMergedFieldInfos(searcher.IndexReader) .Select(x => x.Name) .ToList(); diff --git a/src/Examine.Lucene/Search/SearcherReference.cs b/src/Examine.Lucene/Search/SearcherReference.cs index 9e2335b0b..f658bad51 100644 --- a/src/Examine.Lucene/Search/SearcherReference.cs +++ b/src/Examine.Lucene/Search/SearcherReference.cs @@ -3,49 +3,43 @@ namespace Examine.Lucene.Search { - public class SearcherReference : ISearcherReference + // TODO: struct + public readonly struct SearcherReference : ISearcherReference { - private bool _disposedValue; + //private bool _disposedValue; private readonly SearcherManager _searcherManager; - private IndexSearcher _searcher; + private readonly IndexSearcher _searcher; public SearcherReference(SearcherManager searcherManager) { _searcherManager = searcherManager; + _searcher = _searcherManager.Acquire(); } public IndexSearcher IndexSearcher { get { - if (_disposedValue) - { - throw new ObjectDisposedException($"{nameof(SearcherReference)} is disposed"); - } - return _searcher ?? (_searcher = _searcherManager.Acquire()); - } - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - if (_searcher != null) - { - _searcherManager.Release(_searcher); - } - } + //if (_disposedValue) + //{ + // throw new ObjectDisposedException($"{nameof(SearcherReference)} is disposed"); + //} - _disposedValue = true; + //return _searcher ??= _searcherManager.Acquire(); + return _searcher; } } public void Dispose() { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); + //if (!_disposedValue) + //{ + //if (_searcher != null) + //{ + _searcherManager.Release(_searcher); + //} + // _disposedValue = true; + //} } } } diff --git a/src/Examine.Test/Examine.Lucene/Search/ConcurrentSearchBenchmarks.cs b/src/Examine.Test/Examine.Lucene/Search/ConcurrentSearchBenchmarks.cs index b7cc8e3a2..c8be0c4c3 100644 --- a/src/Examine.Test/Examine.Lucene/Search/ConcurrentSearchBenchmarks.cs +++ b/src/Examine.Test/Examine.Lucene/Search/ConcurrentSearchBenchmarks.cs @@ -44,6 +44,78 @@ namespace Examine.Test.Examine.Lucene.Search | LuceneSimple | 5 | 16.953 ms | 6.1768 ms | 0.3386 ms | 5.0000 | - | 1250.0000 | 1000.0000 | 93.7500 | 15.06 MB | | ExamineStandard | 15 | 657.503 ms | 195.5415 ms | 10.7183 ms | 15.0000 | - | 3000.0000 | 1000.0000 | - | 42.92 MB | | LuceneSimple | 15 | 60.278 ms | 100.6474 ms | 5.5168 ms | 15.0000 | - | 4333.3333 | 2666.6667 | 1000.0000 | 45.2 MB | + + +| Method | ThreadCount | NrtTargetMaxStaleSec | NrtTargetMinStaleSec | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | +|---------------- |------------ |--------------------- |--------------------- |----------:|-------------:|----------:|----------:|---------------------:|-----------------:|----------:|----------:| +|---------------- |------------ |--------------------- |--------------------- |----------:|-------------:|----------:|----------:|---------------------:|-----------------:|----------:|----------:| +| ExamineStandard | 1 | 5 | 0.1 | 13.20 ms | 6.540 ms | 0.358 ms | 250.0000 | 1.0000 | 0.0313 | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 5 | 1 | 12.91 ms | 4.004 ms | 0.219 ms | 250.0000 | 1.0000 | 0.0313 | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 5 | 5 | 13.12 ms | 5.002 ms | 0.274 ms | 250.0000 | 1.0000 | 0.0313 | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 30 | 0.1 | 13.00 ms | 2.902 ms | 0.159 ms | 250.0000 | 1.0000 | - | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 30 | 1 | 12.99 ms | 1.982 ms | 0.109 ms | 250.0000 | 1.0000 | - | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 30 | 5 | 13.12 ms | 4.763 ms | 0.261 ms | 250.0000 | 1.0000 | - | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 60 | 0.1 | 12.94 ms | 4.190 ms | 0.230 ms | 250.0000 | 1.0000 | 0.0313 | 156.2500 | 3.13 MB | +| ExamineStandard | 1 | 60 | 1 | 13.17 ms | 4.026 ms | 0.221 ms | 250.0000 | 1.0000 | 0.0781 | 140.6250 | 3.13 MB | +| ExamineStandard | 1 | 60 | 5 | 13.59 ms | 7.591 ms | 0.416 ms | 250.0000 | 1.0000 | 0.0625 | 156.2500 | 3.13 MB | +| ExamineStandard | 5 | 5 | 0.1 | 164.78 ms | 146.379 ms | 8.024 ms | 1000.0000 | 5.0000 | 4.0000 | 666.6667 | 14.7 MB | +| ExamineStandard | 5 | 5 | 1 | 155.77 ms | 173.985 ms | 9.537 ms | 1000.0000 | 5.0000 | 4.0000 | 750.0000 | 14.7 MB | +| ExamineStandard | 5 | 5 | 5 | 154.61 ms | 184.531 ms | 10.115 ms | 1000.0000 | 5.0000 | 4.0000 | 666.6667 | 14.7 MB | +| ExamineStandard | 5 | 30 | 0.1 | 159.31 ms | 100.583 ms | 5.513 ms | 1000.0000 | 5.0000 | 4.0000 | 750.0000 | 14.7 MB | +| ExamineStandard | 5 | 30 | 1 | 157.80 ms | 79.096 ms | 4.336 ms | 1000.0000 | 5.0000 | 4.0000 | 666.6667 | 14.7 MB | +| ExamineStandard | 5 | 30 | 5 | 164.48 ms | 171.208 ms | 9.384 ms | 1000.0000 | 5.0000 | 4.0000 | 666.6667 | 14.7 MB | +| ExamineStandard | 5 | 60 | 0.1 | 166.63 ms | 163.111 ms | 8.941 ms | 1000.0000 | 5.0000 | 4.0000 | 666.6667 | 14.7 MB | +| ExamineStandard | 5 | 60 | 1 | 156.79 ms | 151.734 ms | 8.317 ms | 1000.0000 | 5.0000 | 4.0000 | 750.0000 | 14.7 MB | +| ExamineStandard | 5 | 60 | 5 | 160.94 ms | 105.412 ms | 5.778 ms | 1000.0000 | 5.0000 | 4.0000 | 666.6667 | 14.7 MB | +| ExamineStandard | 15 | 5 | 0.1 | 661.02 ms | 1,007.163 ms | 55.206 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.61 MB | +| ExamineStandard | 15 | 5 | 1 | 619.05 ms | 282.484 ms | 15.484 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.61 MB | +| ExamineStandard | 15 | 5 | 5 | 615.87 ms | 830.232 ms | 45.508 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.62 MB | +| ExamineStandard | 15 | 30 | 0.1 | 662.71 ms | 1,119.952 ms | 61.388 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.62 MB | +| ExamineStandard | 15 | 30 | 1 | 677.54 ms | 449.274 ms | 24.626 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.61 MB | +| ExamineStandard | 15 | 30 | 5 | 679.11 ms | 963.257 ms | 52.799 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.62 MB | +| ExamineStandard | 15 | 60 | 0.1 | 695.26 ms | 471.371 ms | 25.837 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.62 MB | +| ExamineStandard | 15 | 60 | 1 | 628.51 ms | 421.771 ms | 23.119 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.61 MB | +| ExamineStandard | 15 | 60 | 5 | 706.39 ms | 510.552 ms | 27.985 ms | 3000.0000 | + +Without NRT + +| Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | +|---------------- |------------ |----------:|-----------:|----------:|----------:|---------------------:|-----------------:|----------:|----------:| +| ExamineStandard | 1 | 12.48 ms | 3.218 ms | 0.176 ms | 250.0000 | 1.0000 | 0.0938 | 156.2500 | 3.13 MB | +| ExamineStandard | 5 | 149.31 ms | 88.914 ms | 4.874 ms | 1000.0000 | 5.0000 | 4.0000 | 750.0000 | 14.7 MB | +| ExamineStandard | 15 | 613.14 ms | 897.936 ms | 49.219 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.67 MB | + +Without querying MaxDoc + +| Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | +|---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| +| ExamineStandard | 1 | 5.223 ms | 1.452 ms | 0.0796 ms | 78.1250 | 1.0000 | 0.0313 | 7.8125 | 962 KB | +| ExamineStandard | 5 | 26.772 ms | 9.982 ms | 0.5471 ms | 312.5000 | 5.0000 | 4.0000 | 187.5000 | 3825.35 KB | +| ExamineStandard | 15 | 101.483 ms | 65.690 ms | 3.6007 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10989.05 KB | + +Without apply deletes + +| Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | +|---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| +| ExamineStandard | 1 | 5.554 ms | 1.745 ms | 0.0957 ms | 78.1250 | 1.0000 | - | 31.2500 | 961.73 KB | +| ExamineStandard | 5 | 26.960 ms | 4.797 ms | 0.2629 ms | 312.5000 | 5.0000 | 4.0313 | 187.5000 | 3826.6 KB | +| ExamineStandard | 15 | 103.939 ms | 49.361 ms | 2.7057 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10991.87 KB | + +Using struct (doesn't change anything) + +| Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | +|---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| +| ExamineStandard | 1 | 5.661 ms | 2.477 ms | 0.1357 ms | 78.1250 | 1.0000 | 0.0625 | 31.2500 | 961.56 KB | +| ExamineStandard | 5 | 28.364 ms | 3.615 ms | 0.1981 ms | 312.5000 | 5.0000 | 4.0000 | 187.5000 | 3825.91 KB | +| ExamineStandard | 15 | 100.561 ms | 26.820 ms | 1.4701 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10986.15 KB | + +|---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| +| Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | +|---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| +| ExamineStandard | 1 | 5.471 ms | 1.430 ms | 0.0784 ms | 78.1250 | 1.0000 | 0.0156 | 31.2500 | 958.55 KB | +| ExamineStandard | 5 | 26.521 ms | 1.837 ms | 0.1007 ms | 312.5000 | 5.0000 | 4.0000 | 156.2500 | 3808.24 KB | +| ExamineStandard | 15 | 102.785 ms | 80.640 ms | 4.4202 ms | 833.3333 | 15.0000 | 14.0000 | 500.0000 | 10935.97 KB | + */ [ShortRunJob] [ThreadingDiagnoser] @@ -52,10 +124,10 @@ namespace Examine.Test.Examine.Lucene.Search [CPUUsageDiagnoser] public class ConcurrentSearchBenchmarks : ExamineBaseTest { + private readonly StandardAnalyzer _analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); private ILogger _logger; private string _tempBasePath; private FSDirectory _luceneDir; - private TestIndex _indexer; [GlobalSetup] public override void Setup() @@ -68,9 +140,8 @@ public override void Setup() var tempPath = Path.Combine(_tempBasePath, Guid.NewGuid().ToString()); System.IO.Directory.CreateDirectory(tempPath); var temp = new DirectoryInfo(tempPath); - var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); _luceneDir = FSDirectory.Open(temp); - _indexer = GetTestIndex(_luceneDir, analyzer); + using var indexer = GetTestIndex(_luceneDir, _analyzer); var random = new Random(); var valueSets = new List(); @@ -85,14 +156,14 @@ public override void Setup() })); } - _indexer.IndexItems(valueSets); + indexer.IndexItems(valueSets); } [GlobalCleanup] public override void TearDown() { - _indexer.Dispose(); _luceneDir.Dispose(); + _analyzer.Dispose(); base.TearDown(); @@ -102,9 +173,22 @@ public override void TearDown() [Params(1, 5, 15)] public int ThreadCount { get; set; } + //[Params(5, 30, 60)] + //public double NrtTargetMaxStaleSec { get; set; } + + //[Params(0.1, 1, 5)] + //public double NrtTargetMinStaleSec { get; set; } + [Benchmark] public async Task ExamineStandard() { + using var indexer = GetTestIndex( + _luceneDir, + _analyzer, + nrtEnabled: true); + ////nrtTargetMaxStaleSec: NrtTargetMaxStaleSec, + ////nrtTargetMinStaleSec: NrtTargetMinStaleSec); + var tasks = new List(); for (var i = 0; i < ThreadCount; i++) @@ -112,7 +196,7 @@ public async Task ExamineStandard() tasks.Add(new Task(() => { // always resolve the searcher from the indexer - var searcher = _indexer.Searcher; + var searcher = indexer.Searcher; var query = searcher.CreateQuery("content").Field("nodeName", "location".MultipleCharacterWildcard()); var results = query.Execute(); @@ -131,7 +215,7 @@ public async Task ExamineStandard() await Task.WhenAll(tasks); } - [Benchmark] + //[Benchmark]] public async Task LuceneSimple() { var tasks = new List(); diff --git a/src/Examine.Test/ExamineBaseTest.cs b/src/Examine.Test/ExamineBaseTest.cs index 47dbf80ea..582833101 100644 --- a/src/Examine.Test/ExamineBaseTest.cs +++ b/src/Examine.Test/ExamineBaseTest.cs @@ -25,7 +25,15 @@ public virtual void Setup() [TearDown] public virtual void TearDown() => LoggerFactory.Dispose(); - public TestIndex GetTestIndex(Directory d, Analyzer analyzer, FieldDefinitionCollection fieldDefinitions = null, IndexDeletionPolicy indexDeletionPolicy = null, IReadOnlyDictionary indexValueTypesFactory = null) + public TestIndex GetTestIndex( + Directory d, + Analyzer analyzer, + FieldDefinitionCollection fieldDefinitions = null, + IndexDeletionPolicy indexDeletionPolicy = null, + IReadOnlyDictionary indexValueTypesFactory = null, + double nrtTargetMaxStaleSec = 5.0, + double nrtTargetMinStaleSec = 1.0, + bool nrtEnabled = true) => new TestIndex( LoggerFactory, Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions @@ -34,7 +42,10 @@ public TestIndex GetTestIndex(Directory d, Analyzer analyzer, FieldDefinitionCol DirectoryFactory = new GenericDirectoryFactory(_ => d), Analyzer = analyzer, IndexDeletionPolicy = indexDeletionPolicy, - IndexValueTypesFactory = indexValueTypesFactory + IndexValueTypesFactory = indexValueTypesFactory, + NrtTargetMaxStaleSec = nrtTargetMaxStaleSec, + NrtTargetMinStaleSec = nrtTargetMinStaleSec, + NrtEnabled = nrtEnabled })); public TestIndex GetTestIndex(IndexWriter writer)