Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes Lucene.Net.Index.TestIndexWriter::TestThreadInterruptDeadlock() and Lucene.Net.Index.TestIndexWriter::TestTwoThreadsInterruptDeadlock() #525

Merged
merged 8 commits into from
Oct 17, 2021
71 changes: 62 additions & 9 deletions src/Lucene.Net.Analysis.Common/Analysis/Th/ThaiTokenizer.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
// Lucene version compatibility level 4.8.1
// Lucene version compatibility level 4.8.1
#if FEATURE_BREAKITERATOR
using ICU4N.Text;
using J2N;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Analysis.Util;
using Lucene.Net.Support.Threading;
using System;
using System.Collections.Generic;
using System.Globalization;
Expand Down Expand Up @@ -54,14 +55,28 @@ public class ThaiTokenizer : SegmentingTokenizerBase

private static BreakIterator LoadProto()
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
return BreakIterator.GetWordInstance(new CultureInfo("th"));
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

private static BreakIterator LoadSentenceProto()
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
return BreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

private readonly ThaiWordBreaker wordBreaker;
Expand All @@ -87,45 +102,79 @@ public ThaiTokenizer(AttributeFactory factory, TextReader reader)
{
// LUCENENET specific - DBBI_AVAILABLE removed because ICU always has a dictionary-based BreakIterator

lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
wordBreaker = new ThaiWordBreaker((BreakIterator)proto.Clone());
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
termAtt = AddAttribute<ICharTermAttribute>();
offsetAtt = AddAttribute<IOffsetAttribute>();
}

private static BreakIterator CreateSentenceClone()
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
return (BreakIterator)sentenceProto.Clone();
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

public override void Reset()
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
base.Reset();
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

public override State CaptureState()
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
return base.CaptureState();
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

protected override void SetNextSentence(int sentenceStart, int sentenceEnd)
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
this.sentenceStart = sentenceStart;
this.sentenceEnd = sentenceEnd;
wrapper.SetText(m_buffer, sentenceStart, sentenceEnd - sentenceStart);
wordBreaker.SetText(new string(wrapper.Text, wrapper.Start, wrapper.Length));
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

protected override bool IncrementWord()
{
int start, end;
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
start = wordBreaker.Current;
if (start == BreakIterator.Done)
Expand All @@ -151,6 +200,10 @@ protected override bool IncrementWord()
offsetAtt.SetOffset(CorrectOffset(m_offset + sentenceStart + start), CorrectOffset(m_offset + sentenceStart + end));
return true;
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Lucene version compatibility level 4.8.1
using J2N.Collections.Generic.Extensions;
using Lucene.Net.Support;
using Lucene.Net.Support.Threading;
using Lucene.Net.Util;
using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -60,7 +61,8 @@ public AnalysisSPILoader(string[] suffixes)
/// </summary>
public void Reload()
{
lock (this)
UninterruptableMonitor.Enter(this);
try
{
IDictionary<string, Type> services = new JCG.LinkedDictionary<string, Type>(this.services);
SPIClassIterator<S> loader = SPIClassIterator<S>.Get();
Expand Down Expand Up @@ -98,6 +100,10 @@ public void Reload()
}
this.services = services.AsReadOnly();
}
finally
{
UninterruptableMonitor.Exit(this);
}
}

public S NewInstance(string name, IDictionary<string, string> args)
Expand Down
43 changes: 37 additions & 6 deletions src/Lucene.Net.Analysis.Common/Analysis/Util/BufferedCharFilter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// https://svn.apache.org/repos/asf/harmony/enhanced/java/trunk/

using Lucene.Net.Analysis.CharFilters;
using Lucene.Net.Support.Threading;
using System;
using System.IO;
using System.Text;
Expand Down Expand Up @@ -119,7 +120,8 @@ protected override void Dispose(bool disposing)
#if FEATURE_TEXTWRITER_CLOSE
this.isDisposing = true;
#endif
lock (m_lock)
UninterruptableMonitor.Enter(m_lock);
try
{
if (!IsClosed)
{
Expand All @@ -128,6 +130,10 @@ protected override void Dispose(bool disposing)
buf = null;
}
}
finally
{
UninterruptableMonitor.Exit(m_lock);
}
#if FEATURE_TEXTWRITER_CLOSE
this.isDisposing = false;
#endif
Expand Down Expand Up @@ -225,12 +231,17 @@ public override void Mark(int markLimit)
{
throw new ArgumentOutOfRangeException(nameof(markLimit), "Read-ahead limit < 0");
}
lock (m_lock)
UninterruptableMonitor.Enter(m_lock);
try
{
EnsureOpen();
this.markLimit = markLimit;
mark = pos;
}
finally
{
UninterruptableMonitor.Exit(m_lock);
}
}

/// <summary>
Expand All @@ -253,7 +264,8 @@ public override void Mark(int markLimit)
/// <exception cref="IOException">If this reader is disposed or some other I/O error occurs.</exception>
public override int Read()
{
lock (m_lock)
UninterruptableMonitor.Enter(m_lock);
try
{
EnsureOpen();
/* Are there buffered characters available? */
Expand All @@ -263,6 +275,10 @@ public override int Read()
}
return -1;
}
finally
{
UninterruptableMonitor.Exit(m_lock);
}
}

/// <summary>
Expand Down Expand Up @@ -483,13 +499,18 @@ public override bool IsReady
{
get
{
lock (m_lock)
UninterruptableMonitor.Enter(m_lock);
try
{
EnsureOpen();
// LUCENENET specific: only CharFilter derived types support IsReady
var charFilter = @in as CharFilter;
return ((end - pos) > 0) || (charFilter != null && charFilter.IsReady);
}
finally
{
UninterruptableMonitor.Exit(m_lock);
}
}
}

Expand All @@ -503,7 +524,8 @@ public override bool IsReady
/// <seealso cref="IsMarkSupported"/>
public override void Reset()
{
lock (m_lock)
UninterruptableMonitor.Enter(m_lock);
try
{
EnsureOpen();
if (mark < 0)
Expand All @@ -513,6 +535,10 @@ public override void Reset()
}
pos = mark;
}
finally
{
UninterruptableMonitor.Exit(m_lock);
}
}

/// <summary>
Expand All @@ -534,7 +560,8 @@ public override long Skip(int amount)
{
throw new ArgumentOutOfRangeException(nameof(amount), "skip value is negative");
}
lock (m_lock)
UninterruptableMonitor.Enter(m_lock);
try
{
EnsureOpen();
if (amount < 1)
Expand Down Expand Up @@ -566,6 +593,10 @@ public override long Skip(int amount)
}
return amount;
}
finally
{
UninterruptableMonitor.Exit(m_lock);
}
}

#region LUCENENET Specific Methods
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Diagnostics;
using Lucene.Net.Support;
using Lucene.Net.Support.Threading;
using System;
using System.Diagnostics;
using System.IO;
Expand Down Expand Up @@ -111,7 +112,8 @@ public ICUTokenizer(AttributeFactory factory, TextReader input, ICUTokenizerConf

public override bool IncrementToken()
{
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
ClearAttributes();
if (length == 0)
Expand All @@ -124,14 +126,25 @@ public override bool IncrementToken()
}
return true;
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}


public override void Reset()
{
base.Reset();
lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
breaker.SetText(buffer, 0, 0);
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
length = usableLength = offset = 0;
}

Expand Down Expand Up @@ -193,8 +206,15 @@ private void Refill()
*/
}

lock (syncLock)
UninterruptableMonitor.Enter(syncLock);
try
{
breaker.SetText(buffer, 0, Math.Max(0, usableLength));
}
finally
{
UninterruptableMonitor.Exit(syncLock);
}
}

// TODO: refactor to a shared readFully somewhere
Expand Down
8 changes: 7 additions & 1 deletion src/Lucene.Net.Analysis.OpenNLP/Tools/NLPChunkerOp.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Lucene version compatibility level 8.2.0
using Lucene.Net.Support.Threading;
using opennlp.tools.chunker;


Expand Down Expand Up @@ -36,13 +37,18 @@ public NLPChunkerOp(ChunkerModel chunkerModel)

public virtual string[] GetChunks(string[] words, string[] tags, double[] probs)
{
lock (this)
UninterruptableMonitor.Enter(this);
try
{
string[] chunks = chunker.chunk(words, tags);
if (probs != null)
chunker.probs(probs);
return chunks;
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
}
}
Loading