Skip to content

Commit

Permalink
OPENNLP-1662: Wrap thread-safe classes in try-with resources in Eval …
Browse files Browse the repository at this point in the history
…test (#705)
  • Loading branch information
mawiesne authored Dec 3, 2024
1 parent 8b84a3e commit f4de6c2
Showing 1 changed file with 27 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import org.junit.jupiter.api.Test;

import opennlp.tools.commons.ThreadSafe;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.ThreadSafePOSTaggerME;
import opennlp.tools.sentdetect.SentenceModel;
Expand All @@ -33,37 +34,37 @@
/**
* Test the reentrant tools implementations are really thread safe by running concurrently.
* Replace the thread-safe versions with the non-safe versions to see this test case fail.
*
* @see ThreadSafe
*/
public class MultiThreadedToolsEval extends AbstractEvalTest {

@Test
public void runMEToolsMultiThreaded() throws IOException, InterruptedException {

File sModelFile = new File(getOpennlpDataDir(), "models-sf/en-sent.bin");
File dataDir = getOpennlpDataDir();
File sModelFile = new File(dataDir, "models-sf/en-sent.bin");
File tModelFile = new File(dataDir, "models-sf/en-token.bin");
File pModelFile = new File(dataDir, "models-sf/en-pos-maxent.bin");
SentenceModel sModel = new SentenceModel(sModelFile);
ThreadSafeSentenceDetectorME sentencer = new ThreadSafeSentenceDetectorME(sModel);

File tModelFile = new File(getOpennlpDataDir(), "models-sf/en-token.bin");
TokenizerModel tModel = new TokenizerModel(tModelFile);
ThreadSafeTokenizerME tokenizer = new ThreadSafeTokenizerME(tModel);

File pModelFile = new File(getOpennlpDataDir(), "models-sf/en-pos-maxent.bin");
POSModel pModel = new POSModel(pModelFile);
ThreadSafePOSTaggerME tagger = new ThreadSafePOSTaggerME(pModel);

final String text = "All human beings are born free and equal in dignity and rights. They " +
"are endowed with reason and conscience and should act towards one another in a " +
"spirit of brotherhood.";
try (ThreadSafeSentenceDetectorME sentencer = new ThreadSafeSentenceDetectorME(sModel);
ThreadSafeTokenizerME tokenizer = new ThreadSafeTokenizerME(tModel);
ThreadSafePOSTaggerME tagger = new ThreadSafePOSTaggerME(pModel)) {

// Run numThreads threads, each processing the sample text numRunsPerThread times.
final int numThreads = 8;
final int numRunsPerThread = 1000;
Thread[] threads = new Thread[numThreads];
final String text = "All human beings are born free and equal in dignity and rights. They " +
"are endowed with reason and conscience and should act towards one another in a " +
"spirit of brotherhood.";

for (int i = 0; i < 8; i++) {
threads[i] = new Thread(new Runnable() {
@Override
public void run() {
// Run numThreads threads, each processing the sample text numRunsPerThread times.
final int numThreads = 8;
final int numRunsPerThread = 1000;
Thread[] threads = new Thread[numThreads];

for (int i = 0; i < 8; i++) {
threads[i] = new Thread(() -> {
for (int j = 0; j < numRunsPerThread; j++) {
Span[] sentences = sentencer.sentPosDetect(text);
for (Span span : sentences) {
Expand All @@ -72,19 +73,18 @@ public void run() {
String[] tokenStrings = new String[tokens.length];
for (int k = 0; k < tokens.length; k++) {
tokenStrings[k] = sentence.substring(tokens[k].getStart(),
tokens[k].getEnd());
tokens[k].getEnd());
}
String[] tags = tagger.tag(tokenStrings);
}
}
}
});
threads[i].start();
}
for (Thread t : threads) {
t.join();
});
threads[i].start();
}
for (Thread t : threads) {
t.join();
}
}

}

}

0 comments on commit f4de6c2

Please sign in to comment.