From d1d07109cd0318af2e9e9b6ba63017e9975dde83 Mon Sep 17 00:00:00 2001 From: Gabor Pali Date: Thu, 15 Jun 2023 16:50:16 +0200 Subject: [PATCH] Add metric on the count of NativeFSLocks held Lucene 4.6 is known to leak locks when an exception hits the rollback of an index writer. Introduce an optional metric on the size of the internal store that tracks the currently held NativeFSLocks to be able to assess its severity in practice. The method is specific to the Lucene internals and it uses Java reflection to extract the data as it is officially not accessible. --- .../com/cloudant/clouseau/IndexManagerService.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala b/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala index b25a1f0..7342989 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory import scalang._ import com.yammer.metrics.scala._ import scala.collection.JavaConversions._ +import java.util.HashSet class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Service(ctx) with Instrumented { @@ -94,6 +95,18 @@ class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Servic val openTimer = metrics.timer("opens") val lru = new LRU() val waiters = Map[String, List[(Pid, Any)]]() + val countLocksEnabled = ctx.args.config.getBoolean("clouseau.count_locks", false) + if (countLocksEnabled) { + val lockClass = Class.forName("org.apache.lucene.store.NativeFSLock") + val field = lockClass.getDeclaredField("LOCK_HELD") + field.setAccessible(true) + val LOCK_HELD = field.get(null).asInstanceOf[HashSet[String]] + metrics.gauge("NativeFSLock.count")(getNativeFSLockHeldSize(LOCK_HELD)) + } + + def getNativeFSLockHeldSize(lockHeld: Collection[String]) = lockHeld.synchronized { + lockHeld.size + } override def handleCall(tag: (Pid, Any), msg: Any): Any = msg match { case OpenIndexMsg(peer: Pid, path: String, options: Any) =>