Skip to content

Commit

Permalink
Only log on load transition
Browse files Browse the repository at this point in the history
  • Loading branch information
aednichols committed Nov 15, 2023
1 parent 59f5962 commit e9ce90a
Showing 1 changed file with 13 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package cromwell.backend.google.pipelines.common.api

import java.io.IOException
import java.util.UUID

import akka.actor.{Actor, ActorLogging, ActorRef, Props, SupervisorStrategy, Terminated, Timers}
import akka.dispatch.ControlMessage
import cats.data.NonEmptyList
Expand All @@ -18,7 +17,7 @@ import cromwell.core.Dispatcher.BackendDispatcher
import cromwell.core.retry.SimpleExponentialBackoff
import cromwell.core.{CromwellFatalExceptionMarker, LoadConfig, Mailbox, WorkflowId}
import cromwell.services.instrumentation.CromwellInstrumentationScheduler
import cromwell.services.loadcontroller.LoadControllerService.{HighLoad, LoadMetric, NormalLoad}
import cromwell.services.loadcontroller.LoadControllerService.{HighLoad, LoadLevel, LoadMetric, NormalLoad}
import eu.timepit.refined.api.Refined
import eu.timepit.refined.numeric._

Expand Down Expand Up @@ -89,21 +88,25 @@ class PipelinesApiRequestManager(val qps: Int Refined Positive, requestWorkers:
protected[api] var statusPollers: Vector[ActorRef] = Vector.empty
self ! ResetAllRequestWorkers

private var previousLoad: LoadLevel = NormalLoad

Check warning on line 91 in supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala

View check run for this annotation

Codecov / codecov/patch

supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala#L91

Added line #L91 was not covered by tests

override def preStart() = {
log.info("Running with {} PAPI request workers", requestWorkers.value)
startInstrumentationTimer()
super.preStart()
}

def monitorQueueSize() = {
val load = if (workQueue.size > LoadConfig.PAPIThreshold) {
log.warning(s"PAPI Request Manager notifying HighLoad with queue size ${workQueue.size} exceeding limit of ${LoadConfig.PAPIThreshold}")
HighLoad
} else {
log.debug("PAPI Request Manager notifying NormaLoad")
NormalLoad
}
serviceRegistryActor ! LoadMetric("PAPIQueryManager", load)
val newLoad = if (workQueue.size > LoadConfig.PAPIThreshold) HighLoad else NormalLoad

Check warning on line 100 in supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala

View check run for this annotation

Codecov / codecov/patch

supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala#L100

Added line #L100 was not covered by tests

if (previousLoad == NormalLoad && newLoad == HighLoad)
log.warning(s"PAPI Request Manager transitioned to HighLoad with queue size ${workQueue.size} exceeding limit of ${LoadConfig.PAPIThreshold}")
else if (previousLoad == HighLoad && newLoad == NormalLoad)
log.info("PAPI Request Manager transitioned back to NormaLoad")

Check warning on line 105 in supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala

View check run for this annotation

Codecov / codecov/patch

supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala#L102-L105

Added lines #L102 - L105 were not covered by tests

previousLoad = newLoad

Check warning on line 107 in supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala

View check run for this annotation

Codecov / codecov/patch

supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala#L107

Added line #L107 was not covered by tests

serviceRegistryActor ! LoadMetric("PAPIQueryManager", newLoad)

Check warning on line 109 in supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala

View check run for this annotation

Codecov / codecov/patch

supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestManager.scala#L109

Added line #L109 was not covered by tests
updateQueueSize(workQueue.size)
}

Expand Down

0 comments on commit e9ce90a

Please sign in to comment.