From 767d9c2aaa9e0ab9de0c1cc2736a5f51e016d5e6 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Fri, 15 Dec 2023 12:01:26 +0100 Subject: [PATCH 01/53] wip --- build.sbt | 32 +++++++++---- .../main/scala/sttp/tapir/perf/Common.scala | 13 +++++ .../sttp/tapir/perf/PerfTestServer.scala | 3 ++ .../tapir/perf/apis/SimpleGetEndpoints.scala | 46 ++++++++++++++++++ .../scala/sttp/tapir/perf/http4s/Http4s.scala | 17 +++++-- .../sttp/tapir/perf/netty/NettyFuture.scala | 43 +++++++++++++++++ .../scala/sttp/tapir/perf/Simulations.scala | 47 +++++++++++++++++++ .../FileWriterSubscriber.scala | 2 + .../reactivestreams/SimpleSubscriber.scala | 5 +- .../src/test/resources/logback.xml | 12 +++++ .../src/main/scala/zio/LimitedZioBody.scala | 38 +++++++++++++++ 11 files changed, 243 insertions(+), 15 deletions(-) create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala create mode 100644 server/pekko-http-server/src/test/resources/logback.xml create mode 100644 server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala diff --git a/build.sbt b/build.sbt index 139ba282f5..6719fa5b6f 100644 --- a/build.sbt +++ b/build.sbt @@ -68,7 +68,7 @@ val commonSettings = commonSmlBuildSettings ++ ossPublishSettings ++ Seq( }.value, mimaPreviousArtifacts := Set.empty, // we only use MiMa for `core` for now, using enableMimaSettings ideSkipProject := (scalaVersion.value == scala2_12) || - (scalaVersion.value == scala2_13) || + (scalaVersion.value == scala3) || thisProjectRef.value.project.contains("Native") || thisProjectRef.value.project.contains("JS"), bspEnabled := !ideSkipProject.value, @@ -504,13 +504,20 @@ val http4sVanilla = taskKey[Unit]("http4s-vanilla") val http4sTapir = taskKey[Unit]("http4s-tapir") val http4sVanillaMulti = taskKey[Unit]("http4s-vanilla-multi") val http4sTapirMulti = taskKey[Unit]("http4s-tapir-multi") -def genPerfTestTask(servName: String, simName: String) = Def.taskDyn { - Def.task { - (Compile / runMain).toTask(s" sttp.tapir.perf.${servName}Server").value - (Gatling / testOnly).toTask(s" sttp.tapir.perf.${simName}Simulation").value - } + +import complete.DefaultParsers._ + +val perfTestParser: complete.Parser[(String, String)] = { + Space ~> token(StringBasic.examples("")) ~ (Space ~> token(StringBasic.examples(""))) +} + +def genPerfTestTask(servName: String, simName: String): Def.Initialize[Task[Unit]] = Def.task { + (Compile / runMain).toTask(s" sttp.tapir.perf.${servName}Server").value + (Gatling / testOnly).toTask(s" sttp.tapir.perf.${simName}Simulation").value } +lazy val perfTest = inputKey[Unit]("Run performance test") + lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .enablePlugins(GatlingPlugin) .settings(commonJvmSettings) @@ -534,16 +541,23 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) fork := true, connectInput := true ) + .settings( + perfTest := { + genPerfTestTask.tupled(perfTestParser.parsed).value + // val (servName, simName) = perfTestParser.parsed + // genPerfTestTask(servName, simName).value + } + ) .settings(akkaHttpVanilla := { (genPerfTestTask("akka.Vanilla", "OneRoute")).value }) .settings(akkaHttpTapir := { (genPerfTestTask("akka.Tapir", "OneRoute")).value }) .settings(akkaHttpVanillaMulti := { (genPerfTestTask("akka.VanillaMulti", "MultiRoute")).value }) .settings(akkaHttpTapirMulti := { (genPerfTestTask("akka.TapirMulti", "MultiRoute")).value }) .settings(http4sVanilla := { (genPerfTestTask("http4s.Vanilla", "OneRoute")).value }) - .settings(http4sTapir := { (genPerfTestTask("http4s.Tapir", "OneRoute")).value }) + .settings(http4sTapir := { (genPerfTestTask("netty.TapirMulti", "PostBytes256")).value }) .settings(http4sVanillaMulti := { (genPerfTestTask("http4s.VanillaMulti", "MultiRoute")).value }) - .settings(http4sTapirMulti := { (genPerfTestTask("http4s.TapirMulti", "MultiRoute")).value }) + .settings(http4sTapirMulti := { (genPerfTestTask("http4s.TapirMulti", "PostString")).value }) .jvmPlatform(scalaVersions = examplesScalaVersions) - .dependsOn(core, akkaHttpServer, http4sServer) + .dependsOn(core, akkaHttpServer, http4sServer, nettyServer) // integrations diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index d44b39738d..6ef45594c5 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,8 +1,11 @@ package sttp.tapir.perf +import sttp.monad.MonadError +import sttp.monad.syntax._ import sttp.tapir.{PublicEndpoint, endpoint, path, stringBody} import scala.io.StdIn +import sttp.tapir.server.ServerEndpoint object Common { def genTapirEndpoint(n: Int): PublicEndpoint[Int, String, String, Any] = endpoint.get @@ -11,6 +14,16 @@ object Common { .errorOut(stringBody) .out(stringBody) + def genPostStrTapirEndpoint(n: Int): PublicEndpoint[String, String, String, Any] = endpoint.post + .in("path" + n.toString) + .in(stringBody) + .errorOut(stringBody) + .out(stringBody) + + def replyWithStr[F[_]](endpoint: PublicEndpoint[_, _, String, Any])(implicit monad: MonadError[F]): ServerEndpoint[Any, F] = + endpoint.serverLogicSuccess[F](_ => monad.eval("ok")) + + def blockServer(): Unit = { println(Console.BLUE + "Server now online. Please navigate to http://localhost:8080/path0/1\nPress RETURN to stop..." + Console.RESET) StdIn.readLine() diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala b/perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala new file mode 100644 index 0000000000..5a59c97d38 --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala @@ -0,0 +1,3 @@ +package sttp.tapir.perf + +trait PerfTestServer {} diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala b/perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala new file mode 100644 index 0000000000..ac9bd0b76b --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala @@ -0,0 +1,46 @@ +package sttp.tapir.perf.apis + +import sttp.tapir._ +import sttp.monad.MonadError +import sttp.tapir.server.ServerEndpoint +import sttp.tapir.server.model.EndpointExtensions._ + +trait SimpleGetEndpoints { + type EndpointGen = Int => PublicEndpoint[_, String, String, Any] + type ServerEndpointGen[F[_]] = Int => ServerEndpoint[Any, F] + + val gen_get_in_string_out_string: EndpointGen = { (n: Int) => + endpoint.get + .in("path" + n.toString) + .in(path[Int]("id")) + .errorOut(stringBody) + .out(stringBody) + } + + val gen_post_in_string_out_string: EndpointGen = { (n: Int) => + endpoint.post + .in("path" + n.toString) + .in(path[Int]("id")) + .in(stringBody) + .errorOut(stringBody) + .out(stringBody) + } + + val gen_post_in_file_out_string: EndpointGen = { (n: Int) => + endpoint.post + .in("pathFile" + n.toString) + .in(path[Int]("id")) + .in(fileBody) + .maxRequestBodyLength(300000) + .errorOut(stringBody) + .out(stringBody) + } + + def replyingWithDummyStr[F[_]](endpointGens: List[EndpointGen])(implicit + monad: MonadError[F] + ): Seq[ServerEndpointGen[F]] = + endpointGens.map(gen => gen.andThen(se => se.serverLogicSuccess[F](_ => monad.eval("ok")))) + + def genServerEndpoints[F[_]](gens: Seq[ServerEndpointGen[F]])(routeCount: Int): Seq[ServerEndpoint[Any, F]] = + gens.flatMap(gen => (0 to routeCount).map(i => gen(i))) +} diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 588d30c846..2c825233fd 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -11,6 +11,9 @@ import org.http4s.server.Router import sttp.tapir.perf import sttp.tapir.perf.Common import sttp.tapir.server.http4s.Http4sServerInterpreter +import sttp.tapir.integ.cats.effect.CatsMonadError +import sttp.monad.MonadError +import sttp.tapir.server.ServerEndpoint object Vanilla { val router: Int => HttpRoutes[IO] = (nRoutes: Int) => @@ -27,13 +30,18 @@ object Vanilla { ) } -object Tapir { +object Tapir extends perf.apis.SimpleGetEndpoints { + + implicit val mErr: MonadError[IO] = new CatsMonadError[IO] + + val serverEndpointGens = replyingWithDummyStr[IO]( + List(gen_get_in_string_out_string, gen_post_in_string_out_string) + ) + val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router("/" -> { Http4sServerInterpreter[IO]().toRoutes( - (0 to nRoutes) - .map((n: Int) => Common.genTapirEndpoint(n).serverLogic(id => IO(((id + n).toString).asRight[String]))) - .toList + genServerEndpoints(serverEndpointGens)(nRoutes).toList ) }) } @@ -51,4 +59,5 @@ object Http4s { object TapirServer extends App { Http4s.runServer(Tapir.router(1)).unsafeRunSync() } object TapirMultiServer extends App { Http4s.runServer(Tapir.router(128)).unsafeRunSync() } object VanillaServer extends App { Http4s.runServer(Vanilla.router(1)).unsafeRunSync() } +object PostStringServer extends App { Http4s.runServer(Vanilla.router(1)).unsafeRunSync() } object VanillaMultiServer extends App { Http4s.runServer(Vanilla.router(128)).unsafeRunSync() } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala new file mode 100644 index 0000000000..ed43ab2193 --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala @@ -0,0 +1,43 @@ +package sttp.tapir.perf.netty + +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import sttp.tapir.server.ServerEndpoint +import scala.concurrent.Future +import sttp.tapir.perf.apis.SimpleGetEndpoints +import sttp.monad.MonadError +import sttp.monad.FutureMonad +import scala.concurrent.ExecutionContext +import sttp.tapir.server.netty.{NettyFutureServer, NettyFutureServerBinding} +import ExecutionContext.Implicits.global + +object Tapir extends SimpleGetEndpoints { + + implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) + + val serverEndpointGens = replyingWithDummyStr[Future]( + List(gen_get_in_string_out_string, gen_post_in_string_out_string, gen_post_in_file_out_string) + ) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList +} + +object NettyFuture { + + def runServer(endpoints: List[ServerEndpoint[Any, Future]]): Unit = { + val declaredPort = 8080 + val declaredHost = "0.0.0.0" + // Starting netty server + val serverBinding: NettyFutureServerBinding = + Await.result( + NettyFutureServer() + .port(declaredPort) + .host(declaredHost) + .addEndpoints(endpoints) + .start(), + Duration.Inf + ) + } +} + +object TapirMultiServer extends App { NettyFuture.runServer(Tapir.genEndpoints(128)) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index fb4837b997..26b639caa9 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -5,6 +5,7 @@ import io.gatling.core.structure.PopulationBuilder import io.gatling.http.Predef._ import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.util.Random object CommonSimulations { private val userCount = 100 @@ -19,6 +20,44 @@ object CommonSimulations { .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } + + def scenario_post_string(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + val httpProtocol = http.baseUrl(baseUrl) + val execHttpPost = exec( + http(s"HTTP POST /path$routeNumber/4") + .post(s"/path$routeNumber/4") + .body(StringBody(List.fill(256)('x').mkString)) + ) + + scenario(s"Repeatedly invoke POST with string body of route number $routeNumber") + .during(duration.toSeconds.toInt)(execHttpPost) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) + } + + val random = new Random() + + def randomByteArray(size: Int): Array[Byte] = { + val byteArray = new Array[Byte](size) + random.nextBytes(byteArray) + byteArray + } + val constRandomBytes = randomByteArray(262200) + + def scenario_post_bytes_256k(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + val httpProtocol = http.baseUrl(baseUrl) + val execHttpPost = exec( + http(s"HTTP POST /pathFile$routeNumber/4") + .post(s"/pathFile$routeNumber/4") + .body(ByteArrayBody(constRandomBytes)) + .header("Content-Type", "application/octet-stream") + ) + + scenario(s"Repeatedly invoke POST with file body of route number $routeNumber") + .during(duration.toSeconds.toInt)(execHttpPost) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) + } } class OneRouteSimulation extends Simulation { @@ -28,3 +67,11 @@ class OneRouteSimulation extends Simulation { class MultiRouteSimulation extends Simulation { setUp(CommonSimulations.testScenario(1.minute, 127)) } + +class PostStringSimulation extends Simulation { + setUp(CommonSimulations.scenario_post_string(10.seconds, 127)) +} + +class PostBytes256Simulation extends Simulation { + setUp(CommonSimulations.scenario_post_bytes_256k(30.seconds, 122)) +} diff --git a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala index e7c4ca0479..d342334f3b 100644 --- a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala +++ b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala @@ -43,11 +43,13 @@ class FileWriterSubscriber(path: Path) extends PromisingSubscriber[Unit, HttpCon new java.nio.channels.CompletionHandler[Integer, Unit] { override def completed(result: Integer, attachment: Unit): Unit = { position += result + httpContent.release() subscription.request(1) } override def failed(exc: Throwable, attachment: Unit): Unit = { subscription.cancel() + httpContent.release() onError(exc) } } diff --git a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/SimpleSubscriber.scala b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/SimpleSubscriber.scala index 40138b3614..2bb7b70f09 100644 --- a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/SimpleSubscriber.scala +++ b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/SimpleSubscriber.scala @@ -26,11 +26,12 @@ private[netty] class SimpleSubscriber() extends PromisingSubscriber[Array[Byte], override def onNext(content: HttpContent): Unit = { val a = ByteBufUtil.getBytes(content.content()) + content.release() size += a.length chunks.add(a) subscription.request(1) } - + override def onError(t: Throwable): Unit = { chunks.clear() resultBlockingQueue.add(Left(t)) @@ -61,7 +62,7 @@ object SimpleSubscriber { publisher.subscribe(maxBytes.map(max => new LimitedLengthSubscriber(max, subscriber)).getOrElse(subscriber)) subscriber.resultBlocking() match { case Right(result) => result - case Left(e) => throw e + case Left(e) => throw e } } } diff --git a/server/pekko-http-server/src/test/resources/logback.xml b/server/pekko-http-server/src/test/resources/logback.xml new file mode 100644 index 0000000000..db6c80dae7 --- /dev/null +++ b/server/pekko-http-server/src/test/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %date [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala b/server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala new file mode 100644 index 0000000000..e8d71ba26c --- /dev/null +++ b/server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala @@ -0,0 +1,38 @@ +package zio + +import sttp.capabilities +import sttp.capabilities.zio.ZioStreams + +import zio.http.Body +import zio.Trace +import zio.http.MediaType +import zio.stream.ZStream +import zio.http.Boundary + +class LimitedZioBody(delegate: Body, maxBytes: Long) extends Body { + + override def asArray(implicit trace: Trace): Task[Array[Byte]] = { + println(s">>>>>>>>>>>>>>>>>>>> Delegating asArray to ${delegate.getClass().getName}") + delegate.asArray + } + + override def asStream(implicit trace: Trace): ZStream[Any, Throwable, Byte] = { + println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> asStream?") + ZioStreams.limitBytes(delegate.asStream, maxBytes) + } + + override def mediaType: Option[MediaType] = delegate.mediaType + + override def isEmpty: Boolean = delegate.isEmpty + + override def asChunk(implicit trace: Trace): Task[Chunk[Byte]] = delegate.asChunk + + override def contentType(newMediaType: MediaType, newBoundary: Boundary): Body = delegate.contentType(newMediaType, newBoundary) + + override def contentType(newMediaType: MediaType): Body = delegate.contentType(newMediaType) + + override def isComplete: Boolean = delegate.isComplete + + override def boundary: Option[Boundary] = delegate.boundary + +} From 1998a348ad6733e27b2cc7510c91db5efdebd468 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 17:47:36 +0100 Subject: [PATCH 02/53] Dynamic perf test runner --- build.sbt | 21 +++-- .../main/scala/sttp/tapir/perf/Common.scala | 15 +-- .../sttp/tapir/perf/PerfTestServer.scala | 3 - .../scala/sttp/tapir/perf/akka/AkkaHttp.scala | 46 ++++++---- ...mpleGetEndpoints.scala => Endpoints.scala} | 16 +++- .../sttp/tapir/perf/apis/ServerRunner.scala | 11 +++ .../scala/sttp/tapir/perf/http4s/Http4s.scala | 32 +++---- .../sttp/tapir/perf/netty/NettyFuture.scala | 43 --------- .../tapir/perf/netty/future/NettyFuture.scala | 40 ++++++++ .../scala/sttp/tapir/perf/Simulations.scala | 92 +++++++++++++++++-- 10 files changed, 203 insertions(+), 116 deletions(-) delete mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala rename perf-tests/src/main/scala/sttp/tapir/perf/apis/{SimpleGetEndpoints.scala => Endpoints.scala} (76%) create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/apis/ServerRunner.scala delete mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala diff --git a/build.sbt b/build.sbt index 6719fa5b6f..e141ae8b2c 100644 --- a/build.sbt +++ b/build.sbt @@ -350,6 +350,7 @@ lazy val rootProject = (project in file(".")) ideSkipProject := false, generateMimeByExtensionDB := GenerateMimeByExtensionDB() ) + .settings(commands += perfTestCommand) .aggregate(allAggregates: _*) // start a test server before running tests of a client interpreter; this is required both for JS tests run inside a @@ -516,7 +517,18 @@ def genPerfTestTask(servName: String, simName: String): Def.Initialize[Task[Unit (Gatling / testOnly).toTask(s" sttp.tapir.perf.${simName}Simulation").value } -lazy val perfTest = inputKey[Unit]("Run performance test") +val perfTestCommand = Command.args("perf", " ") { (state, args) => + args match { + case Seq(servName, simName) => + // First command + System.setProperty("tapir.perf.serv-name", servName) + Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) + + case _ => + println("Usage: perf ") + state + } +} lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .enablePlugins(GatlingPlugin) @@ -541,13 +553,6 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) fork := true, connectInput := true ) - .settings( - perfTest := { - genPerfTestTask.tupled(perfTestParser.parsed).value - // val (servName, simName) = perfTestParser.parsed - // genPerfTestTask(servName, simName).value - } - ) .settings(akkaHttpVanilla := { (genPerfTestTask("akka.Vanilla", "OneRoute")).value }) .settings(akkaHttpTapir := { (genPerfTestTask("akka.Tapir", "OneRoute")).value }) .settings(akkaHttpVanillaMulti := { (genPerfTestTask("akka.VanillaMulti", "MultiRoute")).value }) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index 6ef45594c5..4df8e58bbf 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,25 +1,12 @@ package sttp.tapir.perf import sttp.monad.MonadError -import sttp.monad.syntax._ -import sttp.tapir.{PublicEndpoint, endpoint, path, stringBody} import scala.io.StdIn import sttp.tapir.server.ServerEndpoint +import sttp.tapir.PublicEndpoint object Common { - def genTapirEndpoint(n: Int): PublicEndpoint[Int, String, String, Any] = endpoint.get - .in("path" + n.toString) - .in(path[Int]("id")) - .errorOut(stringBody) - .out(stringBody) - - def genPostStrTapirEndpoint(n: Int): PublicEndpoint[String, String, String, Any] = endpoint.post - .in("path" + n.toString) - .in(stringBody) - .errorOut(stringBody) - .out(stringBody) - def replyWithStr[F[_]](endpoint: PublicEndpoint[_, _, String, Any])(implicit monad: MonadError[F]): ServerEndpoint[Any, F] = endpoint.serverLogicSuccess[F](_ => monad.eval("ok")) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala b/perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala deleted file mode 100644 index 5a59c97d38..0000000000 --- a/perf-tests/src/main/scala/sttp/tapir/perf/PerfTestServer.scala +++ /dev/null @@ -1,3 +0,0 @@ -package sttp.tapir.perf - -trait PerfTestServer {} diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala index 6aa0e0fd35..6146d4e579 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala @@ -5,10 +5,15 @@ import akka.http.scaladsl.Http import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import sttp.tapir.perf +import sttp.tapir.perf.apis._ import sttp.tapir.perf.Common import sttp.tapir.server.akkahttp.AkkaHttpServerInterpreter import scala.concurrent.{ExecutionContextExecutor, Future} +import sttp.monad.MonadError +import sttp.monad.FutureMonad +import scala.concurrent.ExecutionContext +import cats.effect.IO object Vanilla { val router: Int => Route = (nRoutes: Int) => @@ -23,16 +28,16 @@ object Vanilla { ) } -object Tapir { +object Tapir extends Endpoints { + implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) + + val serverEndpointGens = replyingWithDummyStr[Future](allEndpoints) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList + val router: Int => Route = (nRoutes: Int) => AkkaHttpServerInterpreter()(AkkaHttp.executionContext).toRoute( - (0 to nRoutes) - .map((n: Int) => - perf.Common - .genTapirEndpoint(n) - .serverLogic((id: Int) => Future.successful(Right((id + n).toString)): Future[Either[String, String]]) - ) - .toList + genEndpoints(nRoutes) ) } @@ -40,16 +45,21 @@ object AkkaHttp { implicit val actorSystem: ActorSystem = ActorSystem("akka-http") implicit val executionContext: ExecutionContextExecutor = actorSystem.dispatcher - def runServer(router: Route): Unit = { - Http() - .newServerAt("127.0.0.1", 8080) - .bind(router) - .flatMap((x) => { Common.blockServer(); x.unbind() }) - .onComplete(_ => actorSystem.terminate()) + def runServer(router: Route): IO[ServerRunner.KillSwitch] = { + IO.fromFuture( + IO( + Http() + .newServerAt("127.0.0.1", 8080) + .bind(router) + .map { binding => + IO.fromFuture(IO(binding.unbind().flatMap(_ => actorSystem.terminate()))).void + } + ) + ) } } -object TapirServer extends App { AkkaHttp.runServer(Tapir.router(1)) } -object TapirMultiServer extends App { AkkaHttp.runServer(Tapir.router(128)) } -object VanillaServer extends App { AkkaHttp.runServer(Vanilla.router(1)) } -object VanillaMultiServer extends App { AkkaHttp.runServer(Vanilla.router(128)) } +object TapirServer extends ServerRunner { override def start = AkkaHttp.runServer(Tapir.router(1)) } +object TapirMultiServer extends ServerRunner { override def start = AkkaHttp.runServer(Tapir.router(128)) } +object VanillaServer extends ServerRunner { override def start = AkkaHttp.runServer(Vanilla.router(1)) } +object VanillaMultiServer extends ServerRunner { override def start = AkkaHttp.runServer(Vanilla.router(128)) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala similarity index 76% rename from perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala rename to perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala index ac9bd0b76b..1faa3c9969 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/apis/SimpleGetEndpoints.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala @@ -3,9 +3,8 @@ package sttp.tapir.perf.apis import sttp.tapir._ import sttp.monad.MonadError import sttp.tapir.server.ServerEndpoint -import sttp.tapir.server.model.EndpointExtensions._ -trait SimpleGetEndpoints { +trait Endpoints { type EndpointGen = Int => PublicEndpoint[_, String, String, Any] type ServerEndpointGen[F[_]] = Int => ServerEndpoint[Any, F] @@ -26,16 +25,27 @@ trait SimpleGetEndpoints { .out(stringBody) } + val gen_post_in_bytes_out_string: EndpointGen = { (n: Int) => + endpoint.post + .in("pathBytes" + n.toString) + .in(path[Int]("id")) + .in(byteArrayBody) + .errorOut(stringBody) + .out(stringBody) + } + val gen_post_in_file_out_string: EndpointGen = { (n: Int) => endpoint.post .in("pathFile" + n.toString) .in(path[Int]("id")) .in(fileBody) - .maxRequestBodyLength(300000) .errorOut(stringBody) .out(stringBody) } + val allEndpoints = + List(gen_get_in_string_out_string, gen_post_in_string_out_string, gen_post_in_bytes_out_string, gen_post_in_file_out_string) + def replyingWithDummyStr[F[_]](endpointGens: List[EndpointGen])(implicit monad: MonadError[F] ): Seq[ServerEndpointGen[F]] = diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/apis/ServerRunner.scala b/perf-tests/src/main/scala/sttp/tapir/perf/apis/ServerRunner.scala new file mode 100644 index 0000000000..f2c7b35013 --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/apis/ServerRunner.scala @@ -0,0 +1,11 @@ +package sttp.tapir.perf.apis + +import cats.effect.IO + +trait ServerRunner { + def start: IO[ServerRunner.KillSwitch] +} + +object ServerRunner { + type KillSwitch = IO[Unit] +} diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 2c825233fd..12079f419f 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -1,19 +1,15 @@ package sttp.tapir.perf.http4s import cats.effect._ -import cats.effect.unsafe.implicits.global -import cats.syntax.all._ import org.http4s._ import org.http4s.blaze.server.BlazeServerBuilder import org.http4s.dsl._ import org.http4s.implicits._ import org.http4s.server.Router -import sttp.tapir.perf -import sttp.tapir.perf.Common +import sttp.tapir.perf.apis._ import sttp.tapir.server.http4s.Http4sServerInterpreter import sttp.tapir.integ.cats.effect.CatsMonadError import sttp.monad.MonadError -import sttp.tapir.server.ServerEndpoint object Vanilla { val router: Int => HttpRoutes[IO] = (nRoutes: Int) => @@ -30,13 +26,11 @@ object Vanilla { ) } -object Tapir extends perf.apis.SimpleGetEndpoints { +object Tapir extends Endpoints { implicit val mErr: MonadError[IO] = new CatsMonadError[IO] - val serverEndpointGens = replyingWithDummyStr[IO]( - List(gen_get_in_string_out_string, gen_post_in_string_out_string) - ) + val serverEndpointGens = replyingWithDummyStr[IO](allEndpoints) val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router("/" -> { @@ -46,18 +40,20 @@ object Tapir extends perf.apis.SimpleGetEndpoints { }) } -object Http4s { - def runServer(router: HttpRoutes[IO]): IO[ExitCode] = { +object server { + def runServer(router: HttpRoutes[IO]): IO[ServerRunner.KillSwitch] = BlazeServerBuilder[IO] .bindHttp(8080, "localhost") .withHttpApp(router.orNotFound) .resource - .use(_ => { perf.Common.blockServer(); IO.pure(ExitCode.Success) }) - } + .allocated + .map(_._2) + .map(_.flatTap { _ => + IO.println("Http4s server closed.") + }) } -object TapirServer extends App { Http4s.runServer(Tapir.router(1)).unsafeRunSync() } -object TapirMultiServer extends App { Http4s.runServer(Tapir.router(128)).unsafeRunSync() } -object VanillaServer extends App { Http4s.runServer(Vanilla.router(1)).unsafeRunSync() } -object PostStringServer extends App { Http4s.runServer(Vanilla.router(1)).unsafeRunSync() } -object VanillaMultiServer extends App { Http4s.runServer(Vanilla.router(128)).unsafeRunSync() } +object TapirServer extends ServerRunner { override def start = server.runServer(Tapir.router(1)) } +object TapirMultiServer extends ServerRunner { override def start = server.runServer(Tapir.router(128)) } +object VanillaServer extends ServerRunner { override def start = server.runServer(Vanilla.router(1)) } +object VanillaMultiServer extends ServerRunner { override def start = server.runServer(Vanilla.router(128)) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala deleted file mode 100644 index ed43ab2193..0000000000 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/NettyFuture.scala +++ /dev/null @@ -1,43 +0,0 @@ -package sttp.tapir.perf.netty - -import scala.concurrent.Await -import scala.concurrent.duration.Duration -import sttp.tapir.server.ServerEndpoint -import scala.concurrent.Future -import sttp.tapir.perf.apis.SimpleGetEndpoints -import sttp.monad.MonadError -import sttp.monad.FutureMonad -import scala.concurrent.ExecutionContext -import sttp.tapir.server.netty.{NettyFutureServer, NettyFutureServerBinding} -import ExecutionContext.Implicits.global - -object Tapir extends SimpleGetEndpoints { - - implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) - - val serverEndpointGens = replyingWithDummyStr[Future]( - List(gen_get_in_string_out_string, gen_post_in_string_out_string, gen_post_in_file_out_string) - ) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList -} - -object NettyFuture { - - def runServer(endpoints: List[ServerEndpoint[Any, Future]]): Unit = { - val declaredPort = 8080 - val declaredHost = "0.0.0.0" - // Starting netty server - val serverBinding: NettyFutureServerBinding = - Await.result( - NettyFutureServer() - .port(declaredPort) - .host(declaredHost) - .addEndpoints(endpoints) - .start(), - Duration.Inf - ) - } -} - -object TapirMultiServer extends App { NettyFuture.runServer(Tapir.genEndpoints(128)) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala new file mode 100644 index 0000000000..4a5684e426 --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala @@ -0,0 +1,40 @@ +package sttp.tapir.perf.netty.future + +import sttp.tapir.server.ServerEndpoint +import scala.concurrent.Future +import sttp.tapir.perf.apis._ +import sttp.monad.MonadError +import sttp.monad.FutureMonad +import scala.concurrent.ExecutionContext +import sttp.tapir.server.netty.{NettyFutureServer, NettyFutureServerBinding} +import ExecutionContext.Implicits.global +import cats.effect.IO + +object Tapir extends Endpoints { + + implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) + + val serverEndpointGens = replyingWithDummyStr[Future](allEndpoints) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList +} + +object NettyFuture { + + def runServer(endpoints: List[ServerEndpoint[Any, Future]]): IO[ServerRunner.KillSwitch] = { + val declaredPort = 8080 + val declaredHost = "0.0.0.0" + // Starting netty server + val serverBinding: IO[NettyFutureServerBinding] = + IO.fromFuture(IO(NettyFutureServer() + .port(declaredPort) + .host(declaredHost) + .addEndpoints(endpoints) + .start())) + + serverBinding.map(b => IO.fromFuture(IO(b.stop()))) + } +} + +object TapirServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpoints(1)) } +object TapirMultiServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpoints(128)) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 26b639caa9..6c63a9cf83 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -5,10 +5,15 @@ import io.gatling.core.structure.PopulationBuilder import io.gatling.http.Predef._ import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.reflect.runtime.universe import scala.util.Random +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import sttp.tapir.perf.apis.ServerRunner object CommonSimulations { private val userCount = 100 + private val largeInputSize = 5 * 1024 * 1024 * 1024 private val baseUrl = "http://127.0.0.1:8080" def testScenario(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { @@ -23,10 +28,11 @@ object CommonSimulations { def scenario_post_string(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) + val body = new String(randomAlphanumByteArray(256)) val execHttpPost = exec( http(s"HTTP POST /path$routeNumber/4") .post(s"/path$routeNumber/4") - .body(StringBody(List.fill(256)('x').mkString)) + .body(StringBody(body)) ) scenario(s"Repeatedly invoke POST with string body of route number $routeNumber") @@ -42,9 +48,14 @@ object CommonSimulations { random.nextBytes(byteArray) byteArray } - val constRandomBytes = randomByteArray(262200) - def scenario_post_bytes_256k(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def randomAlphanumByteArray(size: Int): Array[Byte] = + Random.alphanumeric.take(size).map(_.toByte).toArray + + lazy val constRandomBytes = randomByteArray(largeInputSize) + lazy val constRandomAlphanumBytes = randomAlphanumByteArray(largeInputSize) + + def scenario_post_file(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( http(s"HTTP POST /pathFile$routeNumber/4") @@ -58,20 +69,83 @@ object CommonSimulations { .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } + + def scenario_post_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + val httpProtocol = http.baseUrl(baseUrl) + val execHttpPost = exec( + http(s"HTTP POST /pathBytes$routeNumber/4") + .post(s"/pathBytes$routeNumber/4") + .body(ByteArrayBody(constRandomBytes)) + .header("Content-Type", "application/octet-stream") + ) + + scenario(s"Repeatedly invoke POST with file body of route number $routeNumber") + .during(duration.toSeconds.toInt)(execHttpPost) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) + } + + def scenario_post_long_string(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + val httpProtocol = http.baseUrl(baseUrl) + val execHttpPost = exec( + http(s"HTTP POST /path$routeNumber/4") + .post(s"/path$routeNumber/4") + .body(ByteArrayBody(constRandomAlphanumBytes)) + .header("Content-Type", "application/octet-stream") + ) + + scenario(s"Repeatedly invoke POST with file body of route number $routeNumber") + .during(duration.toSeconds.toInt)(execHttpPost) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) + } } -class OneRouteSimulation extends Simulation { - setUp(CommonSimulations.testScenario(1.minute, 0)) +abstract class TapirPerfTestSimulation extends Simulation { + + implicit val ioRuntime: IORuntime = IORuntime.global + val servNameValue = System.getProperty("tapir.perf.serv-name") + if (servNameValue == null) { + println("Missing tapir.perf.serv-name system property, ensure you're running perf tests correctly (see perfTests/README.md)") + sys.exit(-1) + } + + val serverName = s"sttp.tapir.perf.${System.getProperty("tapir.perf.serv-name")}Server" + val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) + val serverStartAction: IO[ServerRunner.KillSwitch] = try { + val moduleSymbol = runtimeMirror.staticModule(serverName) + val moduleMirror = runtimeMirror.reflectModule(moduleSymbol) + val instance: ServerRunner = moduleMirror.instance.asInstanceOf[ServerRunner] + instance.start + } catch { + case _: Throwable => + println(s"ERROR! Could not find object $serverName or it doesn't extend ServerRunner") + sys.exit(-2) + } + var killSwitch: ServerRunner.KillSwitch = IO.unit + + before({ + println("Starting http server...") + killSwitch = serverStartAction.unsafeRunSync() + }) + after({ + println("Shutting down http server ...") + killSwitch.unsafeRunSync() + }) +} + +class OneRouteSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.testScenario(10.seconds, 0)) } class MultiRouteSimulation extends Simulation { - setUp(CommonSimulations.testScenario(1.minute, 127)) + setUp(CommonSimulations.testScenario(1.minute, 0)) } class PostStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_string(10.seconds, 127)) + setUp(CommonSimulations.scenario_post_string(1.minute, 0)) } -class PostBytes256Simulation extends Simulation { - setUp(CommonSimulations.scenario_post_bytes_256k(30.seconds, 122)) +class PostLongStringSimulation extends Simulation { + setUp(CommonSimulations.scenario_post_long_string(1.minute, 1)) } From f2c5d791f716f1adb70e419f6a4d3c9a8f3c0693 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 18:03:29 +0100 Subject: [PATCH 03/53] Restore netty server dependency --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a6fc3c6159..2f06be1451 100644 --- a/build.sbt +++ b/build.sbt @@ -562,7 +562,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .settings(http4sVanillaMulti := { (genPerfTestTask("http4s.VanillaMulti", "MultiRoute")).value }) .settings(http4sTapirMulti := { (genPerfTestTask("http4s.TapirMulti", "MultiRoute")).value }) .jvmPlatform(scalaVersions = List(scala2_13)) - .dependsOn(core, akkaHttpServer, http4sServer) + .dependsOn(core, akkaHttpServer, http4sServer, nettyServer) // integrations From 7347583d0a4eb4f542d6f0798c3a2e4511ebfcae Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 18:18:56 +0100 Subject: [PATCH 04/53] Remove unneeded files --- .../src/test/resources/logback.xml | 12 ------ .../src/main/scala/zio/LimitedZioBody.scala | 38 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 server/pekko-http-server/src/test/resources/logback.xml delete mode 100644 server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala diff --git a/server/pekko-http-server/src/test/resources/logback.xml b/server/pekko-http-server/src/test/resources/logback.xml deleted file mode 100644 index db6c80dae7..0000000000 --- a/server/pekko-http-server/src/test/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - %date [%thread] %-5level %logger{36} - %msg%n - - - - - - - diff --git a/server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala b/server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala deleted file mode 100644 index e8d71ba26c..0000000000 --- a/server/zio-http-server/src/main/scala/zio/LimitedZioBody.scala +++ /dev/null @@ -1,38 +0,0 @@ -package zio - -import sttp.capabilities -import sttp.capabilities.zio.ZioStreams - -import zio.http.Body -import zio.Trace -import zio.http.MediaType -import zio.stream.ZStream -import zio.http.Boundary - -class LimitedZioBody(delegate: Body, maxBytes: Long) extends Body { - - override def asArray(implicit trace: Trace): Task[Array[Byte]] = { - println(s">>>>>>>>>>>>>>>>>>>> Delegating asArray to ${delegate.getClass().getName}") - delegate.asArray - } - - override def asStream(implicit trace: Trace): ZStream[Any, Throwable, Byte] = { - println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> asStream?") - ZioStreams.limitBytes(delegate.asStream, maxBytes) - } - - override def mediaType: Option[MediaType] = delegate.mediaType - - override def isEmpty: Boolean = delegate.isEmpty - - override def asChunk(implicit trace: Trace): Task[Chunk[Byte]] = delegate.asChunk - - override def contentType(newMediaType: MediaType, newBoundary: Boundary): Body = delegate.contentType(newMediaType, newBoundary) - - override def contentType(newMediaType: MediaType): Body = delegate.contentType(newMediaType) - - override def isComplete: Boolean = delegate.isComplete - - override def boundary: Option[Boundary] = delegate.boundary - -} From 67973b1a410eddf8f08cb309523d145b8135c186 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 18:22:45 +0100 Subject: [PATCH 05/53] Remove unneded build utils --- build.sbt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/build.sbt b/build.sbt index 2f06be1451..1e37dd5e98 100644 --- a/build.sbt +++ b/build.sbt @@ -506,12 +506,6 @@ val http4sTapir = taskKey[Unit]("http4s-tapir") val http4sVanillaMulti = taskKey[Unit]("http4s-vanilla-multi") val http4sTapirMulti = taskKey[Unit]("http4s-tapir-multi") -import complete.DefaultParsers._ - -val perfTestParser: complete.Parser[(String, String)] = { - Space ~> token(StringBasic.examples("")) ~ (Space ~> token(StringBasic.examples(""))) -} - def genPerfTestTask(servName: String, simName: String): Def.Initialize[Task[Unit]] = Def.task { (Compile / runMain).toTask(s" sttp.tapir.perf.${servName}Server").value (Gatling / testOnly).toTask(s" sttp.tapir.perf.${simName}Simulation").value From 917dfebfaf690371ba5fe22c7a00416b5224e2ef Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 18:22:58 +0100 Subject: [PATCH 06/53] Apply base simulation to all sims --- .../src/test/scala/sttp/tapir/perf/Simulations.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 6c63a9cf83..1a5593a8ee 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -138,14 +138,14 @@ class OneRouteSimulation extends TapirPerfTestSimulation { setUp(CommonSimulations.testScenario(10.seconds, 0)) } -class MultiRouteSimulation extends Simulation { +class MultiRouteSimulation extends TapirPerfTestSimulation { setUp(CommonSimulations.testScenario(1.minute, 0)) } -class PostStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_string(1.minute, 0)) +class PostStringSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.scenario_post_string(10.seconds, 0)) } -class PostLongStringSimulation extends Simulation { +class PostLongStringSimulation extends TapirPerfTestSimulation { setUp(CommonSimulations.scenario_post_long_string(1.minute, 1)) } From 46955107b2ff6debbff53d31fee763d43c96af4e Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 18:25:44 +0100 Subject: [PATCH 07/53] Remove old way of running perf tests --- build.sbt | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/build.sbt b/build.sbt index 1e37dd5e98..ec34869b2c 100644 --- a/build.sbt +++ b/build.sbt @@ -497,27 +497,11 @@ lazy val tests: ProjectMatrix = (projectMatrix in file("tests")) ) .dependsOn(core, files, circeJson, cats) -val akkaHttpVanilla = taskKey[Unit]("akka-http-vanilla") -val akkaHttpTapir = taskKey[Unit]("akka-http-tapir") -val akkaHttpVanillaMulti = taskKey[Unit]("akka-http-vanilla-multi") -val akkaHttpTapirMulti = taskKey[Unit]("akka-http-tapir-multi") -val http4sVanilla = taskKey[Unit]("http4s-vanilla") -val http4sTapir = taskKey[Unit]("http4s-tapir") -val http4sVanillaMulti = taskKey[Unit]("http4s-vanilla-multi") -val http4sTapirMulti = taskKey[Unit]("http4s-tapir-multi") - -def genPerfTestTask(servName: String, simName: String): Def.Initialize[Task[Unit]] = Def.task { - (Compile / runMain).toTask(s" sttp.tapir.perf.${servName}Server").value - (Gatling / testOnly).toTask(s" sttp.tapir.perf.${simName}Simulation").value -} - val perfTestCommand = Command.args("perf", " ") { (state, args) => args match { case Seq(servName, simName) => - // First command System.setProperty("tapir.perf.serv-name", servName) Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) - case _ => println("Usage: perf ") state @@ -547,14 +531,6 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) fork := true, connectInput := true ) - .settings(akkaHttpVanilla := { (genPerfTestTask("akka.Vanilla", "OneRoute")).value }) - .settings(akkaHttpTapir := { (genPerfTestTask("akka.Tapir", "OneRoute")).value }) - .settings(akkaHttpVanillaMulti := { (genPerfTestTask("akka.VanillaMulti", "MultiRoute")).value }) - .settings(akkaHttpTapirMulti := { (genPerfTestTask("akka.TapirMulti", "MultiRoute")).value }) - .settings(http4sVanilla := { (genPerfTestTask("http4s.Vanilla", "OneRoute")).value }) - .settings(http4sTapir := { (genPerfTestTask("netty.TapirMulti", "PostBytes256")).value }) - .settings(http4sVanillaMulti := { (genPerfTestTask("http4s.VanillaMulti", "MultiRoute")).value }) - .settings(http4sTapirMulti := { (genPerfTestTask("http4s.TapirMulti", "MultiRoute")).value }) .jvmPlatform(scalaVersions = List(scala2_13)) .dependsOn(core, akkaHttpServer, http4sServer, nettyServer) From 7f8110bf7d8bdd25e62a8e4e2cb468f3a7785c8c Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 2 Jan 2024 18:28:33 +0100 Subject: [PATCH 08/53] Fix file size to 5MB --- perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 1a5593a8ee..3015794237 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -13,7 +13,7 @@ import sttp.tapir.perf.apis.ServerRunner object CommonSimulations { private val userCount = 100 - private val largeInputSize = 5 * 1024 * 1024 * 1024 + private val largeInputSize = 5 * 1024 * 1024 private val baseUrl = "http://127.0.0.1:8080" def testScenario(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { From 4342d97fb1d08f3714b0f40600983ac3219d9ccc Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 08:12:38 +0100 Subject: [PATCH 09/53] Explain why a command is used --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index ec34869b2c..7aae0cfcf5 100644 --- a/build.sbt +++ b/build.sbt @@ -501,6 +501,7 @@ val perfTestCommand = Command.args("perf", " ") { (state, arg args match { case Seq(servName, simName) => System.setProperty("tapir.perf.serv-name", servName) + // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) case _ => println("Usage: perf ") From 6a9bc15a4ff66d3af8bfcf8a7e48d99c442c4fca Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 09:12:16 +0100 Subject: [PATCH 10/53] Parameterize active user count --- build.sbt | 21 +++-- .../scala/sttp/tapir/perf/Simulations.scala | 78 ++++++++++++------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/build.sbt b/build.sbt index 7aae0cfcf5..f962f864ea 100644 --- a/build.sbt +++ b/build.sbt @@ -497,17 +497,22 @@ lazy val tests: ProjectMatrix = (projectMatrix in file("tests")) ) .dependsOn(core, files, circeJson, cats) -val perfTestCommand = Command.args("perf", " ") { (state, args) => - args match { - case Seq(servName, simName) => +val perfTestCommand = Command("perf", ("perf", "servName simName userCount(optional)"), "run performance tests") { state => + val parser = (Space ~> token(StringBasic.examples("servName"))) ~ + (Space ~> token(StringBasic.examples("simName"))) ~ + (Space ~> token(StringBasic.examples("userCount"))).? + parser.map { + case servName ~ simName ~ userCountOpt => + val userCount = userCountOpt.map(_.trim).getOrElse("1") + (servName, simName, userCount) + } + } { (state, args) => + val (servName, simName, userCount) = args System.setProperty("tapir.perf.serv-name", servName) + System.setProperty("tapir.perf.user-count", userCount) // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) - case _ => - println("Usage: perf ") - state - } -} + } lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .enablePlugins(GatlingPlugin) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 3015794237..5ecb78041a 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -12,11 +12,23 @@ import cats.effect.unsafe.IORuntime import sttp.tapir.perf.apis.ServerRunner object CommonSimulations { - private val userCount = 100 private val largeInputSize = 5 * 1024 * 1024 private val baseUrl = "http://127.0.0.1:8080" + private val random = new Random() - def testScenario(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def randomByteArray(size: Int): Array[Byte] = { + val byteArray = new Array[Byte](size) + random.nextBytes(byteArray) + byteArray + } + + def randomAlphanumByteArray(size: Int): Array[Byte] = + Random.alphanumeric.take(size).map(_.toByte).toArray + + lazy val constRandomBytes = randomByteArray(largeInputSize) + lazy val constRandomAlphanumBytes = randomAlphanumByteArray(largeInputSize) + + def simple_get(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpGet = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4")) @@ -25,6 +37,16 @@ object CommonSimulations { .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } + + def getParamOpt(paramName: String): Option[String] = Option(System.getProperty(s"tapir.perf.${paramName}")) + + def getParam(paramName: String): String = + getParamOpt(paramName).getOrElse( + throw new IllegalArgumentException(s"Missing tapir.perf.${paramName} system property, ensure you're running perf tests correctly (see perfTests/README.md)") + ) + + private lazy val userCount = getParam("user-count").toInt + // Scenarios def scenario_post_string(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) @@ -35,26 +57,26 @@ object CommonSimulations { .body(StringBody(body)) ) - scenario(s"Repeatedly invoke POST with string body of route number $routeNumber") + scenario(s"Repeatedly invoke POST with short string body") .during(duration.toSeconds.toInt)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) + } + def scenario_post_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + val httpProtocol = http.baseUrl(baseUrl) + val execHttpPost = exec( + http(s"HTTP POST /path$routeNumber/4") + .post(s"/path$routeNumber/4") + .body(ByteArrayBody(randomByteArray(256))) + ) - val random = new Random() - - def randomByteArray(size: Int): Array[Byte] = { - val byteArray = new Array[Byte](size) - random.nextBytes(byteArray) - byteArray + scenario(s"Repeatedly invoke POST with short byte array body") + .during(duration.toSeconds.toInt)(execHttpPost) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) } - def randomAlphanumByteArray(size: Int): Array[Byte] = - Random.alphanumeric.take(size).map(_.toByte).toArray - - lazy val constRandomBytes = randomByteArray(largeInputSize) - lazy val constRandomAlphanumBytes = randomAlphanumByteArray(largeInputSize) - def scenario_post_file(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( @@ -64,13 +86,13 @@ object CommonSimulations { .header("Content-Type", "application/octet-stream") ) - scenario(s"Repeatedly invoke POST with file body of route number $routeNumber") + scenario(s"Repeatedly invoke POST with file body") .during(duration.toSeconds.toInt)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } - def scenario_post_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def scenario_post_long_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( http(s"HTTP POST /pathBytes$routeNumber/4") @@ -79,7 +101,7 @@ object CommonSimulations { .header("Content-Type", "application/octet-stream") ) - scenario(s"Repeatedly invoke POST with file body of route number $routeNumber") + scenario(s"Repeatedly invoke POST with large byte array") .during(duration.toSeconds.toInt)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -94,7 +116,7 @@ object CommonSimulations { .header("Content-Type", "application/octet-stream") ) - scenario(s"Repeatedly invoke POST with file body of route number $routeNumber") + scenario(s"Repeatedly invoke POST with large byte array, interpreted to a String") .during(duration.toSeconds.toInt)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -104,13 +126,9 @@ object CommonSimulations { abstract class TapirPerfTestSimulation extends Simulation { implicit val ioRuntime: IORuntime = IORuntime.global - val servNameValue = System.getProperty("tapir.perf.serv-name") - if (servNameValue == null) { - println("Missing tapir.perf.serv-name system property, ensure you're running perf tests correctly (see perfTests/README.md)") - sys.exit(-1) - } + val serverNameParam = CommonSimulations.getParam("serv-name") + val serverName = s"sttp.tapir.perf.${serverNameParam}Server" - val serverName = s"sttp.tapir.perf.${System.getProperty("tapir.perf.serv-name")}Server" val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) val serverStartAction: IO[ServerRunner.KillSwitch] = try { val moduleSymbol = runtimeMirror.staticModule(serverName) @@ -134,12 +152,12 @@ abstract class TapirPerfTestSimulation extends Simulation { }) } -class OneRouteSimulation extends TapirPerfTestSimulation { - setUp(CommonSimulations.testScenario(10.seconds, 0)) +class SimpleGetSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.simple_get(1.minute, 0)) } -class MultiRouteSimulation extends TapirPerfTestSimulation { - setUp(CommonSimulations.testScenario(1.minute, 0)) +class SimpleGetMultiRouteSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.simple_get(1.minute, 127)) } class PostStringSimulation extends TapirPerfTestSimulation { @@ -147,5 +165,5 @@ class PostStringSimulation extends TapirPerfTestSimulation { } class PostLongStringSimulation extends TapirPerfTestSimulation { - setUp(CommonSimulations.scenario_post_long_string(1.minute, 1)) + setUp(CommonSimulations.scenario_post_long_string(1.minute, 0)) } From 5bcc1774f216427bdac4a913fc07663256e4e8d3 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 09:22:19 +0100 Subject: [PATCH 11/53] Switch from Akka to Pekko --- build.sbt | 2 +- .../AkkaHttp.scala => pekko/PekkoHttp.scala} | 36 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) rename perf-tests/src/main/scala/sttp/tapir/perf/{akka/AkkaHttp.scala => pekko/PekkoHttp.scala} (60%) diff --git a/build.sbt b/build.sbt index f962f864ea..60fefd1dbc 100644 --- a/build.sbt +++ b/build.sbt @@ -538,7 +538,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) connectInput := true ) .jvmPlatform(scalaVersions = List(scala2_13)) - .dependsOn(core, akkaHttpServer, http4sServer, nettyServer) + .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer) // integrations diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala similarity index 60% rename from perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala rename to perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 6146d4e579..97b034855d 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -1,19 +1,15 @@ -package sttp.tapir.perf.akka +package sttp.tapir.perf.pekko -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import sttp.tapir.perf +import cats.effect.IO +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.http.scaladsl.Http +import org.apache.pekko.http.scaladsl.server.Directives._ +import org.apache.pekko.http.scaladsl.server.Route +import sttp.monad.{FutureMonad, MonadError} import sttp.tapir.perf.apis._ -import sttp.tapir.perf.Common -import sttp.tapir.server.akkahttp.AkkaHttpServerInterpreter +import sttp.tapir.server.pekkohttp.PekkoHttpServerInterpreter -import scala.concurrent.{ExecutionContextExecutor, Future} -import sttp.monad.MonadError -import sttp.monad.FutureMonad -import scala.concurrent.ExecutionContext -import cats.effect.IO +import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} object Vanilla { val router: Int => Route = (nRoutes: Int) => @@ -36,13 +32,13 @@ object Tapir extends Endpoints { def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList val router: Int => Route = (nRoutes: Int) => - AkkaHttpServerInterpreter()(AkkaHttp.executionContext).toRoute( + PekkoHttpServerInterpreter()(PekkoHttp.executionContext).toRoute( genEndpoints(nRoutes) ) } -object AkkaHttp { - implicit val actorSystem: ActorSystem = ActorSystem("akka-http") +object PekkoHttp { + implicit val actorSystem: ActorSystem = ActorSystem("tapir-pekko-http") implicit val executionContext: ExecutionContextExecutor = actorSystem.dispatcher def runServer(router: Route): IO[ServerRunner.KillSwitch] = { @@ -59,7 +55,7 @@ object AkkaHttp { } } -object TapirServer extends ServerRunner { override def start = AkkaHttp.runServer(Tapir.router(1)) } -object TapirMultiServer extends ServerRunner { override def start = AkkaHttp.runServer(Tapir.router(128)) } -object VanillaServer extends ServerRunner { override def start = AkkaHttp.runServer(Vanilla.router(1)) } -object VanillaMultiServer extends ServerRunner { override def start = AkkaHttp.runServer(Vanilla.router(128)) } +object TapirServer extends ServerRunner { override def start = PekkoHttp.runServer(Tapir.router(1)) } +object TapirMultiServer extends ServerRunner { override def start = PekkoHttp.runServer(Tapir.router(128)) } +object VanillaServer extends ServerRunner { override def start = PekkoHttp.runServer(Vanilla.router(1)) } +object VanillaMultiServer extends ServerRunner { override def start = PekkoHttp.runServer(Vanilla.router(128)) } From fc414b21030de374bfce2aff5ded4bd488cddebe Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 09:50:30 +0100 Subject: [PATCH 12/53] Add Netty Cats server --- build.sbt | 2 +- .../sttp/tapir/perf/apis/Endpoints.scala | 6 ++-- .../scala/sttp/tapir/perf/http4s/Http4s.scala | 2 +- .../tapir/perf/netty/cats/NettyCats.scala | 34 +++++++++++++++++++ .../tapir/perf/netty/future/NettyFuture.scala | 2 +- .../sttp/tapir/perf/pekko/PekkoHttp.scala | 5 +-- 6 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala diff --git a/build.sbt b/build.sbt index 60fefd1dbc..88df4bcbbb 100644 --- a/build.sbt +++ b/build.sbt @@ -538,7 +538,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) connectInput := true ) .jvmPlatform(scalaVersions = List(scala2_13)) - .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer) + .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer, nettyServerCats) // integrations diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala index 1faa3c9969..7f65e71a08 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala @@ -46,10 +46,8 @@ trait Endpoints { val allEndpoints = List(gen_get_in_string_out_string, gen_post_in_string_out_string, gen_post_in_bytes_out_string, gen_post_in_file_out_string) - def replyingWithDummyStr[F[_]](endpointGens: List[EndpointGen])(implicit - monad: MonadError[F] - ): Seq[ServerEndpointGen[F]] = - endpointGens.map(gen => gen.andThen(se => se.serverLogicSuccess[F](_ => monad.eval("ok")))) + def replyingWithDummyStr[F[_]](endpointGens: List[EndpointGen], reply: String => F[String]): Seq[ServerEndpointGen[F]] = + endpointGens.map(gen => gen.andThen(se => se.serverLogicSuccess[F](_ => reply("ok")))) def genServerEndpoints[F[_]](gens: Seq[ServerEndpointGen[F]])(routeCount: Int): Seq[ServerEndpoint[Any, F]] = gens.flatMap(gen => (0 to routeCount).map(i => gen(i))) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 12079f419f..9211680e7c 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -30,7 +30,7 @@ object Tapir extends Endpoints { implicit val mErr: MonadError[IO] = new CatsMonadError[IO] - val serverEndpointGens = replyingWithDummyStr[IO](allEndpoints) + val serverEndpointGens = replyingWithDummyStr(allEndpoints, IO.pure) val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router("/" -> { diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala new file mode 100644 index 0000000000..076ea578c8 --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala @@ -0,0 +1,34 @@ +package sttp.tapir.perf.netty.cats + +import sttp.tapir.server.ServerEndpoint +import sttp.tapir.perf.apis._ +import sttp.tapir.server.netty.cats.NettyCatsServer +import cats.effect.IO + +object Tapir extends Endpoints { + + val serverEndpointGens = replyingWithDummyStr(allEndpoints, IO.pure) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList +} + +object NettyCats { + + def runServer(endpoints: List[ServerEndpoint[Any, IO]]): IO[ServerRunner.KillSwitch] = { + val declaredPort = 8080 + val declaredHost = "0.0.0.0" + // Starting netty server + NettyCatsServer.io().allocated.flatMap { + case (server, killSwitch) => + server + .port(declaredPort) + .host(declaredHost) + .addEndpoints(endpoints) + .start() + .map(_ => killSwitch) + } + } +} + +object TapirServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpoints(1)) } +object TapirMultiServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpoints(128)) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala index 4a5684e426..826798a56f 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala @@ -14,7 +14,7 @@ object Tapir extends Endpoints { implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) - val serverEndpointGens = replyingWithDummyStr[Future](allEndpoints) + val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 97b034855d..607015766b 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -5,7 +5,6 @@ import org.apache.pekko.actor.ActorSystem import org.apache.pekko.http.scaladsl.Http import org.apache.pekko.http.scaladsl.server.Directives._ import org.apache.pekko.http.scaladsl.server.Route -import sttp.monad.{FutureMonad, MonadError} import sttp.tapir.perf.apis._ import sttp.tapir.server.pekkohttp.PekkoHttpServerInterpreter @@ -25,9 +24,7 @@ object Vanilla { } object Tapir extends Endpoints { - implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) - - val serverEndpointGens = replyingWithDummyStr[Future](allEndpoints) + val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList From 881f2c374b3cefee167d2a8095f52653a293de0b Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 12:03:41 +0100 Subject: [PATCH 13/53] Add Play server --- build.sbt | 2 +- .../scala/sttp/tapir/perf/play/Play.scala | 66 +++++++++++++++++++ .../scala/sttp/tapir/perf/Simulations.scala | 3 +- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala diff --git a/build.sbt b/build.sbt index 88df4bcbbb..1316b5c958 100644 --- a/build.sbt +++ b/build.sbt @@ -538,7 +538,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) connectInput := true ) .jvmPlatform(scalaVersions = List(scala2_13)) - .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer, nettyServerCats) + .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer, nettyServerCats, playServer) // integrations diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala new file mode 100644 index 0000000000..d964498164 --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -0,0 +1,66 @@ +package sttp.tapir.perf.play + +import cats.effect.IO +import org.apache.pekko.actor.ActorSystem +import play.api.Mode +import play.api.mvc.{Handler, PlayBodyParsers, RequestHeader} +import play.api.routing.Router +import play.api.routing.Router.Routes +import play.core.server.{DefaultPekkoHttpServerComponents, ServerConfig} +import sttp.tapir.server.play.PlayServerInterpreter +import sttp.tapir.perf.apis._ + +import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} + +object Vanilla { + // val router: Int => Route = (nRoutes: Int) => + // concat( + // (0 to nRoutes).map((n: Int) => + // get { + // path(("path" + n.toString) / IntNumber) { id => + // complete((n + id).toString) + // } + // } + // ): _* + // ) +} + +object Tapir extends Endpoints { + val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList + import Play._ + + val router: Int => Routes = (nRoutes: Int) => + PlayServerInterpreter().toRoutes( + genEndpoints(nRoutes) + ) +} + +object Play { + implicit lazy val perfActorSystem: ActorSystem = ActorSystem("tapir-play") + implicit lazy val executionContext: ExecutionContextExecutor = perfActorSystem.dispatcher + + def runServer(routes: Routes): IO[ServerRunner.KillSwitch] = { + val components = new DefaultPekkoHttpServerComponents { + val initialServerConfig = ServerConfig(port = Some(8080), address = "127.0.0.1", mode = Mode.Test) + override lazy val actorSystem: ActorSystem = perfActorSystem + override def router: Router = + Router.from( + List(routes).reduce((a: Routes, b: Routes) => { + val handler: PartialFunction[RequestHeader, Handler] = { case request => + a.applyOrElse(request, b) + } + + handler + }) + ) + } + IO(components.server).map(server => IO(server.stop())) + } +} + +object TapirServer extends ServerRunner { override def start = Play.runServer(Tapir.router(1)) } +object TapirMultiServer extends ServerRunner { override def start = Play.runServer(Tapir.router(128)) } +// object VanillaServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(1)) } +// object VanillaMultiServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(128)) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 5ecb78041a..271292e40c 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -136,7 +136,8 @@ abstract class TapirPerfTestSimulation extends Simulation { val instance: ServerRunner = moduleMirror.instance.asInstanceOf[ServerRunner] instance.start } catch { - case _: Throwable => + case e: Throwable => + e.printStackTrace() println(s"ERROR! Could not find object $serverName or it doesn't extend ServerRunner") sys.exit(-2) } From 5ec74846085aa7e1a0076bf54c6977464240768e Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 14:53:07 +0100 Subject: [PATCH 14/53] Vanilla Play server skeleton --- build.sbt | 40 ++++++++-------- .../scala/sttp/tapir/perf/play/Play.scala | 47 ++++++++++++------- 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/build.sbt b/build.sbt index c9f2b7f05f..530e1876fb 100644 --- a/build.sbt +++ b/build.sbt @@ -499,20 +499,19 @@ lazy val tests: ProjectMatrix = (projectMatrix in file("tests")) val perfTestCommand = Command("perf", ("perf", "servName simName userCount(optional)"), "run performance tests") { state => val parser = (Space ~> token(StringBasic.examples("servName"))) ~ - (Space ~> token(StringBasic.examples("simName"))) ~ - (Space ~> token(StringBasic.examples("userCount"))).? - parser.map { - case servName ~ simName ~ userCountOpt => - val userCount = userCountOpt.map(_.trim).getOrElse("1") - (servName, simName, userCount) - } - } { (state, args) => - val (servName, simName, userCount) = args - System.setProperty("tapir.perf.serv-name", servName) - System.setProperty("tapir.perf.user-count", userCount) - // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") - Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) - } + (Space ~> token(StringBasic.examples("simName"))) ~ + (Space ~> token(StringBasic.examples("userCount"))).? + parser.map { case servName ~ simName ~ userCountOpt => + val userCount = userCountOpt.map(_.trim).getOrElse("1") + (servName, simName, userCount) + } +} { (state, args) => + val (servName, simName, userCount) = args + System.setProperty("tapir.perf.serv-name", servName) + System.setProperty("tapir.perf.user-count", userCount) + // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") + Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) +} lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .enablePlugins(GatlingPlugin) @@ -520,14 +519,15 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .settings( name := "tapir-perf-tests", libraryDependencies ++= Seq( - "io.gatling.highcharts" % "gatling-charts-highcharts" % "3.10.3" % "test", - "io.gatling" % "gatling-test-framework" % "3.10.3" % "test", - "com.typesafe.akka" %% "akka-http" % Versions.akkaHttp, - "com.typesafe.akka" %% "akka-stream" % Versions.akkaStreams, - "org.http4s" %% "http4s-blaze-server" % Versions.http4sBlazeServer, - "org.http4s" %% "http4s-server" % Versions.http4s, + // Required to force newer jackson in Pekko, a version that is compatible with Gatling's Jackson dependency + "io.gatling.highcharts" % "gatling-charts-highcharts" % "3.10.3" % "test" exclude( + "com.fasterxml.jackson.core", "jackson-databind" + ), + "io.gatling" % "gatling-test-framework" % "3.10.3" % "test" exclude ("com.fasterxml.jackson.core", "jackson-databind"), + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.1", "org.http4s" %% "http4s-core" % Versions.http4s, "org.http4s" %% "http4s-dsl" % Versions.http4s, + "org.http4s" %% "http4s-blaze-server" % Versions.http4sBlazeServer, "org.typelevel" %%% "cats-effect" % Versions.catsEffect ) ++ loggerDependencies, publishArtifact := false diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index d964498164..c96985bfc0 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -3,26 +3,41 @@ package sttp.tapir.perf.play import cats.effect.IO import org.apache.pekko.actor.ActorSystem import play.api.Mode -import play.api.mvc.{Handler, PlayBodyParsers, RequestHeader} +import play.api.mvc.{Handler, PlayBodyParsers, RequestHeader, _} import play.api.routing.Router import play.api.routing.Router.Routes +import play.api.routing.sird._ import play.core.server.{DefaultPekkoHttpServerComponents, ServerConfig} -import sttp.tapir.server.play.PlayServerInterpreter +import play.mvc.Http.HttpVerbs import sttp.tapir.perf.apis._ +import sttp.tapir.server.play.PlayServerInterpreter import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} -object Vanilla { - // val router: Int => Route = (nRoutes: Int) => - // concat( - // (0 to nRoutes).map((n: Int) => - // get { - // path(("path" + n.toString) / IntNumber) { id => - // complete((n + id).toString) - // } - // } - // ): _* - // ) +object Vanilla extends Results { + implicit lazy val perfActorSystem: ActorSystem = ActorSystem("vanilla-play") + implicit lazy val perfExecutionContext: ExecutionContextExecutor = perfActorSystem.dispatcher + val verbs = new HttpVerbs {} + val pcc = new ActionBuilder[Request, AnyContent] { + + override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = + block(request) + + override protected def executionContext: ExecutionContext = perfExecutionContext + + override val parser: BodyParser[AnyContent] = PlayBodyParsers.apply().anyContent + } + def simpleGet: Action[AnyContent] = pcc.async { implicit request => + val param = request.path.split("/").last + Future.successful( + Ok(param) + ) + } + + def genRoutesSingle(number: Int): Routes = { case GET(p"/path$number/$param") => + simpleGet + } + def router: Int => Routes = (nRoutes: Int) => (0 until nRoutes).map(genRoutesSingle).reduceLeft(_ orElse _) } object Tapir extends Endpoints { @@ -43,7 +58,7 @@ object Play { def runServer(routes: Routes): IO[ServerRunner.KillSwitch] = { val components = new DefaultPekkoHttpServerComponents { - val initialServerConfig = ServerConfig(port = Some(8080), address = "127.0.0.1", mode = Mode.Test) + override lazy val serverConfig: ServerConfig = ServerConfig(port = Some(8080), address = "127.0.0.1", mode = Mode.Test) override lazy val actorSystem: ActorSystem = perfActorSystem override def router: Router = Router.from( @@ -62,5 +77,5 @@ object Play { object TapirServer extends ServerRunner { override def start = Play.runServer(Tapir.router(1)) } object TapirMultiServer extends ServerRunner { override def start = Play.runServer(Tapir.router(128)) } -// object VanillaServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(1)) } -// object VanillaMultiServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(128)) } +object VanillaServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(1)) } +object VanillaMultiServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(128)) } From 987ccd1d4986a7e2f40ab430af9d62a1c7db8816 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 8 Jan 2024 16:06:18 +0100 Subject: [PATCH 15/53] Finish Play --- .../main/scala/sttp/tapir/perf/Common.scala | 16 +---- .../sttp/tapir/perf/apis/Endpoints.scala | 6 +- .../scala/sttp/tapir/perf/play/Play.scala | 72 ++++++++++++++++--- .../scala/sttp/tapir/perf/Simulations.scala | 30 +++++--- 4 files changed, 92 insertions(+), 32 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index 4df8e58bbf..a1f2497a2b 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,19 +1,5 @@ package sttp.tapir.perf -import sttp.monad.MonadError - -import scala.io.StdIn -import sttp.tapir.server.ServerEndpoint -import sttp.tapir.PublicEndpoint - object Common { - def replyWithStr[F[_]](endpoint: PublicEndpoint[_, _, String, Any])(implicit monad: MonadError[F]): ServerEndpoint[Any, F] = - endpoint.serverLogicSuccess[F](_ => monad.eval("ok")) - - - def blockServer(): Unit = { - println(Console.BLUE + "Server now online. Please navigate to http://localhost:8080/path0/1\nPress RETURN to stop..." + Console.RESET) - StdIn.readLine() - println("Server terminated") - } + val LargeInputSize = 5 * 1024 * 1024 } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala index 7f65e71a08..4553dd05d9 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala @@ -1,8 +1,9 @@ package sttp.tapir.perf.apis import sttp.tapir._ -import sttp.monad.MonadError +import sttp.tapir.perf.Common._ import sttp.tapir.server.ServerEndpoint +import sttp.tapir.server.model.EndpointExtensions._ trait Endpoints { type EndpointGen = Int => PublicEndpoint[_, String, String, Any] @@ -21,6 +22,7 @@ trait Endpoints { .in("path" + n.toString) .in(path[Int]("id")) .in(stringBody) + .maxRequestBodyLength(LargeInputSize + 1024L) .errorOut(stringBody) .out(stringBody) } @@ -30,6 +32,7 @@ trait Endpoints { .in("pathBytes" + n.toString) .in(path[Int]("id")) .in(byteArrayBody) + .maxRequestBodyLength(LargeInputSize + 1024L) .errorOut(stringBody) .out(stringBody) } @@ -39,6 +42,7 @@ trait Endpoints { .in("pathFile" + n.toString) .in(path[Int]("id")) .in(fileBody) + .maxRequestBodyLength(LargeInputSize + 1024L) .errorOut(stringBody) .out(stringBody) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index c96985bfc0..905b471f25 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -3,22 +3,24 @@ package sttp.tapir.perf.play import cats.effect.IO import org.apache.pekko.actor.ActorSystem import play.api.Mode -import play.api.mvc.{Handler, PlayBodyParsers, RequestHeader, _} +import play.api.mvc._ import play.api.routing.Router import play.api.routing.Router.Routes import play.api.routing.sird._ import play.core.server.{DefaultPekkoHttpServerComponents, ServerConfig} -import play.mvc.Http.HttpVerbs +import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ + import sttp.tapir.server.play.PlayServerInterpreter import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} +import org.apache.pekko.util.ByteString +import play.api.libs.Files -object Vanilla extends Results { +object Vanilla extends ControllerHelpers { implicit lazy val perfActorSystem: ActorSystem = ActorSystem("vanilla-play") implicit lazy val perfExecutionContext: ExecutionContextExecutor = perfActorSystem.dispatcher - val verbs = new HttpVerbs {} - val pcc = new ActionBuilder[Request, AnyContent] { + val anyContentActionBuilder = new ActionBuilder[Request, AnyContent] { override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = block(request) @@ -27,15 +29,69 @@ object Vanilla extends Results { override val parser: BodyParser[AnyContent] = PlayBodyParsers.apply().anyContent } - def simpleGet: Action[AnyContent] = pcc.async { implicit request => + + val byteStringActionBuilder = new ActionBuilder[Request, ByteString] { + override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = + block(request) + + override protected def executionContext: ExecutionContext = perfExecutionContext + + override val parser: BodyParser[ByteString] = PlayBodyParsers.apply().byteString(maxLength = LargeInputSize + 1024L) + } + + val stringActionBuilder = new ActionBuilder[Request, String] { + override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = + block(request) + + override protected def executionContext: ExecutionContext = perfExecutionContext + + override val parser: BodyParser[String] = PlayBodyParsers.apply().text(maxLength = LargeInputSize + 1024L) + } + + val fileReqActionBuilder = new ActionBuilder[Request, Files.TemporaryFile] { + override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = + block(request) + + override protected def executionContext: ExecutionContext = perfExecutionContext + + override val parser: BodyParser[Files.TemporaryFile] = PlayBodyParsers.apply().temporaryFile + } + + def simpleGet: Action[AnyContent] = anyContentActionBuilder.async { implicit request => val param = request.path.split("/").last Future.successful( Ok(param) ) } - def genRoutesSingle(number: Int): Routes = { case GET(p"/path$number/$param") => - simpleGet + def postBytes: Action[ByteString] = byteStringActionBuilder.async { implicit request => + val param = request.path.split("/").last + val byteArray: ByteString = request.body + Future.successful(Ok(s"$param-${byteArray.length}")) + } + + def postString: Action[String] = stringActionBuilder.async { implicit request => + val param = request.path.split("/").last + val str: String = request.body + Future.successful(Ok(s"$param-${str.length}")) + } + + def postFile: Action[Files.TemporaryFile] = fileReqActionBuilder.async { implicit request => + val param = request.path.split("/").last + val file: Files.TemporaryFile = request.body + Future.successful(Ok(s"$param-${file.path.toString}")) + } + + + def genRoutesSingle(number: Int): Routes = { + case GET(p"/path$number/$param") => + simpleGet + case POST(p"/path$number/$param") => + postString + case POST(p"/pathBytes$number/$param") => + postBytes + case POST(p"/pathFile$number/$param") => + postFile } def router: Int => Routes = (nRoutes: Int) => (0 until nRoutes).map(genRoutesSingle).reduceLeft(_ orElse _) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 271292e40c..b6e9a94236 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -10,9 +10,9 @@ import scala.util.Random import cats.effect.IO import cats.effect.unsafe.IORuntime import sttp.tapir.perf.apis.ServerRunner +import sttp.tapir.perf.Common._ object CommonSimulations { - private val largeInputSize = 5 * 1024 * 1024 private val baseUrl = "http://127.0.0.1:8080" private val random = new Random() @@ -25,8 +25,8 @@ object CommonSimulations { def randomAlphanumByteArray(size: Int): Array[Byte] = Random.alphanumeric.take(size).map(_.toByte).toArray - lazy val constRandomBytes = randomByteArray(largeInputSize) - lazy val constRandomAlphanumBytes = randomAlphanumByteArray(largeInputSize) + lazy val constRandomBytes = randomByteArray(LargeInputSize) + lazy val constRandomAlphanumBytes = randomAlphanumByteArray(LargeInputSize) def simple_get(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) @@ -55,6 +55,7 @@ object CommonSimulations { http(s"HTTP POST /path$routeNumber/4") .post(s"/path$routeNumber/4") .body(StringBody(body)) + .header("Content-Type", "text/plain") ) scenario(s"Repeatedly invoke POST with short string body") @@ -66,8 +67,8 @@ object CommonSimulations { def scenario_post_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( - http(s"HTTP POST /path$routeNumber/4") - .post(s"/path$routeNumber/4") + http(s"HTTP POST /pathBytes$routeNumber/4") + .post(s"/pathBytes$routeNumber/4") .body(ByteArrayBody(randomByteArray(256))) ) @@ -130,6 +131,7 @@ abstract class TapirPerfTestSimulation extends Simulation { val serverName = s"sttp.tapir.perf.${serverNameParam}Server" val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) + // io.gatling.core.json.Json val serverStartAction: IO[ServerRunner.KillSwitch] = try { val moduleSymbol = runtimeMirror.staticModule(serverName) val moduleMirror = runtimeMirror.reflectModule(moduleSymbol) @@ -154,11 +156,23 @@ abstract class TapirPerfTestSimulation extends Simulation { } class SimpleGetSimulation extends TapirPerfTestSimulation { - setUp(CommonSimulations.simple_get(1.minute, 0)) + setUp(CommonSimulations.simple_get(10.seconds, 0)) } class SimpleGetMultiRouteSimulation extends TapirPerfTestSimulation { - setUp(CommonSimulations.simple_get(1.minute, 127)) + setUp(CommonSimulations.simple_get(10.seconds, 127)) +} + +class PostBytesSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.scenario_post_bytes(10.seconds, 0)) +} + +class PostLongBytesSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.scenario_post_long_bytes(10.seconds, 0)) +} + +class PostFilesSimulation extends TapirPerfTestSimulation { + setUp(CommonSimulations.scenario_post_file(10.seconds, 0)) } class PostStringSimulation extends TapirPerfTestSimulation { @@ -166,5 +180,5 @@ class PostStringSimulation extends TapirPerfTestSimulation { } class PostLongStringSimulation extends TapirPerfTestSimulation { - setUp(CommonSimulations.scenario_post_long_string(1.minute, 0)) + setUp(CommonSimulations.scenario_post_long_string(10.seconds, 0)) } From dc71db9a7fc0c7e588e9b1c1c6203036a8abaac1 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 9 Jan 2024 08:38:01 +0100 Subject: [PATCH 16/53] Refator vanilla Play --- .../scala/sttp/tapir/perf/play/Play.scala | 56 +++++-------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index 905b471f25..097bd70fb6 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -2,7 +2,9 @@ package sttp.tapir.perf.play import cats.effect.IO import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.util.ByteString import play.api.Mode +import play.api.libs.Files import play.api.mvc._ import play.api.routing.Router import play.api.routing.Router.Routes @@ -10,80 +12,50 @@ import play.api.routing.sird._ import play.core.server.{DefaultPekkoHttpServerComponents, ServerConfig} import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ - import sttp.tapir.server.play.PlayServerInterpreter import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} -import org.apache.pekko.util.ByteString -import play.api.libs.Files -object Vanilla extends ControllerHelpers { +object Vanilla extends ControllerHelpers { implicit lazy val perfActorSystem: ActorSystem = ActorSystem("vanilla-play") implicit lazy val perfExecutionContext: ExecutionContextExecutor = perfActorSystem.dispatcher - val anyContentActionBuilder = new ActionBuilder[Request, AnyContent] { - - override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = - block(request) - - override protected def executionContext: ExecutionContext = perfExecutionContext - - override val parser: BodyParser[AnyContent] = PlayBodyParsers.apply().anyContent - } - - val byteStringActionBuilder = new ActionBuilder[Request, ByteString] { - override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = - block(request) - - override protected def executionContext: ExecutionContext = perfExecutionContext - - override val parser: BodyParser[ByteString] = PlayBodyParsers.apply().byteString(maxLength = LargeInputSize + 1024L) - } - - val stringActionBuilder = new ActionBuilder[Request, String] { - override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = - block(request) - - override protected def executionContext: ExecutionContext = perfExecutionContext + def actionBuilder[T](parserParam: BodyParser[T]): ActionBuilder[Request, T] = new ActionBuilder[Request, T] { - override val parser: BodyParser[String] = PlayBodyParsers.apply().text(maxLength = LargeInputSize + 1024L) - } - - val fileReqActionBuilder = new ActionBuilder[Request, Files.TemporaryFile] { override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = block(request) override protected def executionContext: ExecutionContext = perfExecutionContext - override val parser: BodyParser[Files.TemporaryFile] = PlayBodyParsers.apply().temporaryFile + override val parser: BodyParser[T] = parserParam } - def simpleGet: Action[AnyContent] = anyContentActionBuilder.async { implicit request => + val simpleGet: Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { implicit request => val param = request.path.split("/").last Future.successful( Ok(param) ) } - def postBytes: Action[ByteString] = byteStringActionBuilder.async { implicit request => - val param = request.path.split("/").last - val byteArray: ByteString = request.body - Future.successful(Ok(s"$param-${byteArray.length}")) + val postBytes: Action[ByteString] = actionBuilder(PlayBodyParsers().byteString(maxLength = LargeInputSize + 1024L)).async { + implicit request => + val param = request.path.split("/").last + val byteArray: ByteString = request.body + Future.successful(Ok(s"$param-${byteArray.length}")) } - def postString: Action[String] = stringActionBuilder.async { implicit request => + val postString: Action[String] = actionBuilder(PlayBodyParsers().text(maxLength = LargeInputSize + 1024L)).async { implicit request => val param = request.path.split("/").last val str: String = request.body Future.successful(Ok(s"$param-${str.length}")) } - def postFile: Action[Files.TemporaryFile] = fileReqActionBuilder.async { implicit request => + val postFile: Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers.apply().temporaryFile).async { implicit request => val param = request.path.split("/").last val file: Files.TemporaryFile = request.body Future.successful(Ok(s"$param-${file.path.toString}")) } - - def genRoutesSingle(number: Int): Routes = { + def genRoutesSingle(number: Int): Routes = { case GET(p"/path$number/$param") => simpleGet case POST(p"/path$number/$param") => From 01f91bcdfac003a99b2ab157597846c67c3d1d22 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 10 Jan 2024 13:45:03 +0100 Subject: [PATCH 17/53] Use new ActorSystem for each Pekko/Play server run --- .../sttp/tapir/perf/pekko/PekkoHttp.scala | 17 +-- .../scala/sttp/tapir/perf/play/Play.scala | 107 ++++++++++-------- 2 files changed, 67 insertions(+), 57 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 607015766b..21a7994dba 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -9,9 +9,10 @@ import sttp.tapir.perf.apis._ import sttp.tapir.server.pekkohttp.PekkoHttpServerInterpreter import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} +import org.apache.pekko.stream.ActorMaterializer object Vanilla { - val router: Int => Route = (nRoutes: Int) => + val router: Int => ActorSystem => Route = (nRoutes: Int) => (_: ActorSystem) => concat( (0 to nRoutes).map((n: Int) => get { @@ -28,22 +29,22 @@ object Tapir extends Endpoints { def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - val router: Int => Route = (nRoutes: Int) => - PekkoHttpServerInterpreter()(PekkoHttp.executionContext).toRoute( + def router: Int => ActorSystem => Route = (nRoutes: Int) => (actorSystem: ActorSystem) => + PekkoHttpServerInterpreter()(actorSystem.dispatcher).toRoute( genEndpoints(nRoutes) ) } object PekkoHttp { - implicit val actorSystem: ActorSystem = ActorSystem("tapir-pekko-http") - implicit val executionContext: ExecutionContextExecutor = actorSystem.dispatcher - - def runServer(router: Route): IO[ServerRunner.KillSwitch] = { + def runServer(router: ActorSystem => Route): IO[ServerRunner.KillSwitch] = { + // We need to create a new actor system each time server is run + implicit val actorSystem: ActorSystem = ActorSystem("tapir-pekko-http") + implicit val executionContext: ExecutionContextExecutor = actorSystem.dispatcher IO.fromFuture( IO( Http() .newServerAt("127.0.0.1", 8080) - .bind(router) + .bind(router(actorSystem)) .map { binding => IO.fromFuture(IO(binding.unbind().flatMap(_ => actorSystem.terminate()))).void } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index 097bd70fb6..fdc6bb13b6 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -15,82 +15,91 @@ import sttp.tapir.perf.apis._ import sttp.tapir.server.play.PlayServerInterpreter import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.ActorMaterializer object Vanilla extends ControllerHelpers { - implicit lazy val perfActorSystem: ActorSystem = ActorSystem("vanilla-play") - implicit lazy val perfExecutionContext: ExecutionContextExecutor = perfActorSystem.dispatcher - def actionBuilder[T](parserParam: BodyParser[T]): ActionBuilder[Request, T] = new ActionBuilder[Request, T] { - override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = - block(request) + def genRoutesSingle(actorSystem: ActorSystem)(number: Int): Routes = { - override protected def executionContext: ExecutionContext = perfExecutionContext + def actionBuilder[T](parserParam: BodyParser[T]): ActionBuilder[Request, T] = + new ActionBuilder[Request, T] { - override val parser: BodyParser[T] = parserParam - } + override def invokeBlock[A](request: Request[A], block: Request[A] => Future[Result]): Future[Result] = + block(request) - val simpleGet: Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { implicit request => - val param = request.path.split("/").last - Future.successful( - Ok(param) - ) - } + override protected def executionContext: ExecutionContext = actorSystem.dispatcher - val postBytes: Action[ByteString] = actionBuilder(PlayBodyParsers().byteString(maxLength = LargeInputSize + 1024L)).async { - implicit request => - val param = request.path.split("/").last - val byteArray: ByteString = request.body - Future.successful(Ok(s"$param-${byteArray.length}")) - } + override val parser: BodyParser[T] = parserParam + } - val postString: Action[String] = actionBuilder(PlayBodyParsers().text(maxLength = LargeInputSize + 1024L)).async { implicit request => - val param = request.path.split("/").last - val str: String = request.body - Future.successful(Ok(s"$param-${str.length}")) - } + implicit val actorSystemForMaterializer: ActorSystem = actorSystem - val postFile: Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers.apply().temporaryFile).async { implicit request => - val param = request.path.split("/").last - val file: Files.TemporaryFile = request.body - Future.successful(Ok(s"$param-${file.path.toString}")) - } + val simpleGet: Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { + implicit request => + val param = request.path.split("/").last + Future.successful( + Ok(param) + ) + } + + val postBytes: Action[ByteString] = + actionBuilder(PlayBodyParsers().byteString(maxLength = LargeInputSize + 1024L)).async { implicit request => + val param = request.path.split("/").last + val byteArray: ByteString = request.body + Future.successful(Ok(s"$param-${byteArray.length}")) + } - def genRoutesSingle(number: Int): Routes = { - case GET(p"/path$number/$param") => - simpleGet - case POST(p"/path$number/$param") => - postString - case POST(p"/pathBytes$number/$param") => - postBytes - case POST(p"/pathFile$number/$param") => - postFile + val postString: Action[String] = actionBuilder(PlayBodyParsers().text(maxLength = LargeInputSize + 1024L)).async { implicit request => + val param = request.path.split("/").last + val str: String = request.body + Future.successful(Ok(s"$param-${str.length}")) + } + + val postFile: Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers.apply().temporaryFile).async { implicit request => + val param = request.path.split("/").last + val file: Files.TemporaryFile = request.body + Future.successful(Ok(s"$param-${file.path.toString}")) + } + + { + case GET(p"/path$number/$param") => + simpleGet + case POST(p"/path$number/$param") => + postString + case POST(p"/pathBytes$number/$param") => + postBytes + case POST(p"/pathFile$number/$param") => + postFile + } } - def router: Int => Routes = (nRoutes: Int) => (0 until nRoutes).map(genRoutesSingle).reduceLeft(_ orElse _) + def router: Int => ActorSystem => Routes = (nRoutes: Int) => (actorSystem: ActorSystem) => (0 until nRoutes).map(genRoutesSingle(actorSystem)).reduceLeft(_ orElse _) } object Tapir extends Endpoints { val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - import Play._ - val router: Int => Routes = (nRoutes: Int) => - PlayServerInterpreter().toRoutes( - genEndpoints(nRoutes) - ) + val router: Int => ActorSystem => Routes = (nRoutes: Int) => + (actorSystem: ActorSystem) => { + implicit val actorSystemForMaterializer: ActorSystem = actorSystem + PlayServerInterpreter().toRoutes( + genEndpoints(nRoutes) + ) + } } object Play { - implicit lazy val perfActorSystem: ActorSystem = ActorSystem("tapir-play") - implicit lazy val executionContext: ExecutionContextExecutor = perfActorSystem.dispatcher - def runServer(routes: Routes): IO[ServerRunner.KillSwitch] = { + def runServer(routes: ActorSystem => Routes): IO[ServerRunner.KillSwitch] = { + implicit lazy val perfActorSystem: ActorSystem = ActorSystem(s"tapir-play") val components = new DefaultPekkoHttpServerComponents { override lazy val serverConfig: ServerConfig = ServerConfig(port = Some(8080), address = "127.0.0.1", mode = Mode.Test) override lazy val actorSystem: ActorSystem = perfActorSystem override def router: Router = Router.from( - List(routes).reduce((a: Routes, b: Routes) => { + List(routes(actorSystem)).reduce((a: Routes, b: Routes) => { val handler: PartialFunction[RequestHeader, Handler] = { case request => a.applyOrElse(request, b) } From 2aa228ab222e2e6116a6bc62a56ff927c4b2df94 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Thu, 11 Jan 2024 11:21:21 +0100 Subject: [PATCH 18/53] Initial suite runner and log processor --- build.sbt | 12 ++- .../sttp/tapir/perf/GatlingLogProcessor.scala | 93 +++++++++++++++++++ .../scala/sttp/tapir/perf/GatlingRunner.scala | 19 ++++ .../sttp/tapir/perf/PerfTestSuiteResult.scala | 15 +++ .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 70 ++++++++++++++ .../scala/sttp/tapir/perf/Simulations.scala | 45 ++------- project/Versions.scala | 1 + 7 files changed, 215 insertions(+), 40 deletions(-) create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteResult.scala create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala diff --git a/build.sbt b/build.sbt index 530e1876fb..2acc1198c4 100644 --- a/build.sbt +++ b/build.sbt @@ -509,8 +509,15 @@ val perfTestCommand = Command("perf", ("perf", "servName simName userCount(optio val (servName, simName, userCount) = args System.setProperty("tapir.perf.serv-name", servName) System.setProperty("tapir.perf.user-count", userCount) + val customSimId = s"$simName-$servName-${System.currentTimeMillis}" // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") - Command.process(s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state) + val state2= Command.process(s"perfTests/clean", state) + Command.process( + s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", + state2 + ) + // read reports from last directory into report aggregator + // generate aggregated report } lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) @@ -520,11 +527,12 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) name := "tapir-perf-tests", libraryDependencies ++= Seq( // Required to force newer jackson in Pekko, a version that is compatible with Gatling's Jackson dependency - "io.gatling.highcharts" % "gatling-charts-highcharts" % "3.10.3" % "test" exclude( + "io.gatling.highcharts" % "gatling-charts-highcharts" % "3.10.3" % "test" exclude ( "com.fasterxml.jackson.core", "jackson-databind" ), "io.gatling" % "gatling-test-framework" % "3.10.3" % "test" exclude ("com.fasterxml.jackson.core", "jackson-databind"), "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.1", + "nl.grons" %% "metrics4-scala" % Versions.metrics4Scala, "org.http4s" %% "http4s-core" % Versions.http4s, "org.http4s" %% "http4s-dsl" % Versions.http4s, "org.http4s" %% "http4s-blaze-server" % Versions.http4sBlazeServer, diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala new file mode 100644 index 0000000000..2e422fb661 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala @@ -0,0 +1,93 @@ +package sttp.tapir.perf + +import cats.effect.IO +import cats.syntax.all._ +import com.codahale.metrics.{Histogram, MetricRegistry} +import fs2.io.file.{Files => Fs2Files} +import fs2.text + +import java.nio.file.{Files, Path, Paths} +import java.util.stream.Collectors +import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Try} + +/** Reads all entries from Gatling simulation.log file and calculates mean throughput as well as p99, p95, p75 and p50 latencies. + */ +object GatlingLogProcessor { + + val LogFileName = "simulation.log" + + /** + * Searches for the last modified simulation.log in all simulation logs and calculates results. + */ + def processLast(simulationName: String, serverName: String): IO[GatlingSimulationResult] = { + for { + lastLogPath <- IO.fromTry(findLastLogFile) + _ <- IO.println(s"Processing results from $lastLogPath") + result <- Fs2Files[IO] + .readAll(fs2.io.file.Path.fromNioPath(lastLogPath)) + .through(text.utf8.decode) + .through(text.lines) + .fold[State](State.initial) { (state, line) => + val parts = line.split("\\s+") + if (parts.length > 1 && parts(0) == "REQUEST") { + val requestStartTime = parts(4).toLong + val minRequestTs = state.minRequestTs.min(requestStartTime) + val requestEndTime = parts(5).toLong + val maxResponseTs = state.maxResponseTs.max(requestEndTime) + val reqDuration = requestEndTime - requestStartTime + state.histogram.update(reqDuration) + State(state.histogram, minRequestTs, maxResponseTs) + } else state + } + .compile + .lastOrError + .ensure(new IllegalStateException(s"Could not read results from $lastLogPath"))(_.totalDurationMs != State.initial.totalDurationMs) + .map { state => + val snapshot = state.histogram.getSnapshot + val throughput = (state.histogram.getCount().toDouble / state.totalDurationMs) * 1000 + GatlingSimulationResult( + simulationName, + serverName, + state.totalDurationMs.millis, + meanReqsPerSec = throughput.toLong, + latencyP99 = snapshot.get99thPercentile, + latencyP95 = snapshot.get95thPercentile, + latencyP75 = snapshot.get75thPercentile, + latencyP50 = snapshot.getMedian + ) + } + } yield result + } + + private def findLastLogFile: Try[Path] = { + val baseDir = System.getProperty("user.dir") + println(s"Base dir = $baseDir") + val resultsDir: Path = Paths.get(baseDir).resolve("results") + Try { + findAllSimulationLogs(resultsDir).maxBy(p => Files.getLastModifiedTime(p)) + }.recoverWith { case err => + Failure(new IllegalStateException(s"Could not resolve last ${LogFileName} in ${resultsDir}", err)) + } + } + + private def findAllSimulationLogs(basePath: Path): List[Path] = { + Try { + Files + .walk(basePath) + .filter(path => Files.isRegularFile(path) && path.getFileName.toString == LogFileName) + .collect(Collectors.toList[Path]) + .asScala + .toList + }.getOrElse(List.empty[Path]) + } + + case class State(histogram: Histogram, minRequestTs: Long, maxResponseTs: Long) { + def totalDurationMs: Long = maxResponseTs - minRequestTs + } + + object State { + def initial: State = State(new MetricRegistry().histogram("tapir"), Long.MaxValue, 1L) + } +} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala new file mode 100644 index 0000000000..472d12d220 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala @@ -0,0 +1,19 @@ +package sttp.tapir.perf + +import io.gatling.app.Gatling +import io.gatling.core.config.GatlingPropertiesBuilder +import scala.reflect.runtime.universe + +object GatlingRunner { + + /** Blocking, runs the entire Gatling simulation. + */ + def runSimulationBlocking(simulationClassName: String): Int = { + System.setProperty("tapir.perf.user-count", "1") // TODO from config + val props = new GatlingPropertiesBuilder() + .simulationClass(simulationClassName) + .build + + Gatling.fromMap(props) + } +} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteResult.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteResult.scala new file mode 100644 index 0000000000..2c6cdaeee9 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteResult.scala @@ -0,0 +1,15 @@ +package sttp.tapir.perf + +import scala.concurrent.duration.FiniteDuration + +case class GatlingSimulationResult( + simulationName: String, + serverName: String, + duration: FiniteDuration, + meanReqsPerSec: Long, + latencyP99: Double, + latencyP95: Double, + latencyP75: Double, + latencyP50: Double +) + diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala new file mode 100644 index 0000000000..c1eec021c2 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -0,0 +1,70 @@ +package sttp.tapir.perf + +import cats.effect.{ExitCode, IO, IOApp} +import cats.syntax.all._ +import sttp.tapir.perf.apis.ServerRunner + +import scala.reflect.runtime.universe + +/** Main entry point for running suites of performance tests and generating aggregated reports. A suite represents a set of Gatling + * simulations executed on a set of servers, with some additional parameters like concurrent user count. One can run a single simulation on + * a single server, as well as a selection of (servers x simulations). The runner then collects Gatling logs from simulation.log files of + * individual simulation runs and puts them together into an aggregated report comparing results for all the runs. + */ +object PerfTestSuiteRunner extends IOApp { + + val arraySplitPattern = "\\s*[,]+\\s*".r + val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) + + def run(args: List[String]): IO[ExitCode] = { + if (args.length < 2) { + println("Usage: perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner serverName simulationName userCount") + println("Where serverName and simulationName can be a single name, an array of comma separated names, or '*' for all") + println("The userCount parameter is optional (default value is 1)") + sys.exit(1) + } + val shortServerNames = argToList(args(0)) + val shortSimulationNames = argToList(args(1)) + + val serverNames = shortServerNames.map(s => s"sttp.tapir.perf.${s}Server") + val simulationNames = shortSimulationNames.map(s => s"sttp.tapir.perf.${s}Simulation") + + println(serverNames) + println(simulationNames) + // TODO Parse user count + // TODO add comprehensive help + val flatResults: IO[List[GatlingSimulationResult]] = + ((simulationNames, serverNames).mapN((x, y) => (x, y))).traverse { case (simulationName, serverName) => + for { + serverKillSwitch <- startServerByTypeName(serverName) + _ <- IO + .blocking(GatlingRunner.runSimulationBlocking(simulationName)) + .guarantee(serverKillSwitch) + .ensureOr(errCode => new Exception(s"Gatling failed with code $errCode"))(_ == 0) + serverSimulationResult <- GatlingLogProcessor.processLast(simulationName, serverName) + _ <- IO.println(serverSimulationResult) + } yield (serverSimulationResult) + } + + flatResults.as(ExitCode.Success) + } + + private def argToList(arg: String): List[String] = + if (arg.startsWith("[") && arg.endsWith("]")) + arraySplitPattern.split(arg.drop(1).dropRight(1)).toList + else + List(arg) + + private def startServerByTypeName(serverName: String): IO[ServerRunner.KillSwitch] = { + try { + val moduleSymbol = runtimeMirror.staticModule(serverName) + val moduleMirror = runtimeMirror.reflectModule(moduleSymbol) + val instance: ServerRunner = moduleMirror.instance.asInstanceOf[ServerRunner] + instance.start + } catch { + case e: Throwable => + IO.raiseError(new IllegalArgumentException(s"ERROR! Could not find object $serverName or it doesn't extend ServerRunner", e)) + } + } + +} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index b6e9a94236..6b7326a768 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -124,61 +124,30 @@ object CommonSimulations { } } -abstract class TapirPerfTestSimulation extends Simulation { - - implicit val ioRuntime: IORuntime = IORuntime.global - val serverNameParam = CommonSimulations.getParam("serv-name") - val serverName = s"sttp.tapir.perf.${serverNameParam}Server" - - val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) - // io.gatling.core.json.Json - val serverStartAction: IO[ServerRunner.KillSwitch] = try { - val moduleSymbol = runtimeMirror.staticModule(serverName) - val moduleMirror = runtimeMirror.reflectModule(moduleSymbol) - val instance: ServerRunner = moduleMirror.instance.asInstanceOf[ServerRunner] - instance.start - } catch { - case e: Throwable => - e.printStackTrace() - println(s"ERROR! Could not find object $serverName or it doesn't extend ServerRunner") - sys.exit(-2) - } - var killSwitch: ServerRunner.KillSwitch = IO.unit - - before({ - println("Starting http server...") - killSwitch = serverStartAction.unsafeRunSync() - }) - after({ - println("Shutting down http server ...") - killSwitch.unsafeRunSync() - }) -} - -class SimpleGetSimulation extends TapirPerfTestSimulation { +class SimpleGetSimulation extends Simulation { setUp(CommonSimulations.simple_get(10.seconds, 0)) } -class SimpleGetMultiRouteSimulation extends TapirPerfTestSimulation { +class SimpleGetMultiRouteSimulation extends Simulation { setUp(CommonSimulations.simple_get(10.seconds, 127)) } -class PostBytesSimulation extends TapirPerfTestSimulation { +class PostBytesSimulation extends Simulation { setUp(CommonSimulations.scenario_post_bytes(10.seconds, 0)) } -class PostLongBytesSimulation extends TapirPerfTestSimulation { +class PostLongBytesSimulation extends Simulation { setUp(CommonSimulations.scenario_post_long_bytes(10.seconds, 0)) } -class PostFilesSimulation extends TapirPerfTestSimulation { +class PostFilesSimulation extends Simulation { setUp(CommonSimulations.scenario_post_file(10.seconds, 0)) } -class PostStringSimulation extends TapirPerfTestSimulation { +class PostStringSimulation extends Simulation { setUp(CommonSimulations.scenario_post_string(10.seconds, 0)) } -class PostLongStringSimulation extends TapirPerfTestSimulation { +class PostLongStringSimulation extends Simulation { setUp(CommonSimulations.scenario_post_long_string(10.seconds, 0)) } diff --git a/project/Versions.scala b/project/Versions.scala index bc67531aad..25b8375947 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -23,6 +23,7 @@ object Versions { val finatra = "23.11.0" val catbird = "21.12.0" val json4s = "4.0.7" + val metrics4Scala = "4.2.9" val nettyReactiveStreams = "3.0.2" val sprayJson = "1.3.6" val scalaCheck = "1.17.0" From eda178b263dc13aca7a3b7b4ca6ecf3862f7780d Mon Sep 17 00:00:00 2001 From: kciesielski Date: Thu, 11 Jan 2024 15:28:31 +0100 Subject: [PATCH 19/53] Save HTML report --- build.sbt | 3 +- .../sttp/tapir/perf/GatlingLogProcessor.scala | 2 +- .../sttp/tapir/perf/HtmlResultsPrinter.scala | 43 +++++++++++ .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 73 +++++++++++++++---- .../scala/sttp/tapir/perf/Simulations.scala | 2 +- project/Versions.scala | 1 + 6 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala diff --git a/build.sbt b/build.sbt index 2acc1198c4..8700991ce9 100644 --- a/build.sbt +++ b/build.sbt @@ -532,7 +532,8 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) ), "io.gatling" % "gatling-test-framework" % "3.10.3" % "test" exclude ("com.fasterxml.jackson.core", "jackson-databind"), "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.1", - "nl.grons" %% "metrics4-scala" % Versions.metrics4Scala, + "nl.grons" %% "metrics4-scala" % Versions.metrics4Scala % Test, + "com.lihaoyi" %% "scalatags" % Versions.scalaTags % Test, "org.http4s" %% "http4s-core" % Versions.http4s, "org.http4s" %% "http4s-dsl" % Versions.http4s, "org.http4s" %% "http4s-blaze-server" % Versions.http4sBlazeServer, diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala index 2e422fb661..9c17081765 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala @@ -31,7 +31,7 @@ object GatlingLogProcessor { .through(text.lines) .fold[State](State.initial) { (state, line) => val parts = line.split("\\s+") - if (parts.length > 1 && parts(0) == "REQUEST") { + if (parts.length >= 5 && parts(0) == "REQUEST") { val requestStartTime = parts(4).toLong val minRequestTs = state.minRequestTs.min(requestStartTime) val requestEndTime = parts(5).toLong diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala new file mode 100644 index 0000000000..0acec46b08 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala @@ -0,0 +1,43 @@ +package sttp.tapir.perf + +import scalatags.Text.all._ +import scalatags.Text + +object HtmlResultsPrinter { + val tableStyle = "border-collapse: collapse;" + val cellStyle = "border: 1px solid black; padding: 5px;" + + def print(results: List[GatlingSimulationResult]): String = { + + val headers = "Simulation" :: results.groupBy(_.simulationName).head._2.map(_.serverName).map(stripPkg) + createHtmlTable(headers, results.groupBy(_.simulationName).values.toList) + } + + private def createHtmlTable(headers: Seq[String], rows: List[List[GatlingSimulationResult]]): String = { + + table(style := tableStyle)( + thead( + tr(headers.map(header => th(header, style := cellStyle))) + ), + tbody( + for (row <- rows) yield { + tr(td(stripPkg(row.head.simulationName)) :: row.map(toColumn), style := cellStyle) + } + ) + ).render + } + + private def toColumn(result: GatlingSimulationResult): Text.TypedTag[String] = + td( + Seq( + p(s"reqs/sec = ${result.meanReqsPerSec}"), + p(s"p99 latency = ${result.latencyP99}"), + p(s"p95 latency = ${result.latencyP95}"), + p(s"p75 latency = ${result.latencyP75}"), + p(s"p50 latency = ${result.latencyP50}") + ), + style := cellStyle + ) + + private def stripPkg(className: String): String = className.split("\\.").last +} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index c1eec021c2..ece3aa439d 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -5,6 +5,10 @@ import cats.syntax.all._ import sttp.tapir.perf.apis.ServerRunner import scala.reflect.runtime.universe +import scala.util.control.NonFatal +import java.time.format.DateTimeFormatter +import java.time.LocalDateTime +import java.nio.file.Paths /** Main entry point for running suites of performance tests and generating aggregated reports. A suite represents a set of Gatling * simulations executed on a set of servers, with some additional parameters like concurrent user count. One can run a single simulation on @@ -17,24 +21,22 @@ object PerfTestSuiteRunner extends IOApp { val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) def run(args: List[String]): IO[ExitCode] = { - if (args.length < 2) { - println("Usage: perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner serverName simulationName userCount") - println("Where serverName and simulationName can be a single name, an array of comma separated names, or '*' for all") - println("The userCount parameter is optional (default value is 1)") - sys.exit(1) - } + if (args.length < 2) + exitOnIncorrectArgs val shortServerNames = argToList(args(0)) val shortSimulationNames = argToList(args(1)) + if (shortSimulationNames.isEmpty || shortServerNames.isEmpty) + exitOnIncorrectArgs val serverNames = shortServerNames.map(s => s"sttp.tapir.perf.${s}Server") val simulationNames = shortSimulationNames.map(s => s"sttp.tapir.perf.${s}Simulation") - println(serverNames) - println(simulationNames) + // TODO ensure servers and simulations exist // TODO Parse user count // TODO add comprehensive help - val flatResults: IO[List[GatlingSimulationResult]] = - ((simulationNames, serverNames).mapN((x, y) => (x, y))).traverse { case (simulationName, serverName) => + ((simulationNames, serverNames) + .mapN((x, y) => (x, y))) + .traverse { case (simulationName, serverName) => for { serverKillSwitch <- startServerByTypeName(serverName) _ <- IO @@ -45,15 +47,22 @@ object PerfTestSuiteRunner extends IOApp { _ <- IO.println(serverSimulationResult) } yield (serverSimulationResult) } - - flatResults.as(ExitCode.Success) + .flatTap(writeJsonReport) + .flatTap(writeHtmlReport) + .as(ExitCode.Success) } private def argToList(arg: String): List[String] = - if (arg.startsWith("[") && arg.endsWith("]")) - arraySplitPattern.split(arg.drop(1).dropRight(1)).toList - else - List(arg) + try { + if (arg.startsWith("[") && arg.endsWith("]")) + arraySplitPattern.split(arg.drop(1).dropRight(1)).toList + else + List(arg) + } catch { + case NonFatal(e) => + e.printStackTrace() + exitOnIncorrectArgs + } private def startServerByTypeName(serverName: String): IO[ServerRunner.KillSwitch] = { try { @@ -67,4 +76,36 @@ object PerfTestSuiteRunner extends IOApp { } } + private def writeJsonReport(results: List[GatlingSimulationResult]): IO[Unit] = { + // TODO + IO.unit + } + + private def writeHtmlReport(results: List[GatlingSimulationResult]): IO[Unit] = { + val html = HtmlResultsPrinter.print(results) + writeStringReport(html, "html") + } + + private def writeStringReport(report: String, extension: String): IO[Unit] = { + import fs2.io.file + import fs2.text + + val baseDir = System.getProperty("user.dir") + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss") + val currentTime = LocalDateTime.now().format(formatter) + val targetFilePath = Paths.get(baseDir).resolve(s"tapir-perf-tests-${currentTime}.$extension") + fs2.Stream + .emit(report) + .through(text.utf8.encode) + .through(file.Files[IO].writeAll(fs2.io.file.Path.fromNioPath(targetFilePath))) + .compile + .drain >> IO.println(s"******* Test Suite report saved to $targetFilePath") + } + + private def exitOnIncorrectArgs = { + println("Usage: perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner serverName simulationName userCount") + println("Where serverName and simulationName can be a single name, an array of comma separated names, or '*' for all") + println("The userCount parameter is optional (default value is 1)") + sys.exit(1) + } } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 6b7326a768..f26df61488 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -140,7 +140,7 @@ class PostLongBytesSimulation extends Simulation { setUp(CommonSimulations.scenario_post_long_bytes(10.seconds, 0)) } -class PostFilesSimulation extends Simulation { +class PostFileSimulation extends Simulation { setUp(CommonSimulations.scenario_post_file(10.seconds, 0)) } diff --git a/project/Versions.scala b/project/Versions.scala index 25b8375947..f4561c7a7e 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -28,6 +28,7 @@ object Versions { val sprayJson = "1.3.6" val scalaCheck = "1.17.0" val scalaTest = "3.2.17" + val scalaTags = "0.12.0" val scalaTestPlusScalaCheck = "3.2.17.0" val refined = "0.11.0" val iron = "2.4.0" From 13291a0fad5fd8b8ebb89be9403273c9bdf5a073 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Thu, 11 Jan 2024 20:40:15 +0100 Subject: [PATCH 20/53] CSV reported and inmprovements to HTML reporter --- .../sttp/tapir/perf/CsvResultsPrinter.scala | 31 +++++++++++++++++++ .../sttp/tapir/perf/HtmlResultsPrinter.scala | 6 ++-- .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 29 +++++++++-------- 3 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala new file mode 100644 index 0000000000..644e556840 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala @@ -0,0 +1,31 @@ +package sttp.tapir.perf + +object CsvResultsPrinter { + def print(results: List[GatlingSimulationResult]): String = { + + val groupedResults = results.groupBy(_.simulationName) + val headers = "Simulation" :: groupedResults.values.head.flatMap(r => { + val server = r.serverName + List( + s"$server-ops/sec", + s"$server-latency-p99", + s"$server-latency-p95", + s"$server-latency-p75", + s"$server-latency-p50" + ) + }) + val rows: List[String] = groupedResults.toList.map { case (simName, serverResults) => + (simName :: serverResults.flatMap(singleResult => + List( + singleResult.meanReqsPerSec.toString, + singleResult.latencyP99.toString, + singleResult.latencyP95.toString, + singleResult.latencyP75.toString, + singleResult.latencyP75.toString + ) + )).mkString(",") + + } + (headers.mkString(",") :: rows).mkString("\n") + } +} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala index 0acec46b08..73a21dc565 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala @@ -9,7 +9,7 @@ object HtmlResultsPrinter { def print(results: List[GatlingSimulationResult]): String = { - val headers = "Simulation" :: results.groupBy(_.simulationName).head._2.map(_.serverName).map(stripPkg) + val headers = "Simulation" :: results.groupBy(_.simulationName).head._2.map(_.serverName) createHtmlTable(headers, results.groupBy(_.simulationName).values.toList) } @@ -21,7 +21,7 @@ object HtmlResultsPrinter { ), tbody( for (row <- rows) yield { - tr(td(stripPkg(row.head.simulationName)) :: row.map(toColumn), style := cellStyle) + tr(td(row.head.simulationName) :: row.map(toColumn), style := cellStyle) } ) ).render @@ -38,6 +38,4 @@ object HtmlResultsPrinter { ), style := cellStyle ) - - private def stripPkg(className: String): String = className.split("\\.").last } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index ece3aa439d..146af021ab 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -2,13 +2,15 @@ package sttp.tapir.perf import cats.effect.{ExitCode, IO, IOApp} import cats.syntax.all._ +import fs2.io.file +import fs2.text import sttp.tapir.perf.apis.ServerRunner +import java.nio.file.Paths +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter import scala.reflect.runtime.universe import scala.util.control.NonFatal -import java.time.format.DateTimeFormatter -import java.time.LocalDateTime -import java.nio.file.Paths /** Main entry point for running suites of performance tests and generating aggregated reports. A suite represents a set of Gatling * simulations executed on a set of servers, with some additional parameters like concurrent user count. One can run a single simulation on @@ -34,20 +36,20 @@ object PerfTestSuiteRunner extends IOApp { // TODO ensure servers and simulations exist // TODO Parse user count // TODO add comprehensive help - ((simulationNames, serverNames) + ((simulationNames.zip(shortSimulationNames), (serverNames.zip(shortServerNames))) .mapN((x, y) => (x, y))) - .traverse { case (simulationName, serverName) => + .traverse { case ((simulationName, shortSimulationName), (serverName, shortServerName)) => for { serverKillSwitch <- startServerByTypeName(serverName) _ <- IO .blocking(GatlingRunner.runSimulationBlocking(simulationName)) .guarantee(serverKillSwitch) .ensureOr(errCode => new Exception(s"Gatling failed with code $errCode"))(_ == 0) - serverSimulationResult <- GatlingLogProcessor.processLast(simulationName, serverName) + serverSimulationResult <- GatlingLogProcessor.processLast(shortSimulationName, shortServerName) _ <- IO.println(serverSimulationResult) } yield (serverSimulationResult) } - .flatTap(writeJsonReport) + .flatTap(writeCsvReport) .flatTap(writeHtmlReport) .as(ExitCode.Success) } @@ -76,20 +78,17 @@ object PerfTestSuiteRunner extends IOApp { } } - private def writeJsonReport(results: List[GatlingSimulationResult]): IO[Unit] = { - // TODO - IO.unit + private def writeCsvReport(results: List[GatlingSimulationResult]): IO[Unit] = { + val csv = CsvResultsPrinter.print(results) + writeReportFile(csv, "csv") } private def writeHtmlReport(results: List[GatlingSimulationResult]): IO[Unit] = { val html = HtmlResultsPrinter.print(results) - writeStringReport(html, "html") + writeReportFile(html, "html") } - private def writeStringReport(report: String, extension: String): IO[Unit] = { - import fs2.io.file - import fs2.text - + private def writeReportFile(report: String, extension: String): IO[Unit] = { val baseDir = System.getProperty("user.dir") val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss") val currentTime = LocalDateTime.now().format(formatter) From 50a191df7b640c2172564572427b6d8d17d37d85 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Thu, 11 Jan 2024 20:44:33 +0100 Subject: [PATCH 21/53] Ensure same filename for all reports --- .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index 146af021ab..0eb166afdf 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -33,6 +33,8 @@ object PerfTestSuiteRunner extends IOApp { val serverNames = shortServerNames.map(s => s"sttp.tapir.perf.${s}Server") val simulationNames = shortSimulationNames.map(s => s"sttp.tapir.perf.${s}Simulation") + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss") + val currentTime = LocalDateTime.now().format(formatter) // TODO ensure servers and simulations exist // TODO Parse user count // TODO add comprehensive help @@ -49,8 +51,8 @@ object PerfTestSuiteRunner extends IOApp { _ <- IO.println(serverSimulationResult) } yield (serverSimulationResult) } - .flatTap(writeCsvReport) - .flatTap(writeHtmlReport) + .flatTap(writeCsvReport(currentTime)) + .flatTap(writeHtmlReport(currentTime)) .as(ExitCode.Success) } @@ -78,20 +80,18 @@ object PerfTestSuiteRunner extends IOApp { } } - private def writeCsvReport(results: List[GatlingSimulationResult]): IO[Unit] = { + private def writeCsvReport(currentTime: String)(results: List[GatlingSimulationResult]): IO[Unit] = { val csv = CsvResultsPrinter.print(results) - writeReportFile(csv, "csv") + writeReportFile(csv, "csv", currentTime) } - private def writeHtmlReport(results: List[GatlingSimulationResult]): IO[Unit] = { + private def writeHtmlReport(currentTime: String)(results: List[GatlingSimulationResult]): IO[Unit] = { val html = HtmlResultsPrinter.print(results) - writeReportFile(html, "html") + writeReportFile(html, "html", currentTime) } - private def writeReportFile(report: String, extension: String): IO[Unit] = { + private def writeReportFile(report: String, extension: String, currentTime: String): IO[Unit] = { val baseDir = System.getProperty("user.dir") - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss") - val currentTime = LocalDateTime.now().format(formatter) val targetFilePath = Paths.get(baseDir).resolve(s"tapir-perf-tests-${currentTime}.$extension") fs2.Stream .emit(report) From 1449769ce17dad89e65e4b93641d0848030014f7 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Thu, 11 Jan 2024 21:32:54 +0100 Subject: [PATCH 22/53] Allow running all servers or all simulations with '*' --- build.sbt | 3 +- .../main/scala/sttp/tapir/perf/Common.scala | 1 + .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 42 ++++++++++-------- .../scala/sttp/tapir/perf/TypeScanner.scala | 43 +++++++++++++++++++ 4 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala diff --git a/build.sbt b/build.sbt index 8700991ce9..2c7433d1a5 100644 --- a/build.sbt +++ b/build.sbt @@ -511,7 +511,7 @@ val perfTestCommand = Command("perf", ("perf", "servName simName userCount(optio System.setProperty("tapir.perf.user-count", userCount) val customSimId = s"$simName-$servName-${System.currentTimeMillis}" // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") - val state2= Command.process(s"perfTests/clean", state) + val state2 = Command.process(s"perfTests/clean", state) Command.process( s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", state2 @@ -534,6 +534,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.1", "nl.grons" %% "metrics4-scala" % Versions.metrics4Scala % Test, "com.lihaoyi" %% "scalatags" % Versions.scalaTags % Test, + "io.github.classgraph" % "classgraph" % "4.8.165" % Test, "org.http4s" %% "http4s-core" % Versions.http4s, "org.http4s" %% "http4s-dsl" % Versions.http4s, "org.http4s" %% "http4s-blaze-server" % Versions.http4sBlazeServer, diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index a1f2497a2b..d9ccdf31c1 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,5 +1,6 @@ package sttp.tapir.perf object Common { + val rootPackage = "sttp.tapir.perf" val LargeInputSize = 5 * 1024 * 1024 } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index 0eb166afdf..cddf1ba466 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -5,7 +5,7 @@ import cats.syntax.all._ import fs2.io.file import fs2.text import sttp.tapir.perf.apis.ServerRunner - +import Common._ import java.nio.file.Paths import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -25,14 +25,15 @@ object PerfTestSuiteRunner extends IOApp { def run(args: List[String]): IO[ExitCode] = { if (args.length < 2) exitOnIncorrectArgs - val shortServerNames = argToList(args(0)) - val shortSimulationNames = argToList(args(1)) + val shortServerNames = argToList(args(0), ifAll = TypeScanner.allServers) + val shortSimulationNames = argToList(args(1), ifAll = TypeScanner.allSimulations) + println(shortServerNames) + println(shortSimulationNames) if (shortSimulationNames.isEmpty || shortServerNames.isEmpty) exitOnIncorrectArgs - val serverNames = shortServerNames.map(s => s"sttp.tapir.perf.${s}Server") - val simulationNames = shortSimulationNames.map(s => s"sttp.tapir.perf.${s}Simulation") - + val serverNames = shortServerNames.map(s => s"${rootPackage}.${s}Server") + val simulationNames = shortSimulationNames.map(s => s"${rootPackage}.${s}Simulation") val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss") val currentTime = LocalDateTime.now().format(formatter) // TODO ensure servers and simulations exist @@ -56,17 +57,22 @@ object PerfTestSuiteRunner extends IOApp { .as(ExitCode.Success) } - private def argToList(arg: String): List[String] = - try { - if (arg.startsWith("[") && arg.endsWith("]")) - arraySplitPattern.split(arg.drop(1).dropRight(1)).toList - else - List(arg) - } catch { - case NonFatal(e) => - e.printStackTrace() - exitOnIncorrectArgs - } + private def argToList(arg: String, ifAll: List[String]): List[String] = { + println(s"------------$arg") + if (arg.trim == "*") + ifAll + else + try { + if (arg.startsWith("[") && arg.endsWith("]")) + arraySplitPattern.split(arg.drop(1).dropRight(1)).toList + else + List(arg) + } catch { + case NonFatal(e) => + e.printStackTrace() + exitOnIncorrectArgs + } + } private def startServerByTypeName(serverName: String): IO[ServerRunner.KillSwitch] = { try { @@ -102,7 +108,7 @@ object PerfTestSuiteRunner extends IOApp { } private def exitOnIncorrectArgs = { - println("Usage: perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner serverName simulationName userCount") + println(s"Usage: perfTests/Test/runMain ${rootPackage}.PerfTestSuiteRunner serverName simulationName userCount") println("Where serverName and simulationName can be a single name, an array of comma separated names, or '*' for all") println("The userCount parameter is optional (default value is 1)") sys.exit(1) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala new file mode 100644 index 0000000000..433aa80546 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala @@ -0,0 +1,43 @@ +package sttp.tapir.perf + +import io.github.classgraph.ClassGraph +import scala.reflect.ClassTag +import scala.jdk.CollectionConverters._ +import sttp.tapir.perf.apis.ServerRunner +import Common._ +import io.gatling.core.scenario.Simulation + +/** + * Uses the classgraph library to quickly find all possible server runners (objects extending ServerRunner) or + * simulations (classes extending Simulation). + */ +object TypeScanner { + def findAllImplementations[T: ClassTag](rootPackage: String): List[Class[_]] = { + val superClass = scala.reflect.classTag[T].runtimeClass + + val scanResult = new ClassGraph() + .enableClassInfo() + .acceptPackages(rootPackage) + .scan() + + try { + val classes = if (superClass.isInterface) + scanResult.getClassesImplementing(superClass.getName) + else + scanResult.getSubclasses(superClass.getName) + classes.loadClasses().asScala.toList + } finally { + scanResult.close() + } + } + + def allServers: List[String] = + findAllImplementations[ServerRunner](rootPackage) + .map(_.getName) + .map(c => c.stripPrefix(s"${rootPackage}.").stripSuffix("Server$")) + + def allSimulations: List[String] = + findAllImplementations[Simulation](rootPackage) + .map(_.getName) + .map(c => c.stripPrefix(s"${rootPackage}.").stripSuffix("Simulation")) +} From 9f44abf3bfb3e813e8386878064031bd5804f9cc Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 15 Jan 2024 15:36:51 +0100 Subject: [PATCH 23/53] Parse arguments --- build.sbt | 1 + .../sttp/tapir/perf/PerfTestSuiteParams.scala | 80 +++++++++++++++++++ .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 53 +++--------- .../scala/sttp/tapir/perf/Simulations.scala | 51 ++++++------ .../scala/sttp/tapir/perf/TypeScanner.scala | 48 ++++++++--- 5 files changed, 155 insertions(+), 78 deletions(-) create mode 100644 perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala diff --git a/build.sbt b/build.sbt index 2c7433d1a5..0940ebe448 100644 --- a/build.sbt +++ b/build.sbt @@ -534,6 +534,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.1", "nl.grons" %% "metrics4-scala" % Versions.metrics4Scala % Test, "com.lihaoyi" %% "scalatags" % Versions.scalaTags % Test, + "com.github.scopt" %% "scopt" % "4.1.0", "io.github.classgraph" % "classgraph" % "4.8.165" % Test, "org.http4s" %% "http4s-core" % Versions.http4s, "org.http4s" %% "http4s-dsl" % Versions.http4s, diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala new file mode 100644 index 0000000000..3debe07b5d --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -0,0 +1,80 @@ +package sttp.tapir.perf + +import scala.concurrent.duration._ +import Common._ +import scopt.OParser +import scala.util.Failure +import scala.util.Success + +/** Parameters to customize a suite of performance tests. */ +case class PerfTestSuiteParams( + shortServerNames: List[String] = Nil, + shortSimulationNames: List[String] = Nil, + users: Int = 1, + durationSeconds: Int = 10, + skipGatlingReports: Boolean = false +) { + def adjustWildcards: PerfTestSuiteParams = { + val withAdjustedServer: PerfTestSuiteParams = + if (shortServerNames == List("*")) copy(shortServerNames = TypeScanner.allServers) else this + if (shortSimulationNames == List("*")) + withAdjustedServer.copy(shortSimulationNames = TypeScanner.allSimulations) + else + withAdjustedServer + } + + def duration: FiniteDuration = durationSeconds.seconds + + def totalTests: Int = shortServerNames.length * shortSimulationNames.length + + def minTotalDuration: FiniteDuration = (duration * totalTests.toLong).toMinutes.minutes + + /** Returns pairs of (fullServerName, shortServerName), for example: (sttp.tapir.perf.pekko.TapirServer, pekko.Tapir) + */ + def serverNames: List[(String, String)] = shortServerNames.map(s => s"${rootPackage}.${s}Server").zip(shortServerNames) + + /** Returns pairs of (fullSimulationName, shortSimulationName), for example: (sttp.tapir.perf.SimpleGetSimulation, SimpleGet) + */ + def simulationNames: List[(String, String)] = shortSimulationNames.map(s => s"${rootPackage}.${s}Simulation").zip(shortSimulationNames) +} + +object PerfTestSuiteParams { + val builder = OParser.builder[PerfTestSuiteParams] + import builder._ + val argParser = OParser.sequence( + programName("perf"), + opt[Seq[String]]('s', "server") + .required() + .action((x, c) => c.copy(shortServerNames = x.toList)) + .text("Comma-separated list of short server names, or '*' for all"), + opt[Seq[String]]('m', "sim") + .required() + .action((x, c) => c.copy(shortSimulationNames = x.toList)) + .text("Comma-separated list of short simulation names, or '*' for all"), + opt[Int]('u', "users") + .action((x, c) => c.copy(users = x)) + .text("Number of concurrent users"), + opt[Int]('d', "duration") + .action((x, c) => c.copy(durationSeconds = x)) + .text("Single simulation duration in seconds"), + opt[Boolean]('g', "skip-gatling-reports") + .action((x, c) => c.copy(skipGatlingReports = x)) + .text("Generate only aggregated suite report, may significantly shorten total time") + ) + + def parse(args: List[String]): PerfTestSuiteParams = { + OParser.parse(argParser, args, PerfTestSuiteParams()) match { + case Some(p) => + val params = p.adjustWildcards + TypeScanner.enusureExist(params.shortServerNames, params.shortSimulationNames) match { + case Success(_) => params + case Failure(ex) => + println(ex.getMessage) + sys.exit(-1) + } + case _ => + sys.exit(-1) + } + } + +} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index cddf1ba466..0c6b2eb1a0 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -5,12 +5,11 @@ import cats.syntax.all._ import fs2.io.file import fs2.text import sttp.tapir.perf.apis.ServerRunner -import Common._ + import java.nio.file.Paths import java.time.LocalDateTime import java.time.format.DateTimeFormatter import scala.reflect.runtime.universe -import scala.util.control.NonFatal /** Main entry point for running suites of performance tests and generating aggregated reports. A suite represents a set of Gatling * simulations executed on a set of servers, with some additional parameters like concurrent user count. One can run a single simulation on @@ -19,27 +18,23 @@ import scala.util.control.NonFatal */ object PerfTestSuiteRunner extends IOApp { - val arraySplitPattern = "\\s*[,]+\\s*".r val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader) def run(args: List[String]): IO[ExitCode] = { - if (args.length < 2) - exitOnIncorrectArgs - val shortServerNames = argToList(args(0), ifAll = TypeScanner.allServers) - val shortSimulationNames = argToList(args(1), ifAll = TypeScanner.allSimulations) - println(shortServerNames) - println(shortSimulationNames) - if (shortSimulationNames.isEmpty || shortServerNames.isEmpty) - exitOnIncorrectArgs + val params = PerfTestSuiteParams.parse(args) + System.setProperty("tapir.perf.user-count", params.users.toString) + System.setProperty("tapir.perf.duration-seconds", params.durationSeconds.toString) + println("===========================================================================================") + println(s"Running a suite of ${params.totalTests} tests, each for ${params.users} users and ${params.duration}") + println(s"Servers: ${params.shortServerNames}") + println(s"Simulations: ${params.shortSimulationNames}") + println(s"Expected total duration: at least ${params.minTotalDuration}") + println("Generated suite report paths will be printed to stdout after all tests are finished.") + println("===========================================================================================") - val serverNames = shortServerNames.map(s => s"${rootPackage}.${s}Server") - val simulationNames = shortSimulationNames.map(s => s"${rootPackage}.${s}Simulation") val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss") val currentTime = LocalDateTime.now().format(formatter) - // TODO ensure servers and simulations exist - // TODO Parse user count - // TODO add comprehensive help - ((simulationNames.zip(shortSimulationNames), (serverNames.zip(shortServerNames))) + ((params.simulationNames, params.serverNames) .mapN((x, y) => (x, y))) .traverse { case ((simulationName, shortSimulationName), (serverName, shortServerName)) => for { @@ -57,23 +52,6 @@ object PerfTestSuiteRunner extends IOApp { .as(ExitCode.Success) } - private def argToList(arg: String, ifAll: List[String]): List[String] = { - println(s"------------$arg") - if (arg.trim == "*") - ifAll - else - try { - if (arg.startsWith("[") && arg.endsWith("]")) - arraySplitPattern.split(arg.drop(1).dropRight(1)).toList - else - List(arg) - } catch { - case NonFatal(e) => - e.printStackTrace() - exitOnIncorrectArgs - } - } - private def startServerByTypeName(serverName: String): IO[ServerRunner.KillSwitch] = { try { val moduleSymbol = runtimeMirror.staticModule(serverName) @@ -106,11 +84,4 @@ object PerfTestSuiteRunner extends IOApp { .compile .drain >> IO.println(s"******* Test Suite report saved to $targetFilePath") } - - private def exitOnIncorrectArgs = { - println(s"Usage: perfTests/Test/runMain ${rootPackage}.PerfTestSuiteRunner serverName simulationName userCount") - println("Where serverName and simulationName can be a single name, an array of comma separated names, or '*' for all") - println("The userCount parameter is optional (default value is 1)") - sys.exit(1) - } } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index f26df61488..191ec3fd8a 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -28,27 +28,30 @@ object CommonSimulations { lazy val constRandomBytes = randomByteArray(LargeInputSize) lazy val constRandomAlphanumBytes = randomAlphanumByteArray(LargeInputSize) - def simple_get(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def simple_get(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpGet = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4")) scenario(s"Repeatedly invoke GET of route number $routeNumber") - .during(duration.toSeconds.toInt)(execHttpGet) + .during(duration)(execHttpGet) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } - + def getParamOpt(paramName: String): Option[String] = Option(System.getProperty(s"tapir.perf.${paramName}")) - def getParam(paramName: String): String = + def getParam(paramName: String): String = getParamOpt(paramName).getOrElse( - throw new IllegalArgumentException(s"Missing tapir.perf.${paramName} system property, ensure you're running perf tests correctly (see perfTests/README.md)") + throw new IllegalArgumentException( + s"Missing tapir.perf.${paramName} system property, ensure you're running perf tests correctly (see perfTests/README.md)" + ) ) private lazy val userCount = getParam("user-count").toInt - // Scenarios + private lazy val duration = getParam("duration-seconds").toInt - def scenario_post_string(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + // Scenarios + def scenario_post_string(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val body = new String(randomAlphanumByteArray(256)) val execHttpPost = exec( @@ -59,12 +62,12 @@ object CommonSimulations { ) scenario(s"Repeatedly invoke POST with short string body") - .during(duration.toSeconds.toInt)(execHttpPost) + .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) - + } - def scenario_post_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def scenario_post_bytes(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( http(s"HTTP POST /pathBytes$routeNumber/4") @@ -73,12 +76,12 @@ object CommonSimulations { ) scenario(s"Repeatedly invoke POST with short byte array body") - .during(duration.toSeconds.toInt)(execHttpPost) + .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } - def scenario_post_file(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def scenario_post_file(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( http(s"HTTP POST /pathFile$routeNumber/4") @@ -88,12 +91,12 @@ object CommonSimulations { ) scenario(s"Repeatedly invoke POST with file body") - .during(duration.toSeconds.toInt)(execHttpPost) + .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } - def scenario_post_long_bytes(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def scenario_post_long_bytes(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( http(s"HTTP POST /pathBytes$routeNumber/4") @@ -103,12 +106,12 @@ object CommonSimulations { ) scenario(s"Repeatedly invoke POST with large byte array") - .during(duration.toSeconds.toInt)(execHttpPost) + .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } - def scenario_post_long_string(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + def scenario_post_long_string(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( http(s"HTTP POST /path$routeNumber/4") @@ -118,36 +121,36 @@ object CommonSimulations { ) scenario(s"Repeatedly invoke POST with large byte array, interpreted to a String") - .during(duration.toSeconds.toInt)(execHttpPost) + .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) } } class SimpleGetSimulation extends Simulation { - setUp(CommonSimulations.simple_get(10.seconds, 0)) + setUp(CommonSimulations.simple_get(0)) } class SimpleGetMultiRouteSimulation extends Simulation { - setUp(CommonSimulations.simple_get(10.seconds, 127)) + setUp(CommonSimulations.simple_get(127)) } class PostBytesSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_bytes(10.seconds, 0)) + setUp(CommonSimulations.scenario_post_bytes(0)) } class PostLongBytesSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_long_bytes(10.seconds, 0)) + setUp(CommonSimulations.scenario_post_long_bytes(0)) } class PostFileSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_file(10.seconds, 0)) + setUp(CommonSimulations.scenario_post_file(0)) } class PostStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_string(10.seconds, 0)) + setUp(CommonSimulations.scenario_post_string(0)) } class PostLongStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_long_string(10.seconds, 0)) + setUp(CommonSimulations.scenario_post_long_string(0)) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala index 433aa80546..23bbf10ab7 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/TypeScanner.scala @@ -1,15 +1,17 @@ package sttp.tapir.perf +import io.gatling.core.scenario.Simulation import io.github.classgraph.ClassGraph -import scala.reflect.ClassTag -import scala.jdk.CollectionConverters._ import sttp.tapir.perf.apis.ServerRunner + +import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag +import scala.util.{Failure, Success, Try} + import Common._ -import io.gatling.core.scenario.Simulation -/** - * Uses the classgraph library to quickly find all possible server runners (objects extending ServerRunner) or - * simulations (classes extending Simulation). +/** Uses the classgraph library to quickly find all possible server runners (objects extending ServerRunner) or simulations (classes + * extending Simulation). */ object TypeScanner { def findAllImplementations[T: ClassTag](rootPackage: String): List[Class[_]] = { @@ -21,23 +23,43 @@ object TypeScanner { .scan() try { - val classes = if (superClass.isInterface) - scanResult.getClassesImplementing(superClass.getName) - else - scanResult.getSubclasses(superClass.getName) + val classes = + if (superClass.isInterface) + scanResult.getClassesImplementing(superClass.getName) + else + scanResult.getSubclasses(superClass.getName) classes.loadClasses().asScala.toList } finally { scanResult.close() } } - def allServers: List[String] = + lazy val allServers: List[String] = findAllImplementations[ServerRunner](rootPackage) .map(_.getName) .map(c => c.stripPrefix(s"${rootPackage}.").stripSuffix("Server$")) - - def allSimulations: List[String] = + + lazy val allSimulations: List[String] = findAllImplementations[Simulation](rootPackage) .map(_.getName) .map(c => c.stripPrefix(s"${rootPackage}.").stripSuffix("Simulation")) + + def enusureExist(serverShortNames: List[String], simShortNames: List[String]): Try[Unit] = { + val missingServers = serverShortNames.filterNot(allServers.contains) + val missingSims = simShortNames.filterNot(allSimulations.contains) + + if (missingServers.isEmpty && missingSims.isEmpty) { + Success(()) + } else { + val missingServersMessage = + if (missingServers.nonEmpty) + s"Unrecognized servers: ${missingServers.mkString(", ")}. Available servers: ${allServers.mkString(", ")}.\n" + else "" + val missingSimsMessage = + if (missingSims.nonEmpty) + s"Unrecognized simulations: ${missingSims.mkString(", ")}. Available simulations: ${allSimulations.mkString(", ")}" + else "" + Failure(new IllegalArgumentException(s"$missingServersMessage $missingSimsMessage".trim)) + } + } } From 55d3daa2fcb20f2f4869791cfeda6386c8e502b8 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 15 Jan 2024 15:40:01 +0100 Subject: [PATCH 24/53] Remove sbt command --- build.sbt | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/build.sbt b/build.sbt index 0940ebe448..4e45f2225e 100644 --- a/build.sbt +++ b/build.sbt @@ -350,7 +350,6 @@ lazy val rootProject = (project in file(".")) ideSkipProject := false, generateMimeByExtensionDB := GenerateMimeByExtensionDB() ) - .settings(commands += perfTestCommand) .aggregate(allAggregates: _*) // start a test server before running tests of a client interpreter; this is required both for JS tests run inside a @@ -497,29 +496,6 @@ lazy val tests: ProjectMatrix = (projectMatrix in file("tests")) ) .dependsOn(core, files, circeJson, cats) -val perfTestCommand = Command("perf", ("perf", "servName simName userCount(optional)"), "run performance tests") { state => - val parser = (Space ~> token(StringBasic.examples("servName"))) ~ - (Space ~> token(StringBasic.examples("simName"))) ~ - (Space ~> token(StringBasic.examples("userCount"))).? - parser.map { case servName ~ simName ~ userCountOpt => - val userCount = userCountOpt.map(_.trim).getOrElse("1") - (servName, simName, userCount) - } -} { (state, args) => - val (servName, simName, userCount) = args - System.setProperty("tapir.perf.serv-name", servName) - System.setProperty("tapir.perf.user-count", userCount) - val customSimId = s"$simName-$servName-${System.currentTimeMillis}" - // We have to use a command, because sbt macros can't handle string interpolations with dynamic values in (xxx).toTask("str") - val state2 = Command.process(s"perfTests/clean", state) - Command.process( - s"perfTests/Gatling/testOnly sttp.tapir.perf.${simName}Simulation", - state2 - ) - // read reports from last directory into report aggregator - // generate aggregated report -} - lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) .enablePlugins(GatlingPlugin) .settings(commonJvmSettings) From 97093a20a6b8b9242133ed5d62ad654a02820c37 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 16 Jan 2024 09:25:53 +0100 Subject: [PATCH 25/53] Tweak HTML report style --- .../scala/sttp/tapir/perf/HtmlResultsPrinter.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala index 73a21dc565..4fd41e9d98 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala @@ -4,8 +4,12 @@ import scalatags.Text.all._ import scalatags.Text object HtmlResultsPrinter { - val tableStyle = "border-collapse: collapse;" + val tableStyle = "border-collapse: collapse; font-family: Roboto, Helvetica, Arial, sans-serif;" val cellStyle = "border: 1px solid black; padding: 5px;" + val headStyle = + "border: 1px solid black; padding: 5px; color: rgb(245, 245, 245); background-color: rgb(85, 73, 75)" + val simCellStyle = + "border: 1px solid black; padding: 5px; color: black; background-color: rgb(243, 112, 94); font-weight: bold" def print(results: List[GatlingSimulationResult]): String = { @@ -17,11 +21,11 @@ object HtmlResultsPrinter { table(style := tableStyle)( thead( - tr(headers.map(header => th(header, style := cellStyle))) + tr(headers.map(header => th(header, style := headStyle))) ), tbody( for (row <- rows) yield { - tr(td(row.head.simulationName) :: row.map(toColumn), style := cellStyle) + tr(td(row.head.simulationName, style := simCellStyle) :: row.map(toColumn), style := cellStyle) } ) ).render From 3fbe5a5aecea29336c8dfed1629b39a026a453a1 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 16 Jan 2024 16:06:15 +0100 Subject: [PATCH 26/53] Allow skipping Gatling reports --- .../test/scala/sttp/tapir/perf/GatlingRunner.scala | 13 +++++++------ .../scala/sttp/tapir/perf/PerfTestSuiteRunner.scala | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala index 472d12d220..1f0826703f 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala @@ -2,18 +2,19 @@ package sttp.tapir.perf import io.gatling.app.Gatling import io.gatling.core.config.GatlingPropertiesBuilder -import scala.reflect.runtime.universe object GatlingRunner { /** Blocking, runs the entire Gatling simulation. */ - def runSimulationBlocking(simulationClassName: String): Int = { - System.setProperty("tapir.perf.user-count", "1") // TODO from config - val props = new GatlingPropertiesBuilder() + def runSimulationBlocking(simulationClassName: String, params: PerfTestSuiteParams): Int = { + val initialProps = new GatlingPropertiesBuilder() .simulationClass(simulationClassName) - .build - Gatling.fromMap(props) + val props = + if (params.skipGatlingReports) + initialProps.noReports() + else initialProps + Gatling.fromMap(props.build) } } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index 0c6b2eb1a0..69474254ff 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -40,7 +40,7 @@ object PerfTestSuiteRunner extends IOApp { for { serverKillSwitch <- startServerByTypeName(serverName) _ <- IO - .blocking(GatlingRunner.runSimulationBlocking(simulationName)) + .blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) .guarantee(serverKillSwitch) .ensureOr(errCode => new Exception(s"Gatling failed with code $errCode"))(_ == 0) serverSimulationResult <- GatlingLogProcessor.processLast(shortSimulationName, shortServerName) From cfd243c401df54fcea1adad93734dbe5f9fee250 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 16 Jan 2024 16:20:57 +0100 Subject: [PATCH 27/53] Add Vert.x Tapir server --- build.sbt | 2 +- .../main/scala/sttp/tapir/perf/Common.scala | 1 + .../scala/sttp/tapir/perf/vertx/Vertx.scala | 50 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala diff --git a/build.sbt b/build.sbt index 4e45f2225e..389f0c555a 100644 --- a/build.sbt +++ b/build.sbt @@ -525,7 +525,7 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) connectInput := true ) .jvmPlatform(scalaVersions = List(scala2_13)) - .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer, nettyServerCats, playServer) + .dependsOn(core, pekkoHttpServer, http4sServer, nettyServer, nettyServerCats, playServer, vertxServer, vertxServerCats) // integrations diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index d9ccdf31c1..c77984ec0f 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -3,4 +3,5 @@ package sttp.tapir.perf object Common { val rootPackage = "sttp.tapir.perf" val LargeInputSize = 5 * 1024 * 1024 + val Port = 8080 } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala new file mode 100644 index 0000000000..48801febad --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -0,0 +1,50 @@ +package sttp.tapir.perf.vertx + +import cats.effect.IO +import cats.effect.kernel.Resource +import io.vertx.core.http.HttpServerOptions +import io.vertx.core.{Future => VFuture, Vertx} +import io.vertx.ext.web.{Route, Router} +import sttp.tapir.perf.apis.{Endpoints, ServerRunner} +import sttp.tapir.server.vertx.VertxFutureServerInterpreter +import sttp.tapir.perf.Common._ +import scala.concurrent.Future + +object Tapir extends Endpoints { + val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList + + def route: Int => Router => Route = { (nRoutes: Int) => router => + val interpreter = VertxFutureServerInterpreter() + genEndpoints(nRoutes).map(interpreter.route(_)(router)).last + } +} + +object VertxRunner { + def runServer(route: Router => Route): IO[ServerRunner.KillSwitch] = { + Resource + .make(IO.delay(Vertx.vertx()))(vertx => IO.delay(vertx.close()).void) + .flatMap { vertx => + val router = Router.router(vertx) + val server = vertx.createHttpServer(new HttpServerOptions().setPort(Port)).requestHandler(router) + val listenIO = vertxFutureToIo(server.listen(Port)) + route.apply(router): Unit + Resource.make(listenIO)(s => vertxFutureToIo(s.close()).void) + } + .allocated + .map(_._2) + } + + private def vertxFutureToIo[A](future: => VFuture[A]): IO[A] = + IO.async[A] { cb => + IO { + future + .onFailure { cause => cb(Left(cause)) } + .onSuccess { result => cb(Right(result)) } + Some(IO.unit) + } + } +} + +object TapirServer extends ServerRunner { override def start = VertxRunner.runServer(Tapir.route(1)) } From d677a0e0a6b38a4fe779b70d28a39a1af9049231 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 14:23:20 +0100 Subject: [PATCH 28/53] Skip duplicates --- .../test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala index 3debe07b5d..c577c37a86 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -31,11 +31,12 @@ case class PerfTestSuiteParams( /** Returns pairs of (fullServerName, shortServerName), for example: (sttp.tapir.perf.pekko.TapirServer, pekko.Tapir) */ - def serverNames: List[(String, String)] = shortServerNames.map(s => s"${rootPackage}.${s}Server").zip(shortServerNames) + def serverNames: List[(String, String)] = shortServerNames.map(s => s"${rootPackage}.${s}Server").zip(shortServerNames).distinct /** Returns pairs of (fullSimulationName, shortSimulationName), for example: (sttp.tapir.perf.SimpleGetSimulation, SimpleGet) */ - def simulationNames: List[(String, String)] = shortSimulationNames.map(s => s"${rootPackage}.${s}Simulation").zip(shortSimulationNames) + def simulationNames: List[(String, String)] = + shortSimulationNames.map(s => s"${rootPackage}.${s}Simulation").zip(shortSimulationNames).distinct } object PerfTestSuiteParams { @@ -68,7 +69,7 @@ object PerfTestSuiteParams { val params = p.adjustWildcards TypeScanner.enusureExist(params.shortServerNames, params.shortSimulationNames) match { case Success(_) => params - case Failure(ex) => + case Failure(ex) => println(ex.getMessage) sys.exit(-1) } From 4311fb6b9029b88169a6a43b824878b153e0ecba Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 16:06:45 +0100 Subject: [PATCH 29/53] Finish Vertx vanilla server --- .../main/scala/sttp/tapir/perf/Common.scala | 1 + .../scala/sttp/tapir/perf/vertx/Vertx.scala | 60 ++++++++++++++++++- .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 1 + 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index c77984ec0f..ab93951760 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -4,4 +4,5 @@ object Common { val rootPackage = "sttp.tapir.perf" val LargeInputSize = 5 * 1024 * 1024 val Port = 8080 + val TmpDir = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala index 48801febad..a1ccde840e 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -4,11 +4,15 @@ import cats.effect.IO import cats.effect.kernel.Resource import io.vertx.core.http.HttpServerOptions import io.vertx.core.{Future => VFuture, Vertx} -import io.vertx.ext.web.{Route, Router} +import io.vertx.ext.web.handler.BodyHandler +import io.vertx.ext.web.{Route, Router, RoutingContext} +import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis.{Endpoints, ServerRunner} import sttp.tapir.server.vertx.VertxFutureServerInterpreter -import sttp.tapir.perf.Common._ + +import java.util.Date import scala.concurrent.Future +import scala.util.Random object Tapir extends Endpoints { val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) @@ -20,6 +24,55 @@ object Tapir extends Endpoints { genEndpoints(nRoutes).map(interpreter.route(_)(router)).last } } +object Vanilla extends Endpoints { + + def bodyHandler = BodyHandler.create(false).setBodyLimit(LargeInputSize + 100L) + + def route: Int => Router => Route = { (nRoutes: Int) => router => + (0 until nRoutes).map { number => + router.get(s"/path$number/4").handler { + ctx: RoutingContext => + val _ = ctx + .response() + .putHeader("content-type", "text/plain") + .end("Ok") + } + + router.post(s"/path$number/4").handler(bodyHandler).handler { + ctx: RoutingContext => + val body = ctx.body.asString() + val _ = ctx + .response() + .putHeader("content-type", "text/plain") + .end(s"Ok, body length = ${body.length}") + } + + router.post(s"/pathBytes$number/4").handler(bodyHandler).handler { + ctx: RoutingContext => + val bytes = ctx.body().asString() + val _ = ctx + .response() + .putHeader("content-type", "text/plain") + .end(s"Received ${bytes.length} bytes") + } + + router.post(s"/pathFile$number/4").handler(bodyHandler).handler { + ctx: RoutingContext => + val filePath = s"${TmpDir.getAbsolutePath}/tapir-${new Date().getTime}-${Random.nextLong()}" + val fs = ctx.vertx.fileSystem + val _ = fs + .createFile(filePath) + .flatMap(_ => fs.writeFile(filePath, ctx.body().buffer())) + .flatMap(_ => + ctx + .response() + .putHeader("content-type", "text/plain") + .end(s"Received binary stored as: ${filePath}") + ) + } + }.last + } +} object VertxRunner { def runServer(route: Router => Route): IO[ServerRunner.KillSwitch] = { @@ -48,3 +101,6 @@ object VertxRunner { } object TapirServer extends ServerRunner { override def start = VertxRunner.runServer(Tapir.route(1)) } +object TapirMultiServer extends ServerRunner { override def start = VertxRunner.runServer(Tapir.route(127)) } +object VanillaServer extends ServerRunner { override def start = VertxRunner.runServer(Vanilla.route(1)) } +object VanillaMultiServer extends ServerRunner { override def start = VertxRunner.runServer(Vanilla.route(127)) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index 69474254ff..6d392a442a 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -39,6 +39,7 @@ object PerfTestSuiteRunner extends IOApp { .traverse { case ((simulationName, shortSimulationName), (serverName, shortServerName)) => for { serverKillSwitch <- startServerByTypeName(serverName) + _ <- IO.println(s"Running server $shortServerName") _ <- IO .blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) .guarantee(serverKillSwitch) From 73e0506eeabf314d8cb5bd54a16909a2c7e9a210 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 16:09:57 +0100 Subject: [PATCH 30/53] Display servers as rows in HTML - There are much less sims than servers --- .../src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala index 4fd41e9d98..9bd135c74d 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala @@ -13,8 +13,8 @@ object HtmlResultsPrinter { def print(results: List[GatlingSimulationResult]): String = { - val headers = "Simulation" :: results.groupBy(_.simulationName).head._2.map(_.serverName) - createHtmlTable(headers, results.groupBy(_.simulationName).values.toList) + val headers = "Server" :: results.groupBy(_.serverName).head._2.map(_.simulationName) + createHtmlTable(headers, results.groupBy(_.serverName).values.toList) } private def createHtmlTable(headers: Seq[String], rows: List[List[GatlingSimulationResult]]): String = { @@ -25,7 +25,7 @@ object HtmlResultsPrinter { ), tbody( for (row <- rows) yield { - tr(td(row.head.simulationName, style := simCellStyle) :: row.map(toColumn), style := cellStyle) + tr(td(row.head.serverName, style := simCellStyle) :: row.map(toColumn), style := cellStyle) } ) ).render From 3b0969c94703e8bea281c42ac3aabb35c9d6d3b8 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 16:14:53 +0100 Subject: [PATCH 31/53] Disable Gatling reports by defaults, improve flag --- .../src/test/scala/sttp/tapir/perf/GatlingRunner.scala | 6 +++--- .../test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala index 1f0826703f..162e8dbadd 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingRunner.scala @@ -12,9 +12,9 @@ object GatlingRunner { .simulationClass(simulationClassName) val props = - if (params.skipGatlingReports) - initialProps.noReports() - else initialProps + if (params.buildGatlingReports) + initialProps + else initialProps.noReports() Gatling.fromMap(props.build) } } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala index c577c37a86..56a309eb21 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -12,7 +12,7 @@ case class PerfTestSuiteParams( shortSimulationNames: List[String] = Nil, users: Int = 1, durationSeconds: Int = 10, - skipGatlingReports: Boolean = false + buildGatlingReports: Boolean = false ) { def adjustWildcards: PerfTestSuiteParams = { val withAdjustedServer: PerfTestSuiteParams = @@ -58,9 +58,9 @@ object PerfTestSuiteParams { opt[Int]('d', "duration") .action((x, c) => c.copy(durationSeconds = x)) .text("Single simulation duration in seconds"), - opt[Boolean]('g', "skip-gatling-reports") - .action((x, c) => c.copy(skipGatlingReports = x)) - .text("Generate only aggregated suite report, may significantly shorten total time") + opt[Unit]('g', "gatling-reports") + .action((x, c) => c.copy(buildGatlingReports = true)) + .text("Generate Gatling reports for individuals sims, may significantly affect total time") ) def parse(args: List[String]): PerfTestSuiteParams = { From af446967a206afbe5d5af1e1a4b5b4172c7375d4 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 17:00:29 +0100 Subject: [PATCH 32/53] Add missing endpoints to http4s Vanilla --- .../main/scala/sttp/tapir/perf/Common.scala | 9 +++++- .../scala/sttp/tapir/perf/http4s/Http4s.scala | 29 +++++++++++++++---- .../scala/sttp/tapir/perf/vertx/Vertx.scala | 2 +- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index ab93951760..b07d6a9ae0 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,8 +1,15 @@ package sttp.tapir.perf +import java.util.Date +import scala.util.Random +import java.io.File +import java.nio.file.Path + object Common { val rootPackage = "sttp.tapir.perf" val LargeInputSize = 5 * 1024 * 1024 val Port = 8080 - val TmpDir = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile + val TmpDir: File = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile + def tempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}") + } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 9211680e7c..39b19962b6 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -6,21 +6,40 @@ import org.http4s.blaze.server.BlazeServerBuilder import org.http4s.dsl._ import org.http4s.implicits._ import org.http4s.server.Router +import fs2.io.file.{Files, Path => Fs2Path} import sttp.tapir.perf.apis._ +import sttp.tapir.perf.Common._ import sttp.tapir.server.http4s.Http4sServerInterpreter import sttp.tapir.integ.cats.effect.CatsMonadError import sttp.monad.MonadError object Vanilla { - val router: Int => HttpRoutes[IO] = (nRoutes: Int) => + val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router( (0 to nRoutes).map((n: Int) => - ("/path" + n.toString) -> { + ("/") -> { val dsl = Http4sDsl[IO] import dsl._ - HttpRoutes.of[IO] { case GET -> Root / IntVar(id) => - Ok((id + n).toString) - } + HttpRoutes.of[IO] { + case GET -> Root / s"path$n" / IntVar(id) => + Ok((id + n).toString) + case req @ POST -> Root / s"path$n" / IntVar(id) => + req.as[String].flatMap { _ => + Ok((id +n).toString) + } + case req @ POST -> Root / s"pathBytes$n" / IntVar(id) => + req.as[Array[Byte]].flatMap { bytes => + Ok(s"Received ${bytes.length} bytes") + } + case req @ POST -> Root / s"pathFile$n" / IntVar(id) => + val filePath = tempFilePath() + val sink = Files[IO].writeAll(Fs2Path.fromNioPath(filePath)) + req.body + .through(sink) + .compile + .drain + .flatMap(_ => Ok(s"File saved to ${filePath.toAbsolutePath.toString}")) + } } ): _* ) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala index a1ccde840e..86fa0b2436 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -61,7 +61,7 @@ object Vanilla extends Endpoints { val filePath = s"${TmpDir.getAbsolutePath}/tapir-${new Date().getTime}-${Random.nextLong()}" val fs = ctx.vertx.fileSystem val _ = fs - .createFile(filePath) + .createFile(tempFilePath().toString) .flatMap(_ => fs.writeFile(filePath, ctx.body().buffer())) .flatMap(_ => ctx From 42df1832d8a0be9b44bd250bbe6680ddd76ce1bc Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 17:32:25 +0100 Subject: [PATCH 33/53] Add missing vanilla endpoints for Pekko --- .../sttp/tapir/perf/pekko/PekkoHttp.scala | 66 ++++++++++++++----- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 21a7994dba..863e16da7b 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -3,25 +3,58 @@ package sttp.tapir.perf.pekko import cats.effect.IO import org.apache.pekko.actor.ActorSystem import org.apache.pekko.http.scaladsl.Http +import org.apache.pekko.http.scaladsl.model.HttpEntity import org.apache.pekko.http.scaladsl.server.Directives._ import org.apache.pekko.http.scaladsl.server.Route +import org.apache.pekko.stream.scaladsl.FileIO +import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.pekkohttp.PekkoHttpServerInterpreter -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} -import org.apache.pekko.stream.ActorMaterializer +import scala.concurrent.{ExecutionContextExecutor, Future} object Vanilla { - val router: Int => ActorSystem => Route = (nRoutes: Int) => (_: ActorSystem) => - concat( - (0 to nRoutes).map((n: Int) => - get { - path(("path" + n.toString) / IntNumber) { id => - complete((n + id).toString) - } - } - ): _* - ) + val router: Int => ActorSystem => Route = (nRoutes: Int) => + (_: ActorSystem) => + concat( + (0 to nRoutes).flatMap { (n: Int) => + List( + get { + path(("path" + n.toString) / IntNumber) { id => + complete((n + id).toString) + } + }, + post { + path(("path" + n.toString) / IntNumber) { id => + entity(as[String]) { _ => + complete((n + id).toString) + } + } + }, + post { + path(("pathBytes" + n.toString) / IntNumber) { id => + entity(as[Array[Byte]]) { bytes => + complete(s"Received ${bytes.length} bytes") + } + } + }, + post { + path(("pathFile" + n.toString) / IntNumber) { id => + extractRequestContext { ctx => + entity(as[HttpEntity]) { httpEntity => + val path = tempFilePath() + val sink = FileIO.toPath(path) + val finishedWriting = httpEntity.dataBytes.runWith(sink)(ctx.materializer) + onSuccess(finishedWriting) { _ => + complete(s"File saved to $path") + } + } + } + } + } + ) + }: _* + ) } object Tapir extends Endpoints { @@ -29,10 +62,11 @@ object Tapir extends Endpoints { def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - def router: Int => ActorSystem => Route = (nRoutes: Int) => (actorSystem: ActorSystem) => - PekkoHttpServerInterpreter()(actorSystem.dispatcher).toRoute( - genEndpoints(nRoutes) - ) + def router: Int => ActorSystem => Route = (nRoutes: Int) => + (actorSystem: ActorSystem) => + PekkoHttpServerInterpreter()(actorSystem.dispatcher).toRoute( + genEndpoints(nRoutes) + ) } object PekkoHttp { From 90fe6e6c2e67722548542c62a76e3d24dcec9831 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 17:38:30 +0100 Subject: [PATCH 34/53] Refactoring and cleanup --- .../main/scala/sttp/tapir/perf/Common.scala | 4 +-- .../scala/sttp/tapir/perf/http4s/Http4s.scala | 18 +++++------ .../tapir/perf/netty/cats/NettyCats.scala | 24 +++++++-------- .../tapir/perf/netty/future/NettyFuture.scala | 30 ++++++++++--------- .../sttp/tapir/perf/pekko/PekkoHttp.scala | 2 +- .../scala/sttp/tapir/perf/play/Play.scala | 20 ++++++------- 6 files changed, 49 insertions(+), 49 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index b07d6a9ae0..b9f805a664 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,9 +1,9 @@ package sttp.tapir.perf -import java.util.Date -import scala.util.Random import java.io.File import java.nio.file.Path +import java.util.Date +import scala.util.Random object Common { val rootPackage = "sttp.tapir.perf" diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 39b19962b6..0ca19f5189 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -1,31 +1,31 @@ package sttp.tapir.perf.http4s import cats.effect._ +import fs2.io.file.{Files, Path => Fs2Path} import org.http4s._ import org.http4s.blaze.server.BlazeServerBuilder import org.http4s.dsl._ import org.http4s.implicits._ import org.http4s.server.Router -import fs2.io.file.{Files, Path => Fs2Path} -import sttp.tapir.perf.apis._ +import sttp.monad.MonadError +import sttp.tapir.integ.cats.effect.CatsMonadError import sttp.tapir.perf.Common._ +import sttp.tapir.perf.apis._ import sttp.tapir.server.http4s.Http4sServerInterpreter -import sttp.tapir.integ.cats.effect.CatsMonadError -import sttp.monad.MonadError object Vanilla { - val router: Int => HttpRoutes[IO] = (nRoutes: Int) => + val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router( (0 to nRoutes).map((n: Int) => ("/") -> { val dsl = Http4sDsl[IO] import dsl._ - HttpRoutes.of[IO] { + HttpRoutes.of[IO] { case GET -> Root / s"path$n" / IntVar(id) => Ok((id + n).toString) case req @ POST -> Root / s"path$n" / IntVar(id) => - req.as[String].flatMap { _ => - Ok((id +n).toString) + req.as[String].flatMap { _ => + Ok((id + n).toString) } case req @ POST -> Root / s"pathBytes$n" / IntVar(id) => req.as[Array[Byte]].flatMap { bytes => @@ -39,7 +39,7 @@ object Vanilla { .compile .drain .flatMap(_ => Ok(s"File saved to ${filePath.toAbsolutePath.toString}")) - } + } } ): _* ) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala index 076ea578c8..fabe7a6273 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala @@ -1,9 +1,10 @@ package sttp.tapir.perf.netty.cats -import sttp.tapir.server.ServerEndpoint +import cats.effect.IO +import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ +import sttp.tapir.server.ServerEndpoint import sttp.tapir.server.netty.cats.NettyCatsServer -import cats.effect.IO object Tapir extends Endpoints { @@ -15,18 +16,17 @@ object Tapir extends Endpoints { object NettyCats { def runServer(endpoints: List[ServerEndpoint[Any, IO]]): IO[ServerRunner.KillSwitch] = { - val declaredPort = 8080 + val declaredPort = Port val declaredHost = "0.0.0.0" // Starting netty server - NettyCatsServer.io().allocated.flatMap { - case (server, killSwitch) => - server - .port(declaredPort) - .host(declaredHost) - .addEndpoints(endpoints) - .start() - .map(_ => killSwitch) - } + NettyCatsServer.io().allocated.flatMap { case (server, killSwitch) => + server + .port(declaredPort) + .host(declaredHost) + .addEndpoints(endpoints) + .start() + .map(_ => killSwitch) + } } } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala index 826798a56f..8b4b2f43a5 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala @@ -1,19 +1,17 @@ package sttp.tapir.perf.netty.future -import sttp.tapir.server.ServerEndpoint -import scala.concurrent.Future +import cats.effect.IO import sttp.tapir.perf.apis._ -import sttp.monad.MonadError -import sttp.monad.FutureMonad -import scala.concurrent.ExecutionContext +import sttp.tapir.perf.Common._ import sttp.tapir.server.netty.{NettyFutureServer, NettyFutureServerBinding} +import sttp.tapir.server.ServerEndpoint + +import scala.concurrent.ExecutionContext import ExecutionContext.Implicits.global -import cats.effect.IO +import scala.concurrent.Future object Tapir extends Endpoints { - implicit val mErr: MonadError[Future] = new FutureMonad()(ExecutionContext.Implicits.global) - val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList @@ -22,15 +20,19 @@ object Tapir extends Endpoints { object NettyFuture { def runServer(endpoints: List[ServerEndpoint[Any, Future]]): IO[ServerRunner.KillSwitch] = { - val declaredPort = 8080 + val declaredPort = Port val declaredHost = "0.0.0.0" // Starting netty server val serverBinding: IO[NettyFutureServerBinding] = - IO.fromFuture(IO(NettyFutureServer() - .port(declaredPort) - .host(declaredHost) - .addEndpoints(endpoints) - .start())) + IO.fromFuture( + IO( + NettyFutureServer() + .port(declaredPort) + .host(declaredHost) + .addEndpoints(endpoints) + .start() + ) + ) serverBinding.map(b => IO.fromFuture(IO(b.stop()))) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 863e16da7b..4697c61ba5 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -77,7 +77,7 @@ object PekkoHttp { IO.fromFuture( IO( Http() - .newServerAt("127.0.0.1", 8080) + .newServerAt("127.0.0.1", Port) .bind(router(actorSystem)) .map { binding => IO.fromFuture(IO(binding.unbind().flatMap(_ => actorSystem.terminate()))).void diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index fdc6bb13b6..bccae118d0 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -14,9 +14,7 @@ import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.play.PlayServerInterpreter -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} -import org.apache.pekko.stream.Materializer -import org.apache.pekko.stream.ActorMaterializer +import scala.concurrent.{ExecutionContext, Future} object Vanilla extends ControllerHelpers { @@ -35,12 +33,11 @@ object Vanilla extends ControllerHelpers { implicit val actorSystemForMaterializer: ActorSystem = actorSystem - val simpleGet: Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { - implicit request => - val param = request.path.split("/").last - Future.successful( - Ok(param) - ) + val simpleGet: Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { implicit request => + val param = request.path.split("/").last + Future.successful( + Ok(param) + ) } val postBytes: Action[ByteString] = @@ -73,7 +70,8 @@ object Vanilla extends ControllerHelpers { postFile } } - def router: Int => ActorSystem => Routes = (nRoutes: Int) => (actorSystem: ActorSystem) => (0 until nRoutes).map(genRoutesSingle(actorSystem)).reduceLeft(_ orElse _) + def router: Int => ActorSystem => Routes = (nRoutes: Int) => + (actorSystem: ActorSystem) => (0 until nRoutes).map(genRoutesSingle(actorSystem)).reduceLeft(_ orElse _) } object Tapir extends Endpoints { @@ -95,7 +93,7 @@ object Play { def runServer(routes: ActorSystem => Routes): IO[ServerRunner.KillSwitch] = { implicit lazy val perfActorSystem: ActorSystem = ActorSystem(s"tapir-play") val components = new DefaultPekkoHttpServerComponents { - override lazy val serverConfig: ServerConfig = ServerConfig(port = Some(8080), address = "127.0.0.1", mode = Mode.Test) + override lazy val serverConfig: ServerConfig = ServerConfig(port = Some(Port), address = "127.0.0.1", mode = Mode.Test) override lazy val actorSystem: ActorSystem = perfActorSystem override def router: Router = Router.from( From a5146c027df0e3b3f3ad78433e1f1575da7e9fbe Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 20:50:53 +0100 Subject: [PATCH 35/53] Fix netty-cats server shutdown --- .../tapir/perf/netty/cats/NettyCats.scala | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala index fabe7a6273..f0bc1e282d 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala @@ -1,6 +1,7 @@ package sttp.tapir.perf.netty.cats import cats.effect.IO +import cats.effect.kernel.Resource import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.ServerEndpoint @@ -19,14 +20,19 @@ object NettyCats { val declaredPort = Port val declaredHost = "0.0.0.0" // Starting netty server - NettyCatsServer.io().allocated.flatMap { case (server, killSwitch) => - server - .port(declaredPort) - .host(declaredHost) - .addEndpoints(endpoints) - .start() - .map(_ => killSwitch) - } + NettyCatsServer + .io() + .flatMap { server => + Resource.make( + server + .port(declaredPort) + .host(declaredHost) + .addEndpoints(endpoints) + .start() + )(binding => binding.stop()) + } + .allocated + .map(_._2) } } From 3ffd82e5c59a595940253db3e3dae456caa54a4e Mon Sep 17 00:00:00 2001 From: kciesielski Date: Wed, 17 Jan 2024 21:19:59 +0100 Subject: [PATCH 36/53] Use constant --- perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 0ca19f5189..7d8c327b12 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -62,7 +62,7 @@ object Tapir extends Endpoints { object server { def runServer(router: HttpRoutes[IO]): IO[ServerRunner.KillSwitch] = BlazeServerBuilder[IO] - .bindHttp(8080, "localhost") + .bindHttp(Port, "localhost") .withHttpApp(router.orNotFound) .resource .allocated From 8732f0d4684ccf82af2400314251b4c32bebd4e1 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Thu, 18 Jan 2024 08:34:32 +0100 Subject: [PATCH 37/53] Minor tweaks in Vertx vanilla file endpoint --- .../src/main/scala/sttp/tapir/perf/vertx/Vertx.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala index 86fa0b2436..c4dbf6e3b3 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -10,9 +10,7 @@ import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis.{Endpoints, ServerRunner} import sttp.tapir.server.vertx.VertxFutureServerInterpreter -import java.util.Date import scala.concurrent.Future -import scala.util.Random object Tapir extends Endpoints { val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) @@ -58,16 +56,16 @@ object Vanilla extends Endpoints { router.post(s"/pathFile$number/4").handler(bodyHandler).handler { ctx: RoutingContext => - val filePath = s"${TmpDir.getAbsolutePath}/tapir-${new Date().getTime}-${Random.nextLong()}" + val filePath = tempFilePath() val fs = ctx.vertx.fileSystem val _ = fs - .createFile(tempFilePath().toString) - .flatMap(_ => fs.writeFile(filePath, ctx.body().buffer())) + .createFile(filePath.toString) + .flatMap(_ => fs.writeFile(filePath.toString, ctx.body().buffer())) .flatMap(_ => ctx .response() .putHeader("content-type", "text/plain") - .end(s"Received binary stored as: ${filePath}") + .end(s"Received binary stored as: $filePath") ) } }.last From 33763ee6bc0870ef8afed2d75eebb3ace7f1ea38 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Fri, 19 Jan 2024 14:40:44 +0100 Subject: [PATCH 38/53] Use text/plain because otherwise Play complains --- .../scala/sttp/tapir/perf/play/Play.scala | 2 +- .../scala/sttp/tapir/perf/Simulations.scala | 20 ++++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index bccae118d0..ea7ff12d86 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -53,7 +53,7 @@ object Vanilla extends ControllerHelpers { Future.successful(Ok(s"$param-${str.length}")) } - val postFile: Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers.apply().temporaryFile).async { implicit request => + val postFile: Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers().temporaryFile).async { implicit request => val param = request.path.split("/").last val file: Files.TemporaryFile = request.body Future.successful(Ok(s"$param-${file.path.toString}")) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 191ec3fd8a..28e95fda9d 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -4,12 +4,7 @@ import io.gatling.core.Predef._ import io.gatling.core.structure.PopulationBuilder import io.gatling.http.Predef._ -import scala.concurrent.duration.{DurationInt, FiniteDuration} -import scala.reflect.runtime.universe import scala.util.Random -import cats.effect.IO -import cats.effect.unsafe.IORuntime -import sttp.tapir.perf.apis.ServerRunner import sttp.tapir.perf.Common._ object CommonSimulations { @@ -25,8 +20,8 @@ object CommonSimulations { def randomAlphanumByteArray(size: Int): Array[Byte] = Random.alphanumeric.take(size).map(_.toByte).toArray - lazy val constRandomBytes = randomByteArray(LargeInputSize) - lazy val constRandomAlphanumBytes = randomAlphanumByteArray(LargeInputSize) + lazy val constRandomLongBytes = randomByteArray(LargeInputSize) + lazy val constRandomLongAlphanumBytes = randomAlphanumByteArray(LargeInputSize) def simple_get(routeNumber: Int): PopulationBuilder = { val httpProtocol = http.baseUrl(baseUrl) @@ -72,7 +67,8 @@ object CommonSimulations { val execHttpPost = exec( http(s"HTTP POST /pathBytes$routeNumber/4") .post(s"/pathBytes$routeNumber/4") - .body(ByteArrayBody(randomByteArray(256))) + .body(ByteArrayBody(randomAlphanumByteArray(256))) + .header("Content-Type", "application/octet-stream") ) scenario(s"Repeatedly invoke POST with short byte array body") @@ -86,7 +82,7 @@ object CommonSimulations { val execHttpPost = exec( http(s"HTTP POST /pathFile$routeNumber/4") .post(s"/pathFile$routeNumber/4") - .body(ByteArrayBody(constRandomBytes)) + .body(ByteArrayBody(constRandomLongBytes)) .header("Content-Type", "application/octet-stream") ) @@ -101,7 +97,7 @@ object CommonSimulations { val execHttpPost = exec( http(s"HTTP POST /pathBytes$routeNumber/4") .post(s"/pathBytes$routeNumber/4") - .body(ByteArrayBody(constRandomBytes)) + .body(ByteArrayBody(constRandomLongAlphanumBytes)) .header("Content-Type", "application/octet-stream") ) @@ -116,8 +112,8 @@ object CommonSimulations { val execHttpPost = exec( http(s"HTTP POST /path$routeNumber/4") .post(s"/path$routeNumber/4") - .body(ByteArrayBody(constRandomAlphanumBytes)) - .header("Content-Type", "application/octet-stream") + .body(ByteArrayBody(constRandomLongAlphanumBytes)) + .header("Content-Type", "text/plain") ) scenario(s"Repeatedly invoke POST with large byte array, interpreted to a String") From 35052167e57511a2398fa623c9e1be909b184479 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Fri, 19 Jan 2024 14:41:10 +0100 Subject: [PATCH 39/53] Sort rows in HTML report --- .../src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala index 9bd135c74d..4fd7edb22a 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala @@ -14,7 +14,7 @@ object HtmlResultsPrinter { def print(results: List[GatlingSimulationResult]): String = { val headers = "Server" :: results.groupBy(_.serverName).head._2.map(_.simulationName) - createHtmlTable(headers, results.groupBy(_.serverName).values.toList) + createHtmlTable(headers, results.groupBy(_.serverName).values.toList.sortBy(_.head.serverName)) } private def createHtmlTable(headers: Seq[String], rows: List[List[GatlingSimulationResult]]): String = { From 5fa603fdc274338173dae11d3905a728c92a3749 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Fri, 19 Jan 2024 14:49:26 +0100 Subject: [PATCH 40/53] Fixes --- ...ltsPrinter.scala => CsvReportPrinter.scala} | 2 +- ...tsPrinter.scala => HtmlReportPrinter.scala} | 2 +- .../sttp/tapir/perf/PerfTestSuiteParams.scala | 2 +- .../sttp/tapir/perf/PerfTestSuiteRunner.scala | 4 ++-- .../scala/sttp/tapir/perf/Simulations.scala | 18 +++++++++--------- 5 files changed, 14 insertions(+), 14 deletions(-) rename perf-tests/src/test/scala/sttp/tapir/perf/{CsvResultsPrinter.scala => CsvReportPrinter.scala} (97%) rename perf-tests/src/test/scala/sttp/tapir/perf/{HtmlResultsPrinter.scala => HtmlReportPrinter.scala} (98%) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/CsvReportPrinter.scala similarity index 97% rename from perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala rename to perf-tests/src/test/scala/sttp/tapir/perf/CsvReportPrinter.scala index 644e556840..4234b45d2f 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/CsvResultsPrinter.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/CsvReportPrinter.scala @@ -1,6 +1,6 @@ package sttp.tapir.perf -object CsvResultsPrinter { +object CsvReportPrinter { def print(results: List[GatlingSimulationResult]): String = { val groupedResults = results.groupBy(_.simulationName) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlReportPrinter.scala similarity index 98% rename from perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala rename to perf-tests/src/test/scala/sttp/tapir/perf/HtmlReportPrinter.scala index 4fd7edb22a..81e1168196 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/HtmlResultsPrinter.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/HtmlReportPrinter.scala @@ -3,7 +3,7 @@ package sttp.tapir.perf import scalatags.Text.all._ import scalatags.Text -object HtmlResultsPrinter { +object HtmlReportPrinter { val tableStyle = "border-collapse: collapse; font-family: Roboto, Helvetica, Arial, sans-serif;" val cellStyle = "border: 1px solid black; padding: 5px;" val headStyle = diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala index 56a309eb21..fa2130d19e 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -59,7 +59,7 @@ object PerfTestSuiteParams { .action((x, c) => c.copy(durationSeconds = x)) .text("Single simulation duration in seconds"), opt[Unit]('g', "gatling-reports") - .action((x, c) => c.copy(buildGatlingReports = true)) + .action((_, c) => c.copy(buildGatlingReports = true)) .text("Generate Gatling reports for individuals sims, may significantly affect total time") ) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index 6d392a442a..bf2ff938d1 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -66,12 +66,12 @@ object PerfTestSuiteRunner extends IOApp { } private def writeCsvReport(currentTime: String)(results: List[GatlingSimulationResult]): IO[Unit] = { - val csv = CsvResultsPrinter.print(results) + val csv = CsvReportPrinter.print(results) writeReportFile(csv, "csv", currentTime) } private def writeHtmlReport(currentTime: String)(results: List[GatlingSimulationResult]): IO[Unit] = { - val html = HtmlResultsPrinter.print(results) + val html = HtmlReportPrinter.print(results) writeReportFile(html, "html", currentTime) } diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 28e95fda9d..32290f4338 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -68,7 +68,7 @@ object CommonSimulations { http(s"HTTP POST /pathBytes$routeNumber/4") .post(s"/pathBytes$routeNumber/4") .body(ByteArrayBody(randomAlphanumByteArray(256))) - .header("Content-Type", "application/octet-stream") + .header("Content-Type", "text/plain") // otherwise Play complains ) scenario(s"Repeatedly invoke POST with short byte array body") @@ -98,7 +98,7 @@ object CommonSimulations { http(s"HTTP POST /pathBytes$routeNumber/4") .post(s"/pathBytes$routeNumber/4") .body(ByteArrayBody(constRandomLongAlphanumBytes)) - .header("Content-Type", "application/octet-stream") + .header("Content-Type", "text/plain") // otherwise Play complains ) scenario(s"Repeatedly invoke POST with large byte array") @@ -124,29 +124,29 @@ object CommonSimulations { } class SimpleGetSimulation extends Simulation { - setUp(CommonSimulations.simple_get(0)) + setUp(CommonSimulations.simple_get(0)): Unit } class SimpleGetMultiRouteSimulation extends Simulation { - setUp(CommonSimulations.simple_get(127)) + setUp(CommonSimulations.simple_get(127)): Unit } class PostBytesSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_bytes(0)) + setUp(CommonSimulations.scenario_post_bytes(0)): Unit } class PostLongBytesSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_long_bytes(0)) + setUp(CommonSimulations.scenario_post_long_bytes(0)): Unit } class PostFileSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_file(0)) + setUp(CommonSimulations.scenario_post_file(0)): Unit } class PostStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_string(0)) + setUp(CommonSimulations.scenario_post_string(0)): Unit } class PostLongStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_long_string(0)) + setUp(CommonSimulations.scenario_post_long_string(0)): Unit } From f30379106df7338ad11798aede3ddc8c63ecca57 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Fri, 19 Jan 2024 15:18:18 +0100 Subject: [PATCH 41/53] Add Vertx Cats Effect server --- .../scala/sttp/tapir/perf/vertx/Vertx.scala | 4 +-- .../tapir/perf/vertx/cats/VertxCats.scala | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala index c4dbf6e3b3..9a9a93c965 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -1,7 +1,7 @@ package sttp.tapir.perf.vertx -import cats.effect.IO -import cats.effect.kernel.Resource +import _root_.cats.effect.IO +import _root_.cats.effect.kernel.Resource import io.vertx.core.http.HttpServerOptions import io.vertx.core.{Future => VFuture, Vertx} import io.vertx.ext.web.handler.BodyHandler diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala new file mode 100644 index 0000000000..6d26352e0f --- /dev/null +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala @@ -0,0 +1,31 @@ +package sttp.tapir.perf.vertx.cats + +import cats.effect.IO +import cats.effect.std.Dispatcher +import io.vertx.ext.web.Route +import io.vertx.ext.web.Router +import sttp.tapir.perf.apis.{Endpoints, ServerRunner} +import sttp.tapir.server.vertx.cats.VertxCatsServerInterpreter +import sttp.tapir.perf.vertx.VertxRunner + +object Tapir extends Endpoints { + val serverEndpointGens = replyingWithDummyStr(allEndpoints, IO.pure) + + def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList + + def route(dispatcher: Dispatcher[IO]): Int => Router => Route = { (nRoutes: Int) => router => + val interpreter = VertxCatsServerInterpreter(dispatcher) + genEndpoints(nRoutes).map(interpreter.route(_)(router)).last + } +} + +class VertxCatsRunner(numRoutes: Int) extends ServerRunner { + + override def start = + Dispatcher.parallel[IO].allocated.flatMap { case (dispatcher, releaseDispatcher) => + VertxRunner.runServer(Tapir.route(dispatcher)(numRoutes)).map(_.flatMap(_ => releaseDispatcher)) + } +} + +object TapirServer extends VertxCatsRunner(numRoutes = 1) +object TapirMultiServer extends VertxCatsRunner(numRoutes = 1) From 1bfc3545faa33f8ddbb6b8f5521e133aac67efff Mon Sep 17 00:00:00 2001 From: kciesielski Date: Fri, 19 Jan 2024 19:46:09 +0100 Subject: [PATCH 42/53] Many improvements --- .../sttp/tapir/perf/apis/Endpoints.scala | 95 ++++++++++--------- .../scala/sttp/tapir/perf/http4s/Http4s.scala | 19 ++-- .../tapir/perf/netty/cats/NettyCats.scala | 11 +-- .../tapir/perf/netty/future/NettyFuture.scala | 11 +-- .../sttp/tapir/perf/pekko/PekkoHttp.scala | 16 ++-- .../scala/sttp/tapir/perf/play/Play.scala | 52 +++++----- .../scala/sttp/tapir/perf/vertx/Vertx.scala | 27 +++--- .../tapir/perf/vertx/cats/VertxCats.scala | 16 ++-- .../sttp/tapir/perf/GatlingLogProcessor.scala | 5 +- .../scala/sttp/tapir/perf/Simulations.scala | 88 ++++++++++------- .../netty/internal/NettyBootstrap.scala | 2 +- 11 files changed, 168 insertions(+), 174 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala index 4553dd05d9..58f3667ab6 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/apis/Endpoints.scala @@ -1,58 +1,67 @@ package sttp.tapir.perf.apis +import cats.effect.IO import sttp.tapir._ import sttp.tapir.perf.Common._ import sttp.tapir.server.ServerEndpoint import sttp.tapir.server.model.EndpointExtensions._ +import java.io.File +import scala.concurrent.Future + trait Endpoints { type EndpointGen = Int => PublicEndpoint[_, String, String, Any] type ServerEndpointGen[F[_]] = Int => ServerEndpoint[Any, F] - val gen_get_in_string_out_string: EndpointGen = { (n: Int) => - endpoint.get - .in("path" + n.toString) - .in(path[Int]("id")) - .errorOut(stringBody) - .out(stringBody) - } - - val gen_post_in_string_out_string: EndpointGen = { (n: Int) => - endpoint.post - .in("path" + n.toString) - .in(path[Int]("id")) - .in(stringBody) - .maxRequestBodyLength(LargeInputSize + 1024L) - .errorOut(stringBody) - .out(stringBody) - } - - val gen_post_in_bytes_out_string: EndpointGen = { (n: Int) => - endpoint.post - .in("pathBytes" + n.toString) - .in(path[Int]("id")) - .in(byteArrayBody) - .maxRequestBodyLength(LargeInputSize + 1024L) - .errorOut(stringBody) - .out(stringBody) - } - - val gen_post_in_file_out_string: EndpointGen = { (n: Int) => - endpoint.post - .in("pathFile" + n.toString) - .in(path[Int]("id")) - .in(fileBody) - .maxRequestBodyLength(LargeInputSize + 1024L) - .errorOut(stringBody) - .out(stringBody) + def serverEndpoints[F[_]](reply: String => F[String]): List[ServerEndpointGen[F]] = { + List( + { (n: Int) => + endpoint.get + .in("path" + n.toString) + .in(path[Int]("id")) + .out(stringBody) + .serverLogicSuccess { id => + reply((id + n).toString) + } + }, + { (n: Int) => + endpoint.post + .in("path" + n.toString) + .in(stringBody) + .maxRequestBodyLength(LargeInputSize + 1024L) + .out(stringBody) + .serverLogicSuccess { + body: String => + reply(s"Ok [$n], string length = ${body.length}") + } + }, + { (n: Int) => + endpoint.post + .in("pathBytes" + n.toString) + .in(byteArrayBody) + .maxRequestBodyLength(LargeInputSize + 1024L) + .out(stringBody) + .serverLogicSuccess { body: Array[Byte] => + reply(s"Ok [$n], bytes length = ${body.length}") + } + }, + { (n: Int) => + endpoint.post + .in("pathFile" + n.toString) + .in(fileBody) + .maxRequestBodyLength(LargeInputSize + 1024L) + .out(stringBody) + .serverLogicSuccess { + body: File => + reply(s"Ok [$n], file saved to ${body.toPath}") + } + } + ) } - val allEndpoints = - List(gen_get_in_string_out_string, gen_post_in_string_out_string, gen_post_in_bytes_out_string, gen_post_in_file_out_string) - - def replyingWithDummyStr[F[_]](endpointGens: List[EndpointGen], reply: String => F[String]): Seq[ServerEndpointGen[F]] = - endpointGens.map(gen => gen.andThen(se => se.serverLogicSuccess[F](_ => reply("ok")))) + def genServerEndpoints[F[_]](routeCount: Int)(reply: String => F[String]): List[ServerEndpoint[Any, F]] = + serverEndpoints[F](reply).flatMap(gen => (0 to routeCount).map(i => gen(i))) - def genServerEndpoints[F[_]](gens: Seq[ServerEndpointGen[F]])(routeCount: Int): Seq[ServerEndpoint[Any, F]] = - gens.flatMap(gen => (0 to routeCount).map(i => gen(i))) + def genEndpointsFuture(count: Int): List[ServerEndpoint[Any, Future]] = genServerEndpoints(count)(Future.successful) + def genEndpointsIO(count: Int): List[ServerEndpoint[Any, IO]] = genServerEndpoints(count)(IO.pure) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 7d8c327b12..d9e7904fbb 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -22,23 +22,23 @@ object Vanilla { import dsl._ HttpRoutes.of[IO] { case GET -> Root / s"path$n" / IntVar(id) => - Ok((id + n).toString) - case req @ POST -> Root / s"path$n" / IntVar(id) => - req.as[String].flatMap { _ => - Ok((id + n).toString) + Ok((id + n.toInt).toString) + case req @ POST -> Root / s"path$n" => + req.as[String].flatMap { str => + Ok(s"Ok [$n], string length = ${str.length}") } - case req @ POST -> Root / s"pathBytes$n" / IntVar(id) => + case req @ POST -> Root / s"pathBytes$n" => req.as[Array[Byte]].flatMap { bytes => - Ok(s"Received ${bytes.length} bytes") + Ok(s"Ok [$n], bytes length = ${bytes.length}") } - case req @ POST -> Root / s"pathFile$n" / IntVar(id) => + case req @ POST -> Root / s"pathFile$n" => val filePath = tempFilePath() val sink = Files[IO].writeAll(Fs2Path.fromNioPath(filePath)) req.body .through(sink) .compile .drain - .flatMap(_ => Ok(s"File saved to ${filePath.toAbsolutePath.toString}")) + .flatMap(_ => Ok(s"Ok [$n], file saved to ${filePath.toAbsolutePath.toString}")) } } ): _* @@ -49,12 +49,11 @@ object Tapir extends Endpoints { implicit val mErr: MonadError[IO] = new CatsMonadError[IO] - val serverEndpointGens = replyingWithDummyStr(allEndpoints, IO.pure) val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router("/" -> { Http4sServerInterpreter[IO]().toRoutes( - genServerEndpoints(serverEndpointGens)(nRoutes).toList + genEndpointsIO(nRoutes) ) }) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala index f0bc1e282d..0612430957 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala @@ -7,12 +7,7 @@ import sttp.tapir.perf.apis._ import sttp.tapir.server.ServerEndpoint import sttp.tapir.server.netty.cats.NettyCatsServer -object Tapir extends Endpoints { - - val serverEndpointGens = replyingWithDummyStr(allEndpoints, IO.pure) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList -} +object Tapir extends Endpoints object NettyCats { @@ -36,5 +31,5 @@ object NettyCats { } } -object TapirServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpoints(1)) } -object TapirMultiServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpoints(128)) } +object TapirServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpointsIO(1)) } +object TapirMultiServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpointsIO(128)) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala index 8b4b2f43a5..d405adc7e3 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala @@ -10,12 +10,7 @@ import scala.concurrent.ExecutionContext import ExecutionContext.Implicits.global import scala.concurrent.Future -object Tapir extends Endpoints { - - val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList -} +object Tapir extends Endpoints object NettyFuture { @@ -38,5 +33,5 @@ object NettyFuture { } } -object TapirServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpoints(1)) } -object TapirMultiServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpoints(128)) } +object TapirServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpointsFuture(1)) } +object TapirMultiServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpointsFuture(128)) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 4697c61ba5..047cc1668c 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -11,7 +11,7 @@ import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.pekkohttp.PekkoHttpServerInterpreter -import scala.concurrent.{ExecutionContextExecutor, Future} +import scala.concurrent.ExecutionContextExecutor object Vanilla { val router: Int => ActorSystem => Route = (nRoutes: Int) => @@ -25,21 +25,21 @@ object Vanilla { } }, post { - path(("path" + n.toString) / IntNumber) { id => + path(("path" + n.toString)) { entity(as[String]) { _ => - complete((n + id).toString) + complete((n).toString) } } }, post { - path(("pathBytes" + n.toString) / IntNumber) { id => + path(("pathBytes" + n.toString)) { entity(as[Array[Byte]]) { bytes => complete(s"Received ${bytes.length} bytes") } } }, post { - path(("pathFile" + n.toString) / IntNumber) { id => + path(("pathFile" + n.toString)) { extractRequestContext { ctx => entity(as[HttpEntity]) { httpEntity => val path = tempFilePath() @@ -58,14 +58,10 @@ object Vanilla { } object Tapir extends Endpoints { - val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - def router: Int => ActorSystem => Route = (nRoutes: Int) => (actorSystem: ActorSystem) => PekkoHttpServerInterpreter()(actorSystem.dispatcher).toRoute( - genEndpoints(nRoutes) + genEndpointsFuture(nRoutes) ) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index ea7ff12d86..a8bd314797 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -33,41 +33,39 @@ object Vanilla extends ControllerHelpers { implicit val actorSystemForMaterializer: ActorSystem = actorSystem - val simpleGet: Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { implicit request => + def simpleGet(n: Int): Action[AnyContent] = actionBuilder(PlayBodyParsers().anyContent).async { implicit request => val param = request.path.split("/").last Future.successful( - Ok(param) + Ok((n + param.toInt).toString) ) } - val postBytes: Action[ByteString] = + def postString(n: Int): Action[String] = actionBuilder(PlayBodyParsers().text(maxLength = LargeInputSize + 1024L)).async { + implicit request => + val body: String = request.body + Future.successful(Ok(s"Ok [$n], string length = ${body.length}")) + } + + def postBytes(n: Int): Action[ByteString] = actionBuilder(PlayBodyParsers().byteString(maxLength = LargeInputSize + 1024L)).async { implicit request => - val param = request.path.split("/").last - val byteArray: ByteString = request.body - Future.successful(Ok(s"$param-${byteArray.length}")) + val body: ByteString = request.body + Future.successful(Ok(s"Ok [$n], bytes length = ${body.length}")) } - val postString: Action[String] = actionBuilder(PlayBodyParsers().text(maxLength = LargeInputSize + 1024L)).async { implicit request => - val param = request.path.split("/").last - val str: String = request.body - Future.successful(Ok(s"$param-${str.length}")) - } - - val postFile: Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers().temporaryFile).async { implicit request => - val param = request.path.split("/").last - val file: Files.TemporaryFile = request.body - Future.successful(Ok(s"$param-${file.path.toString}")) + def postFile(n: Int): Action[Files.TemporaryFile] = actionBuilder(PlayBodyParsers().temporaryFile).async { implicit request => + val body: Files.TemporaryFile = request.body + Future.successful(Ok(s"Ok [$n], file saved to ${body.toPath}")) } { - case GET(p"/path$number/$param") => - simpleGet - case POST(p"/path$number/$param") => - postString - case POST(p"/pathBytes$number/$param") => - postBytes - case POST(p"/pathFile$number/$param") => - postFile + case GET(p"/path$number/$_") => + simpleGet(number.toInt) + case POST(p"/pathBytes$number") => + postBytes(number.toInt) + case POST(p"/pathFile$number") => + postFile(number.toInt) + case POST(p"/path$number") => + postString(number.toInt) } } def router: Int => ActorSystem => Routes = (nRoutes: Int) => @@ -75,15 +73,11 @@ object Vanilla extends ControllerHelpers { } object Tapir extends Endpoints { - val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - val router: Int => ActorSystem => Routes = (nRoutes: Int) => (actorSystem: ActorSystem) => { implicit val actorSystemForMaterializer: ActorSystem = actorSystem PlayServerInterpreter().toRoutes( - genEndpoints(nRoutes) + genEndpointsFuture(nRoutes) ) } } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala index 9a9a93c965..8cc8bc7e28 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -10,16 +10,10 @@ import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis.{Endpoints, ServerRunner} import sttp.tapir.server.vertx.VertxFutureServerInterpreter -import scala.concurrent.Future - object Tapir extends Endpoints { - val serverEndpointGens = replyingWithDummyStr(allEndpoints, Future.successful) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - def route: Int => Router => Route = { (nRoutes: Int) => router => val interpreter = VertxFutureServerInterpreter() - genEndpoints(nRoutes).map(interpreter.route(_)(router)).last + genEndpointsFuture(nRoutes).map(interpreter.route(_)(router)).last } } object Vanilla extends Endpoints { @@ -27,34 +21,35 @@ object Vanilla extends Endpoints { def bodyHandler = BodyHandler.create(false).setBodyLimit(LargeInputSize + 100L) def route: Int => Router => Route = { (nRoutes: Int) => router => - (0 until nRoutes).map { number => - router.get(s"/path$number/4").handler { + (0 until nRoutes).map { n => + router.get(s"/path$n/:id").handler { ctx: RoutingContext => + val id = ctx.request().getParam("id").toInt val _ = ctx .response() .putHeader("content-type", "text/plain") - .end("Ok") + .end(s"${id + n}") } - router.post(s"/path$number/4").handler(bodyHandler).handler { + router.post(s"/path$n").handler(bodyHandler).handler { ctx: RoutingContext => val body = ctx.body.asString() val _ = ctx .response() .putHeader("content-type", "text/plain") - .end(s"Ok, body length = ${body.length}") + .end(s"Ok [$n], string length = ${body.length}") } - router.post(s"/pathBytes$number/4").handler(bodyHandler).handler { + router.post(s"/pathBytes$n").handler(bodyHandler).handler { ctx: RoutingContext => val bytes = ctx.body().asString() val _ = ctx .response() .putHeader("content-type", "text/plain") - .end(s"Received ${bytes.length} bytes") + .end(s"Ok [$n], bytes length = ${bytes.length}") } - router.post(s"/pathFile$number/4").handler(bodyHandler).handler { + router.post(s"/pathFile$n").handler(bodyHandler).handler { ctx: RoutingContext => val filePath = tempFilePath() val fs = ctx.vertx.fileSystem @@ -65,7 +60,7 @@ object Vanilla extends Endpoints { ctx .response() .putHeader("content-type", "text/plain") - .end(s"Received binary stored as: $filePath") + .end(s"Ok [$n], file saved to $filePath") ) } }.last diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala index 6d26352e0f..780a0fb5a4 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala @@ -9,23 +9,19 @@ import sttp.tapir.server.vertx.cats.VertxCatsServerInterpreter import sttp.tapir.perf.vertx.VertxRunner object Tapir extends Endpoints { - val serverEndpointGens = replyingWithDummyStr(allEndpoints, IO.pure) - - def genEndpoints(i: Int) = genServerEndpoints(serverEndpointGens)(i).toList - def route(dispatcher: Dispatcher[IO]): Int => Router => Route = { (nRoutes: Int) => router => val interpreter = VertxCatsServerInterpreter(dispatcher) - genEndpoints(nRoutes).map(interpreter.route(_)(router)).last + genEndpointsIO(nRoutes).map(interpreter.route(_)(router)).last } } -class VertxCatsRunner(numRoutes: Int) extends ServerRunner { +class VertxCatsRunner(numRoutes: Int) { - override def start = + def start: IO[ServerRunner.KillSwitch] = Dispatcher.parallel[IO].allocated.flatMap { case (dispatcher, releaseDispatcher) => - VertxRunner.runServer(Tapir.route(dispatcher)(numRoutes)).map(_.flatMap(_ => releaseDispatcher)) + VertxRunner.runServer(Tapir.route(dispatcher)(numRoutes)).map(releaseVertx => releaseVertx >> releaseDispatcher) } } -object TapirServer extends VertxCatsRunner(numRoutes = 1) -object TapirMultiServer extends VertxCatsRunner(numRoutes = 1) +object TapirServer extends VertxCatsRunner(numRoutes = 1) with ServerRunner +object TapirMultiServer extends VertxCatsRunner(numRoutes = 1) with ServerRunner diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala index 9c17081765..bb43a449c7 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala @@ -18,8 +18,7 @@ object GatlingLogProcessor { val LogFileName = "simulation.log" - /** - * Searches for the last modified simulation.log in all simulation logs and calculates results. + /** Searches for the last modified simulation.log in all simulation logs and calculates results. */ def processLast(simulationName: String, serverName: String): IO[GatlingSimulationResult] = { for { @@ -31,7 +30,7 @@ object GatlingLogProcessor { .through(text.lines) .fold[State](State.initial) { (state, line) => val parts = line.split("\\s+") - if (parts.length >= 5 && parts(0) == "REQUEST") { + if (parts.length >= 5 && parts(0) == "REQUEST" && parts(3) != "Warm-Up") { val requestStartTime = parts(4).toLong val minRequestTs = state.minRequestTs.min(requestStartTime) val requestEndTime = parts(5).toLong diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 32290f4338..c08b363a64 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -3,9 +3,11 @@ package sttp.tapir.perf import io.gatling.core.Predef._ import io.gatling.core.structure.PopulationBuilder import io.gatling.http.Predef._ +import sttp.tapir.perf.Common._ +import scala.concurrent.duration._ import scala.util.Random -import sttp.tapir.perf.Common._ +import io.gatling.core.structure.ChainBuilder object CommonSimulations { private val baseUrl = "http://127.0.0.1:8080" @@ -23,16 +25,6 @@ object CommonSimulations { lazy val constRandomLongBytes = randomByteArray(LargeInputSize) lazy val constRandomLongAlphanumBytes = randomAlphanumByteArray(LargeInputSize) - def simple_get(routeNumber: Int): PopulationBuilder = { - val httpProtocol = http.baseUrl(baseUrl) - val execHttpGet = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4")) - - scenario(s"Repeatedly invoke GET of route number $routeNumber") - .during(duration)(execHttpGet) - .inject(atOnceUsers(userCount)) - .protocols(httpProtocol) - } - def getParamOpt(paramName: String): Option[String] = Option(System.getProperty(s"tapir.perf.${paramName}")) def getParam(paramName: String): String = @@ -44,15 +36,41 @@ object CommonSimulations { private lazy val userCount = getParam("user-count").toInt private lazy val duration = getParam("duration-seconds").toInt + private val httpProtocol = http.baseUrl(baseUrl) // Scenarios + val warmUpScenario = scenario("Warm-Up Scenario") + .exec( + http("HTTP GET Warm-Up") + .get("/path0/1") + .check(status.is(200)) + ) + .exec( + http("HTTP POST Warm-Up") + .post("/path0") + .body(StringBody("warmup")) + .header("Content-Type", "text/plain") + .check(status.is(200)) + ) + .inject( + constantConcurrentUsers(3).during(5.seconds) + ) + .protocols(httpProtocol) + + def scenario_simple_get(routeNumber: Int): PopulationBuilder = { + val execHttpGet: ChainBuilder = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4")) + + scenario(s"Repeatedly invoke GET of route number $routeNumber") + .during(duration)(execHttpGet) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) + } + def scenario_post_string(routeNumber: Int): PopulationBuilder = { - val httpProtocol = http.baseUrl(baseUrl) - val body = new String(randomAlphanumByteArray(256)) val execHttpPost = exec( - http(s"HTTP POST /path$routeNumber/4") - .post(s"/path$routeNumber/4") - .body(StringBody(body)) + http(s"HTTP POST /path$routeNumber") + .post(s"/path$routeNumber") + .body(StringBody(_ => new String(randomAlphanumByteArray(256)))) .header("Content-Type", "text/plain") ) @@ -63,11 +81,10 @@ object CommonSimulations { } def scenario_post_bytes(routeNumber: Int): PopulationBuilder = { - val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( - http(s"HTTP POST /pathBytes$routeNumber/4") - .post(s"/pathBytes$routeNumber/4") - .body(ByteArrayBody(randomAlphanumByteArray(256))) + http(s"HTTP POST /pathBytes$routeNumber") + .post(s"/pathBytes$routeNumber") + .body(ByteArrayBody(_ => randomAlphanumByteArray(256))) .header("Content-Type", "text/plain") // otherwise Play complains ) @@ -78,10 +95,9 @@ object CommonSimulations { } def scenario_post_file(routeNumber: Int): PopulationBuilder = { - val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( - http(s"HTTP POST /pathFile$routeNumber/4") - .post(s"/pathFile$routeNumber/4") + http(s"HTTP POST /pathFile$routeNumber") + .post(s"/pathFile$routeNumber") .body(ByteArrayBody(constRandomLongBytes)) .header("Content-Type", "application/octet-stream") ) @@ -93,10 +109,9 @@ object CommonSimulations { } def scenario_post_long_bytes(routeNumber: Int): PopulationBuilder = { - val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( - http(s"HTTP POST /pathBytes$routeNumber/4") - .post(s"/pathBytes$routeNumber/4") + http(s"HTTP POST /pathBytes$routeNumber") + .post(s"/pathBytes$routeNumber") .body(ByteArrayBody(constRandomLongAlphanumBytes)) .header("Content-Type", "text/plain") // otherwise Play complains ) @@ -108,10 +123,9 @@ object CommonSimulations { } def scenario_post_long_string(routeNumber: Int): PopulationBuilder = { - val httpProtocol = http.baseUrl(baseUrl) val execHttpPost = exec( - http(s"HTTP POST /path$routeNumber/4") - .post(s"/path$routeNumber/4") + http(s"HTTP POST /path$routeNumber") + .post(s"/path$routeNumber") .body(ByteArrayBody(constRandomLongAlphanumBytes)) .header("Content-Type", "text/plain") ) @@ -123,30 +137,32 @@ object CommonSimulations { } } +import CommonSimulations._ + class SimpleGetSimulation extends Simulation { - setUp(CommonSimulations.simple_get(0)): Unit + setUp(warmUpScenario.andThen(scenario_simple_get(0))): Unit } class SimpleGetMultiRouteSimulation extends Simulation { - setUp(CommonSimulations.simple_get(127)): Unit + setUp(warmUpScenario.andThen(scenario_simple_get(127))): Unit } class PostBytesSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_bytes(0)): Unit + setUp(warmUpScenario.andThen(scenario_post_bytes(0))): Unit } class PostLongBytesSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_long_bytes(0)): Unit + setUp(warmUpScenario.andThen(scenario_post_long_bytes(0))): Unit } class PostFileSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_file(0)): Unit + setUp(warmUpScenario.andThen(scenario_post_file(0))): Unit } class PostStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_string(0)): Unit + setUp(warmUpScenario.andThen(scenario_post_string(0))): Unit } class PostLongStringSimulation extends Simulation { - setUp(CommonSimulations.scenario_post_long_string(0)): Unit + setUp(warmUpScenario.andThen(scenario_post_long_string(0))): Unit } diff --git a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala index 5cfb836ad1..45766fd5b6 100644 --- a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala +++ b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala @@ -44,7 +44,7 @@ object NettyBootstrap { nettyConfig.socketConfig.receiveBuffer.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_RCVBUF, i)) nettyConfig.socketConfig.sendBuffer.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_SNDBUF, i)) nettyConfig.socketConfig.typeOfService.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.IP_TOS, i)) - nettyConfig.socketTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_TIMEOUT, i.toSeconds.toInt)) + // nettyConfig.socketTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_TIMEOUT, i.toSeconds.toInt)) nettyConfig.lingerTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_LINGER, i.toSeconds.toInt)) nettyConfig.connectionTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.CONNECT_TIMEOUT_MILLIS, i.toMillis.toInt) From 0188b7b97fa14f3a402c881a87350a06f84cc0e6 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 08:36:11 +0100 Subject: [PATCH 43/53] Fix warmup --- .../scala/sttp/tapir/perf/Simulations.scala | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index c08b363a64..b20d13073c 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -40,21 +40,19 @@ object CommonSimulations { // Scenarios val warmUpScenario = scenario("Warm-Up Scenario") - .exec( - http("HTTP GET Warm-Up") - .get("/path0/1") - .check(status.is(200)) - ) - .exec( - http("HTTP POST Warm-Up") - .post("/path0") - .body(StringBody("warmup")) - .header("Content-Type", "text/plain") - .check(status.is(200)) - ) - .inject( - constantConcurrentUsers(3).during(5.seconds) + .during(5.seconds)( + exec( + http("HTTP GET Warm-Up") + .get("/path0/1") + ) + .exec( + http("HTTP POST Warm-Up") + .post("/path0") + .body(StringBody("warmup")) + .header("Content-Type", "text/plain") + ) ) + .inject(atOnceUsers(3)) .protocols(httpProtocol) def scenario_simple_get(routeNumber: Int): PopulationBuilder = { From 818c9538b6bc03ab92fc6029408c1a951989f3b9 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 11:29:22 +0100 Subject: [PATCH 44/53] Include warmup in total duration estimations --- perf-tests/src/main/scala/sttp/tapir/perf/Common.scala | 3 +++ .../src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala | 2 +- perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index b9f805a664..5d9a24cc56 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -3,11 +3,14 @@ package sttp.tapir.perf import java.io.File import java.nio.file.Path import java.util.Date + +import scala.concurrent.duration._ import scala.util.Random object Common { val rootPackage = "sttp.tapir.perf" val LargeInputSize = 5 * 1024 * 1024 + val WarmupDuration = 5.seconds val Port = 8080 val TmpDir: File = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile def tempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}") diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala index fa2130d19e..1a699e5128 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -27,7 +27,7 @@ case class PerfTestSuiteParams( def totalTests: Int = shortServerNames.length * shortSimulationNames.length - def minTotalDuration: FiniteDuration = (duration * totalTests.toLong).toMinutes.minutes + def minTotalDuration: FiniteDuration = ((duration + WarmupDuration) * totalTests.toLong).toMinutes.minutes /** Returns pairs of (fullServerName, shortServerName), for example: (sttp.tapir.perf.pekko.TapirServer, pekko.Tapir) */ diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index b20d13073c..4952055caf 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -40,7 +40,7 @@ object CommonSimulations { // Scenarios val warmUpScenario = scenario("Warm-Up Scenario") - .during(5.seconds)( + .during(WarmupDuration)( exec( http("HTTP GET Warm-Up") .get("/path0/1") From a4ceeb96932dba69329a85939c089c9e4206d99d Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 12:21:25 +0100 Subject: [PATCH 45/53] Restore --- .../scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala index 45766fd5b6..5cfb836ad1 100644 --- a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala +++ b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/NettyBootstrap.scala @@ -44,7 +44,7 @@ object NettyBootstrap { nettyConfig.socketConfig.receiveBuffer.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_RCVBUF, i)) nettyConfig.socketConfig.sendBuffer.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_SNDBUF, i)) nettyConfig.socketConfig.typeOfService.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.IP_TOS, i)) - // nettyConfig.socketTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_TIMEOUT, i.toSeconds.toInt)) + nettyConfig.socketTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_TIMEOUT, i.toSeconds.toInt)) nettyConfig.lingerTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.SO_LINGER, i.toSeconds.toInt)) nettyConfig.connectionTimeout.foreach(i => httpBootstrap.childOption[java.lang.Integer](ChannelOption.CONNECT_TIMEOUT_MILLIS, i.toMillis.toInt) From cb814cbbd57f57d048af7002a3f6606216697bdd Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 16:08:14 +0100 Subject: [PATCH 46/53] Display default values of optional parameters --- .../scala/sttp/tapir/perf/PerfTestSuiteParams.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala index 1a699e5128..4c3378b491 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -10,8 +10,8 @@ import scala.util.Success case class PerfTestSuiteParams( shortServerNames: List[String] = Nil, shortSimulationNames: List[String] = Nil, - users: Int = 1, - durationSeconds: Int = 10, + users: Int = PerfTestSuiteParams.defaultUserCount, + durationSeconds: Int = PerfTestSuiteParams.defaultDurationSeconds, buildGatlingReports: Boolean = false ) { def adjustWildcards: PerfTestSuiteParams = { @@ -40,6 +40,8 @@ case class PerfTestSuiteParams( } object PerfTestSuiteParams { + val defaultUserCount = 1 + val defaultDurationSeconds = 10 val builder = OParser.builder[PerfTestSuiteParams] import builder._ val argParser = OParser.sequence( @@ -54,13 +56,13 @@ object PerfTestSuiteParams { .text("Comma-separated list of short simulation names, or '*' for all"), opt[Int]('u', "users") .action((x, c) => c.copy(users = x)) - .text("Number of concurrent users"), + .text(s"Number of concurrent users, default is $defaultUserCount"), opt[Int]('d', "duration") .action((x, c) => c.copy(durationSeconds = x)) - .text("Single simulation duration in seconds"), + .text(s"Single simulation duration in seconds, default is $defaultDurationSeconds"), opt[Unit]('g', "gatling-reports") .action((_, c) => c.copy(buildGatlingReports = true)) - .text("Generate Gatling reports for individuals sims, may significantly affect total time") + .text("Generate Gatling reports for individuals sims, may significantly affect total time (disabled by default)") ) def parse(args: List[String]): PerfTestSuiteParams = { @@ -77,5 +79,4 @@ object PerfTestSuiteParams { sys.exit(-1) } } - } From f8dbbc1aadaca49b3bbb359b71bca874ba53bcde Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 16:13:07 +0100 Subject: [PATCH 47/53] Disable logging --- .../src/main/scala/sttp/tapir/perf/Common.scala | 2 +- .../scala/sttp/tapir/perf/http4s/Http4s.scala | 9 +++++++-- .../sttp/tapir/perf/netty/cats/NettyCats.scala | 15 ++++++++------- .../tapir/perf/netty/future/NettyFuture.scala | 7 ++++--- .../scala/sttp/tapir/perf/pekko/PekkoHttp.scala | 13 +++++++++---- .../main/scala/sttp/tapir/perf/play/Play.scala | 5 ++++- .../main/scala/sttp/tapir/perf/vertx/Vertx.scala | 6 ++++-- .../sttp/tapir/perf/vertx/cats/VertxCats.scala | 6 ++++-- 8 files changed, 41 insertions(+), 22 deletions(-) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index 5d9a24cc56..ef90fa1322 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -13,6 +13,6 @@ object Common { val WarmupDuration = 5.seconds val Port = 8080 val TmpDir: File = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile - def tempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}") + def newTempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}") } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index d9e7904fbb..81fb4b342c 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -12,6 +12,7 @@ import sttp.tapir.integ.cats.effect.CatsMonadError import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.http4s.Http4sServerInterpreter +import sttp.tapir.server.http4s.Http4sServerOptions object Vanilla { val router: Int => HttpRoutes[IO] = (nRoutes: Int) => @@ -32,7 +33,7 @@ object Vanilla { Ok(s"Ok [$n], bytes length = ${bytes.length}") } case req @ POST -> Root / s"pathFile$n" => - val filePath = tempFilePath() + val filePath = newTempFilePath() val sink = Files[IO].writeAll(Fs2Path.fromNioPath(filePath)) req.body .through(sink) @@ -49,10 +50,14 @@ object Tapir extends Endpoints { implicit val mErr: MonadError[IO] = new CatsMonadError[IO] + val serverOptions = Http4sServerOptions + .customiseInterceptors[IO] + .serverLog(None) + .options val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router("/" -> { - Http4sServerInterpreter[IO]().toRoutes( + Http4sServerInterpreter[IO](serverOptions).toRoutes( genEndpointsIO(nRoutes) ) }) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala index 0612430957..cb24e73c8a 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala @@ -2,10 +2,12 @@ package sttp.tapir.perf.netty.cats import cats.effect.IO import cats.effect.kernel.Resource +import cats.effect.std.Dispatcher import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.ServerEndpoint import sttp.tapir.server.netty.cats.NettyCatsServer +import sttp.tapir.server.netty.cats.NettyCatsServerOptions object Tapir extends Endpoints @@ -14,10 +16,11 @@ object NettyCats { def runServer(endpoints: List[ServerEndpoint[Any, IO]]): IO[ServerRunner.KillSwitch] = { val declaredPort = Port val declaredHost = "0.0.0.0" - // Starting netty server - NettyCatsServer - .io() - .flatMap { server => + (for { + dispatcher <- Dispatcher.parallel[IO] + serverOptions = NettyCatsServerOptions.customiseInterceptors(dispatcher).serverLog(None).options + server <- NettyCatsServer.io() + _ <- Resource.make( server .port(declaredPort) @@ -25,9 +28,7 @@ object NettyCats { .addEndpoints(endpoints) .start() )(binding => binding.stop()) - } - .allocated - .map(_._2) + } yield ()).allocated.map(_._2) } } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala index d405adc7e3..fe15d8957c 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala @@ -3,25 +3,26 @@ package sttp.tapir.perf.netty.future import cats.effect.IO import sttp.tapir.perf.apis._ import sttp.tapir.perf.Common._ -import sttp.tapir.server.netty.{NettyFutureServer, NettyFutureServerBinding} +import sttp.tapir.server.netty.{NettyFutureServer, NettyFutureServerBinding, NettyFutureServerOptions} import sttp.tapir.server.ServerEndpoint import scala.concurrent.ExecutionContext import ExecutionContext.Implicits.global import scala.concurrent.Future -object Tapir extends Endpoints +object Tapir extends Endpoints object NettyFuture { def runServer(endpoints: List[ServerEndpoint[Any, Future]]): IO[ServerRunner.KillSwitch] = { val declaredPort = Port val declaredHost = "0.0.0.0" + val serverOptions = NettyFutureServerOptions.customiseInterceptors.serverLog(None).options // Starting netty server val serverBinding: IO[NettyFutureServerBinding] = IO.fromFuture( IO( - NettyFutureServer() + NettyFutureServer(serverOptions) .port(declaredPort) .host(declaredHost) .addEndpoints(endpoints) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala index 047cc1668c..ad6d4f200c 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala @@ -9,9 +9,9 @@ import org.apache.pekko.http.scaladsl.server.Route import org.apache.pekko.stream.scaladsl.FileIO import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ -import sttp.tapir.server.pekkohttp.PekkoHttpServerInterpreter +import sttp.tapir.server.pekkohttp.{PekkoHttpServerInterpreter, PekkoHttpServerOptions} -import scala.concurrent.ExecutionContextExecutor +import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} object Vanilla { val router: Int => ActorSystem => Route = (nRoutes: Int) => @@ -42,7 +42,7 @@ object Vanilla { path(("pathFile" + n.toString)) { extractRequestContext { ctx => entity(as[HttpEntity]) { httpEntity => - val path = tempFilePath() + val path = newTempFilePath() val sink = FileIO.toPath(path) val finishedWriting = httpEntity.dataBytes.runWith(sink)(ctx.materializer) onSuccess(finishedWriting) { _ => @@ -58,9 +58,14 @@ object Vanilla { } object Tapir extends Endpoints { + val serverOptions = PekkoHttpServerOptions + .customiseInterceptors(ExecutionContext.Implicits.global) + .serverLog(None) + .options + def router: Int => ActorSystem => Route = (nRoutes: Int) => (actorSystem: ActorSystem) => - PekkoHttpServerInterpreter()(actorSystem.dispatcher).toRoute( + PekkoHttpServerInterpreter(serverOptions)(actorSystem.dispatcher).toRoute( genEndpointsFuture(nRoutes) ) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala index a8bd314797..7b226b04a9 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala @@ -13,6 +13,7 @@ import play.core.server.{DefaultPekkoHttpServerComponents, ServerConfig} import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis._ import sttp.tapir.server.play.PlayServerInterpreter +import sttp.tapir.server.play.PlayServerOptions import scala.concurrent.{ExecutionContext, Future} @@ -76,7 +77,9 @@ object Tapir extends Endpoints { val router: Int => ActorSystem => Routes = (nRoutes: Int) => (actorSystem: ActorSystem) => { implicit val actorSystemForMaterializer: ActorSystem = actorSystem - PlayServerInterpreter().toRoutes( + implicit val ec: ExecutionContext = actorSystem.dispatcher + val serverOptions = PlayServerOptions.customiseInterceptors().serverLog(None).options + PlayServerInterpreter(serverOptions).toRoutes( genEndpointsFuture(nRoutes) ) } diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala index 8cc8bc7e28..d6cffbe438 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala @@ -9,10 +9,12 @@ import io.vertx.ext.web.{Route, Router, RoutingContext} import sttp.tapir.perf.Common._ import sttp.tapir.perf.apis.{Endpoints, ServerRunner} import sttp.tapir.server.vertx.VertxFutureServerInterpreter +import sttp.tapir.server.vertx.VertxFutureServerOptions object Tapir extends Endpoints { def route: Int => Router => Route = { (nRoutes: Int) => router => - val interpreter = VertxFutureServerInterpreter() + val serverOptions = VertxFutureServerOptions.customiseInterceptors.serverLog(None).options + val interpreter = VertxFutureServerInterpreter(serverOptions) genEndpointsFuture(nRoutes).map(interpreter.route(_)(router)).last } } @@ -51,7 +53,7 @@ object Vanilla extends Endpoints { router.post(s"/pathFile$n").handler(bodyHandler).handler { ctx: RoutingContext => - val filePath = tempFilePath() + val filePath = newTempFilePath() val fs = ctx.vertx.fileSystem val _ = fs .createFile(filePath.toString) diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala index 780a0fb5a4..828faf92f7 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala @@ -5,12 +5,14 @@ import cats.effect.std.Dispatcher import io.vertx.ext.web.Route import io.vertx.ext.web.Router import sttp.tapir.perf.apis.{Endpoints, ServerRunner} -import sttp.tapir.server.vertx.cats.VertxCatsServerInterpreter import sttp.tapir.perf.vertx.VertxRunner +import sttp.tapir.server.vertx.cats.VertxCatsServerInterpreter +import sttp.tapir.server.vertx.cats.VertxCatsServerOptions object Tapir extends Endpoints { def route(dispatcher: Dispatcher[IO]): Int => Router => Route = { (nRoutes: Int) => router => - val interpreter = VertxCatsServerInterpreter(dispatcher) + val serverOptions = VertxCatsServerOptions.customiseInterceptors[IO](dispatcher).serverLog(None).options + val interpreter = VertxCatsServerInterpreter(serverOptions) genEndpointsIO(nRoutes).map(interpreter.route(_)(router)).last } } From 1bf64420c1276364816d7ddb7bcb344a9a26b3e4 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 16:41:33 +0100 Subject: [PATCH 48/53] List available servers and sims in the help view --- .../src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala index 4c3378b491..f31285db5b 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala @@ -49,11 +49,11 @@ object PerfTestSuiteParams { opt[Seq[String]]('s', "server") .required() .action((x, c) => c.copy(shortServerNames = x.toList)) - .text("Comma-separated list of short server names, or '*' for all"), + .text(s"Comma-separated list of short server names, or '*' for all. Available servers: ${TypeScanner.allServers.mkString(", ")}"), opt[Seq[String]]('m', "sim") .required() .action((x, c) => c.copy(shortSimulationNames = x.toList)) - .text("Comma-separated list of short simulation names, or '*' for all"), + .text(s"Comma-separated list of short simulation names, or '*' for all. Available simulations: ${TypeScanner.allSimulations.mkString(", ")}"), opt[Int]('u', "users") .action((x, c) => c.copy(users = x)) .text(s"Number of concurrent users, default is $defaultUserCount"), From 76a5e4ba92f8d033c4c8848d91f0612a1e1bed85 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 16:41:49 +0100 Subject: [PATCH 49/53] Documentation --- perf-tests/README.md | 70 +++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/perf-tests/README.md b/perf-tests/README.md index 0103ab3fca..b3a50ec40d 100644 --- a/perf-tests/README.md +++ b/perf-tests/README.md @@ -1,34 +1,70 @@ -# seperate testing +# Performance tests -To start a server, run `perfTests/run` and select the server you want to test, or in a single command: +Performance tests are executed by running `PerfTestSuiteRunner`, which is a standard "Main" Scala application, configured by command line parameters. It executes a sequence of tests, where +each test consist of: + +1. Starting a HTTP server (Like Tapir-based Pekko, Vartx, http4s, or a "vanilla", tapirless one) +2. Sending a bunch of warmup requests +3. Sending simulation-specific requests +4. Closing the server + +The sequence is repeated for a set of servers multiplied by simulations, all configurable as arguments. Command parameters can be viewed by running: ``` -sbt "perfTests/runMain sttp.tapir.perf.akka.VanillaMultiServer" -// or -sbt "perfTests/runMain sttp.tapir.perf.akka.TapirMultiServer" -// or others ... +perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner ``` -Then run the test: +which displays help similar to: + ``` -sbt "perfTests/Gatling/testOnly sttp.tapir.perf.OneRouteSimulation" -// or -sbt "perfTests/Gatling/testOnly sttp.tapir.perf.MultiRouteSimulation" +[error] Usage: perf [options] +[error] -s, --server Comma-separated list of short server names, or '*' for all. Available servers: http4s.TapirMulti, http4s.Tapir, http4s.VanillaMulti, http4s.Vanilla, +netty.cats.TapirMulti, netty.cats.Tapir, netty.future.TapirMulti, netty.future.Tapir, pekko.TapirMulti, pekko.Tapir, pekko.VanillaMulti, pekko.Vanilla, play.TapirMulti, play.Tapir, +play.VanillaMulti, play.Vanilla, vertx.TapirMulti, vertx.Tapir, vertx.VanillaMulti, vertx.Vanilla, vertx.cats.TapirMulti, vertx.cats.Tapir +[error] -m, --sim Comma-separated list of short simulation names, or '*' for all. Available simulations: PostBytes, PostFile, PostLongBytes, PostLongString, +PostString, SimpleGetMultiRoute, SimpleGet +[error] -u, --users Number of concurrent users, default is 1 +[error] -d, --duration Single simulation duration in seconds, default is 10 +[error] -g, --gatling-reports Generate Gatling reports for individuals sims, may significantly affect total time (disabled by default) ``` -This method yields the most performant results, but requires running the commands in two seperate sbt instacnes. +## Examples -# single-command testing +1. Run all sims on all servers with other options set to default (Careful, may take quite some time!): +``` +perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner -s * -m * +``` -To run the server together with a test, simply: +2. Run all sims on http4s servers, with each simulation running for 5 seconds: +``` +perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner -s http4s.Tapir,http4s.TapirMulti,http4s.Vanilla,http4s.VanillaMulti -s * -d 5 +``` +3. Run some simulations on some servers, with 3 concurrent users instead of default 1, each simulation running for 15 seconds, +and enabled Gatling report generation: ``` -perfTests/akkaHttpVanilla +perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner -s http4s.Tapir,netty.future.Tapir,play.Tapir -s PostLongBytes,PostFile -d 15 -u 3 -g ``` -or + +## Reports + +After all tests finish successfully, your console output will point to report files, +containing aggregated results from the entire suite: ``` -perfTests/akkaHttpTapir +[info] ******* Test Suite report saved to /home/kc/code/oss/tapir/.sbt/matrix/perfTests/tapir-perf-tests-2024-01-22_16_33_14.csv +[info] ******* Test Suite report saved to /home/kc/code/oss/tapir/.sbt/matrix/perfTests/tapir-perf-tests-2024-01-22_16_33_14.html ``` -Servers under this method are slightly less performant, but do not need to be run from seperate terminals. The performance loss doesn't seem to affect the relative performance of different servers. +These reports include information about throughput and latency of each server for each simulation. + +How the aggregation works: After each test the results are read from `simulation.log` produced by Gatling and aggregated by `GatlingLogProcessor`. +Entires related to warm-up process are not counted. The processor then uses 'com.codehale.metrics.Histogram' to calculate +p99, p95, p75, and p50 percentiles for latencies of all requests sent during the simulation. + +## Adding new servers and simulations + +To add a new server, go to `src/main/scala` and put an object extending `sttp.tapir.perf.apis.ServerRunner` in a subpackage of `sttp.tapir.perf`. +It should be automatically resoled by the `TypeScanner` utility used by the `PerfTestSuiteRunner`. +Similarly with simulations. Go to `src/test/scala` and a class extending `io.gatling.core.Predef.Simulation` under `sttp.tapir.perf`. See the `Simulations.scala` +file for examples. From 6afe52fca729afb6d333374432147b59fb299273 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 16:46:57 +0100 Subject: [PATCH 50/53] Remove accidental double release after merge --- .../netty/internal/reactivestreams/FileWriterSubscriber.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala index 868e5500a5..28dfbb6d87 100644 --- a/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala +++ b/server/netty-server/src/main/scala/sttp/tapir/server/netty/internal/reactivestreams/FileWriterSubscriber.scala @@ -44,14 +44,12 @@ class FileWriterSubscriber(path: Path) extends PromisingSubscriber[Unit, HttpCon override def completed(result: Integer, attachment: Unit): Unit = { httpContent.release() position += result - httpContent.release() subscription.request(1) } override def failed(exc: Throwable, attachment: Unit): Unit = { httpContent.release() subscription.cancel() - httpContent.release() onError(exc) } } From f0f9f85defa229287b3f183b3fb07b7c9ff2bbeb Mon Sep 17 00:00:00 2001 From: kciesielski Date: Mon, 22 Jan 2024 16:50:01 +0100 Subject: [PATCH 51/53] Less personal example ;) --- perf-tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-tests/README.md b/perf-tests/README.md index b3a50ec40d..00e5e7b0bf 100644 --- a/perf-tests/README.md +++ b/perf-tests/README.md @@ -51,8 +51,8 @@ perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner -s http4s.Tapir,netty After all tests finish successfully, your console output will point to report files, containing aggregated results from the entire suite: ``` -[info] ******* Test Suite report saved to /home/kc/code/oss/tapir/.sbt/matrix/perfTests/tapir-perf-tests-2024-01-22_16_33_14.csv -[info] ******* Test Suite report saved to /home/kc/code/oss/tapir/.sbt/matrix/perfTests/tapir-perf-tests-2024-01-22_16_33_14.html +[info] ******* Test Suite report saved to /home/alice/projects/tapir/.sbt/matrix/perfTests/tapir-perf-tests-2024-01-22_16_33_14.csv +[info] ******* Test Suite report saved to /home/alice/projects/tapir/.sbt/matrix/perfTests/tapir-perf-tests-2024-01-22_16_33_14.html ``` These reports include information about throughput and latency of each server for each simulation. From 18c11df01a5808838148e73bba00bcb49934c7ef Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 23 Jan 2024 09:56:15 +0100 Subject: [PATCH 52/53] Restore ideSkipProject --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d19f7b0a08..1dbf7ec4a1 100644 --- a/build.sbt +++ b/build.sbt @@ -68,7 +68,7 @@ val commonSettings = commonSmlBuildSettings ++ ossPublishSettings ++ Seq( }.value, mimaPreviousArtifacts := Set.empty, // we only use MiMa for `core` for now, using enableMimaSettings ideSkipProject := (scalaVersion.value == scala2_12) || - (scalaVersion.value == scala3) || + (scalaVersion.value == scala2_13) || thisProjectRef.value.project.contains("Native") || thisProjectRef.value.project.contains("JS"), bspEnabled := !ideSkipProject.value, From 8904ef2850b6d0d6376d4b3ede0a2f23bebe1261 Mon Sep 17 00:00:00 2001 From: kciesielski Date: Tue, 23 Jan 2024 09:59:52 +0100 Subject: [PATCH 53/53] Add warmup to the info screen --- .../src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index bf2ff938d1..ca724039b2 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -5,6 +5,7 @@ import cats.syntax.all._ import fs2.io.file import fs2.text import sttp.tapir.perf.apis.ServerRunner +import sttp.tapir.perf.Common._ import java.nio.file.Paths import java.time.LocalDateTime @@ -25,7 +26,8 @@ object PerfTestSuiteRunner extends IOApp { System.setProperty("tapir.perf.user-count", params.users.toString) System.setProperty("tapir.perf.duration-seconds", params.durationSeconds.toString) println("===========================================================================================") - println(s"Running a suite of ${params.totalTests} tests, each for ${params.users} users and ${params.duration}") + println(s"Running a suite of ${params.totalTests} tests, each for ${params.users} users and ${params.duration}.") + println(s"Additional warm-up phase of $WarmupDuration will be performed before each simulation.") println(s"Servers: ${params.shortServerNames}") println(s"Simulations: ${params.shortSimulationNames}") println(s"Expected total duration: at least ${params.minTotalDuration}")