From 9fb1a10adafcddbf69ab15249b9c822cdff7b8da Mon Sep 17 00:00:00 2001 From: YukioZzz <424114284@qq.com> Date: Mon, 24 Oct 2022 12:58:34 +0200 Subject: [PATCH] Add an alternative locust client (#46) * add locust client as an alternative * add a feature: retry if FAIL_RATIO's high * provide both httpclient and grpcclient * Add README.md of the locust client --- pkg/client_locust/Dockerfile | 27 ++ pkg/client_locust/README.md | 41 ++ pkg/client_locust/api_pb2.py | 426 ++++++++++++++++++ pkg/client_locust/api_pb2_grpc.py | 123 +++++ pkg/client_locust/image.jpg | Bin 0 -> 13007 bytes pkg/client_locust/invokust/__init__.py | 4 + pkg/client_locust/invokust/loadtest.py | 153 +++++++ .../invokust/prometheus_exporter.py | 110 +++++ pkg/client_locust/invokust/settings.py | 96 ++++ pkg/client_locust/jobtemplate.yaml | 41 ++ pkg/client_locust/locust_grpc.py | 86 ++++ pkg/client_locust/locustfile_grpcuser.py | 103 +++++ pkg/client_locust/locustfile_httpuser.py | 14 + pkg/client_locust/morphling_client_locust.py | 87 ++++ pkg/client_locust/requirements.txt | 51 +++ 15 files changed, 1362 insertions(+) create mode 100644 pkg/client_locust/Dockerfile create mode 100644 pkg/client_locust/README.md create mode 100644 pkg/client_locust/api_pb2.py create mode 100644 pkg/client_locust/api_pb2_grpc.py create mode 100644 pkg/client_locust/image.jpg create mode 100644 pkg/client_locust/invokust/__init__.py create mode 100644 pkg/client_locust/invokust/loadtest.py create mode 100644 pkg/client_locust/invokust/prometheus_exporter.py create mode 100644 pkg/client_locust/invokust/settings.py create mode 100644 pkg/client_locust/jobtemplate.yaml create mode 100644 pkg/client_locust/locust_grpc.py create mode 100644 pkg/client_locust/locustfile_grpcuser.py create mode 100644 pkg/client_locust/locustfile_httpuser.py create mode 100644 pkg/client_locust/morphling_client_locust.py create mode 100644 pkg/client_locust/requirements.txt diff --git a/pkg/client_locust/Dockerfile b/pkg/client_locust/Dockerfile new file mode 100644 index 0000000..926b037 --- /dev/null +++ b/pkg/client_locust/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.6 + +RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \ + apt-get -y update && \ + apt-get -y install gfortran libopenblas-dev liblapack-dev && \ + pip install cython 'numpy>=1.13.3'; \ + fi +RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \ + if [ "$(uname -m)" = "ppc64le" ]; then \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \ + elif [ "$(uname -m)" = "aarch64" ]; then \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \ + else \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64; \ + fi && \ + chmod +x /bin/grpc_health_probe + +WORKDIR /workspace +ADD requirements.txt requirements.txt +RUN /usr/local/bin/python -m pip install --upgrade pip +#RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +ADD . /workspace + +ENV BATCH_SIZE 1 +ENTRYPOINT ["/bin/sh", "-c", "sleep 100000"] diff --git a/pkg/client_locust/README.md b/pkg/client_locust/README.md new file mode 100644 index 0000000..bae27da --- /dev/null +++ b/pkg/client_locust/README.md @@ -0,0 +1,41 @@ +## Motivation +As we might want the client to perform a more customized load test to the inference server in a more simple and flexible way, an idea is to use locust as the backend so that the user can directly specify load test params or provide its own locusttest script. A demo client image based on locust is provided here. + +## File structure description +Three meta files describe how to build and use the image. + +- `README.md` +- `Dockerfile`: used to build the client image +- `jobtemplate.yaml`: a trial client job template using `morphling-client-plugin` image + +The other files will be used in the client image. + +- `requirements.txt`: python scripts dependencies +- `morphling_client_locust.py`: main function, responsible for args/env parsing, locust invocation and result saving. When the error rate exceeds `FAIL_RATE`, it will retry with a lower `LOCUST_NUM_USERS` +- `invokust`: locust wrapper，responsible for launching the pressure test and returning the results. As locust itself does not provide a complete library API, and is more often called as a CLI tool, a wrapper is needed here. In addition, a new feature which supports prometheus metrics export is added. +- `locust_grpc.py`: constructed a `GrpcUser` super class, easy to be inherited and extended by the user. It will automatically record the results via the event hook `events.request.fire`. An error threshold exit mechanism is also enabled here by the event listener. +- `locustfile_grpcuser.py` or `locustfile_httpuser.py`: the load test demo script, reserved models are provided which can be used directedly; when cusomization is needed, what we need to do is just to inherit class `GrpcUser` or the native `HttpUser`, and define the function decorated by `@task` +- `api_pb2.py` and `api_pb2_grpc.py`: used for database connection +- `image.jpg`: used for the demo load test + +As described above, the client's functions are decoupled into several modules/files as a plug-in approach. The user can just specify the param or provids its own locustfile without effort. gRPC and HTTP protocols are both supported. + +## Usage +Currently, we can specify test parameters via ENV. + +The exposed APIs are as follows, see examples in the `jobtemplate.yaml`: + +- `LOCUST_NUM_USERS`: maximum number of concurrent users, default: 10 +- `LOCUST_SPAWN_RATE`: spawning rate of concurrent users, default: 10 +- `LOCUST_RUN_TIME`: test running duration, default: 15 +- `LOCUST_LOCUSTFILE`: custom test file name, default: locustfile.py +- `LOCUST_METRICS_EXPORT`: export Prometheus metrics or not, default: False +- `FAIL_RATIO`: fail ratio threshold, default: 0.2 +- `PRINTLOG`: print log or not, default: false + +## Conclusion / Pro and Cons +To avoid reinvent the wheels, the open-source locust load testing tool is chosen here because of its ease of use and good scalability. It supports HTTP/gRPC and many other protocols and at the same time it can be extended easily to perform distributed load testing. Therefore, compared with the heavyweight Jmeter and lightweight K6, which is hard to implement complex requests, locust is more suitable. +But there are also some drawbacks. + +- locust itself does not provide a complete Library API and is mostly called as a CLI tool. Therefore, the invokust locust wrapper is used here, which is responsible for launching the load test and returning the results. However, only local runner mode is currently supported. The distributed load tests are not supported now. Since in most cases the QPS is less than 1000 for DL inference tasks on CPU, the pressure generated by the local runner is sufficient. +- QPS results with SLO guarantee is currently not supported yet. Most of the pressure testing tools do not have this option. It might be realized in the future by the same event hook mechanism as `FAIL_RATIO` retry strategy or by filtering the final qps result. diff --git a/pkg/client_locust/api_pb2.py b/pkg/client_locust/api_pb2.py new file mode 100644 index 0000000..938a0bd --- /dev/null +++ b/pkg/client_locust/api_pb2.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: api.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="api.proto", + package="api.storage", + syntax="proto3", + serialized_options=b"Z\022../grpc_storage/go", + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\tapi.proto\x12\x0b\x61pi.storage"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t"\x11\n\x0fSaveResultReply"b\n\x11SaveResultRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntrial_name\x18\x02 \x01(\t\x12&\n\x07results\x18\x04 \x03(\x0b\x32\x15.api.storage.KeyValue"9\n\x10GetResultRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntrial_name\x18\x02 \x01(\t"_\n\x0eGetResultReply\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x12\n\ntrial_name\x18\x02 \x01(\t\x12&\n\x07results\x18\x04 \x03(\x0b\x32\x15.api.storage.KeyValue2\x99\x01\n\x02\x44\x42\x12J\n\nSaveResult\x12\x1e.api.storage.SaveResultRequest\x1a\x1c.api.storage.SaveResultReply\x12G\n\tGetResult\x12\x1d.api.storage.GetResultRequest\x1a\x1b.api.storage.GetResultReplyB\x14Z\x12../grpc_storage/gob\x06proto3', +) + + +_KEYVALUE = _descriptor.Descriptor( + name="KeyValue", + full_name="api.storage.KeyValue", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="key", + full_name="api.storage.KeyValue.key", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="value", + full_name="api.storage.KeyValue.value", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=26, + serialized_end=64, +) + + +_SAVERESULTREPLY = _descriptor.Descriptor( + name="SaveResultReply", + full_name="api.storage.SaveResultReply", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=66, + serialized_end=83, +) + + +_SAVERESULTREQUEST = _descriptor.Descriptor( + name="SaveResultRequest", + full_name="api.storage.SaveResultRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="namespace", + full_name="api.storage.SaveResultRequest.namespace", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="trial_name", + full_name="api.storage.SaveResultRequest.trial_name", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="results", + full_name="api.storage.SaveResultRequest.results", + index=2, + number=4, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=85, + serialized_end=183, +) + + +_GETRESULTREQUEST = _descriptor.Descriptor( + name="GetResultRequest", + full_name="api.storage.GetResultRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="namespace", + full_name="api.storage.GetResultRequest.namespace", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="trial_name", + full_name="api.storage.GetResultRequest.trial_name", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=185, + serialized_end=242, +) + + +_GETRESULTREPLY = _descriptor.Descriptor( + name="GetResultReply", + full_name="api.storage.GetResultReply", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="namespace", + full_name="api.storage.GetResultReply.namespace", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="trial_name", + full_name="api.storage.GetResultReply.trial_name", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="results", + full_name="api.storage.GetResultReply.results", + index=2, + number=4, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=244, + serialized_end=339, +) + +_SAVERESULTREQUEST.fields_by_name["results"].message_type = _KEYVALUE +_GETRESULTREPLY.fields_by_name["results"].message_type = _KEYVALUE +DESCRIPTOR.message_types_by_name["KeyValue"] = _KEYVALUE +DESCRIPTOR.message_types_by_name["SaveResultReply"] = _SAVERESULTREPLY +DESCRIPTOR.message_types_by_name["SaveResultRequest"] = _SAVERESULTREQUEST +DESCRIPTOR.message_types_by_name["GetResultRequest"] = _GETRESULTREQUEST +DESCRIPTOR.message_types_by_name["GetResultReply"] = _GETRESULTREPLY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +KeyValue = _reflection.GeneratedProtocolMessageType( + "KeyValue", + (_message.Message,), + { + "DESCRIPTOR": _KEYVALUE, + "__module__": "api_pb2" + # @@protoc_insertion_point(class_scope:api.storage.KeyValue) + }, +) +_sym_db.RegisterMessage(KeyValue) + +SaveResultReply = _reflection.GeneratedProtocolMessageType( + "SaveResultReply", + (_message.Message,), + { + "DESCRIPTOR": _SAVERESULTREPLY, + "__module__": "api_pb2" + # @@protoc_insertion_point(class_scope:api.storage.SaveResultReply) + }, +) +_sym_db.RegisterMessage(SaveResultReply) + +SaveResultRequest = _reflection.GeneratedProtocolMessageType( + "SaveResultRequest", + (_message.Message,), + { + "DESCRIPTOR": _SAVERESULTREQUEST, + "__module__": "api_pb2" + # @@protoc_insertion_point(class_scope:api.storage.SaveResultRequest) + }, +) +_sym_db.RegisterMessage(SaveResultRequest) + +GetResultRequest = _reflection.GeneratedProtocolMessageType( + "GetResultRequest", + (_message.Message,), + { + "DESCRIPTOR": _GETRESULTREQUEST, + "__module__": "api_pb2" + # @@protoc_insertion_point(class_scope:api.storage.GetResultRequest) + }, +) +_sym_db.RegisterMessage(GetResultRequest) + +GetResultReply = _reflection.GeneratedProtocolMessageType( + "GetResultReply", + (_message.Message,), + { + "DESCRIPTOR": _GETRESULTREPLY, + "__module__": "api_pb2" + # @@protoc_insertion_point(class_scope:api.storage.GetResultReply) + }, +) +_sym_db.RegisterMessage(GetResultReply) + + +DESCRIPTOR._options = None + +_DB = _descriptor.ServiceDescriptor( + name="DB", + full_name="api.storage.DB", + file=DESCRIPTOR, + index=0, + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_start=342, + serialized_end=495, + methods=[ + _descriptor.MethodDescriptor( + name="SaveResult", + full_name="api.storage.DB.SaveResult", + index=0, + containing_service=None, + input_type=_SAVERESULTREQUEST, + output_type=_SAVERESULTREPLY, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name="GetResult", + full_name="api.storage.DB.GetResult", + index=1, + containing_service=None, + input_type=_GETRESULTREQUEST, + output_type=_GETRESULTREPLY, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + ], +) +_sym_db.RegisterServiceDescriptor(_DB) + +DESCRIPTOR.services_by_name["DB"] = _DB + +# @@protoc_insertion_point(module_scope) diff --git a/pkg/client_locust/api_pb2_grpc.py b/pkg/client_locust/api_pb2_grpc.py new file mode 100644 index 0000000..2061f4e --- /dev/null +++ b/pkg/client_locust/api_pb2_grpc.py @@ -0,0 +1,123 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import api_pb2 as api__pb2 +import grpc + + +class DBStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.SaveResult = channel.unary_unary( + "/api.storage.DB/SaveResult", + request_serializer=api__pb2.SaveResultRequest.SerializeToString, + response_deserializer=api__pb2.SaveResultReply.FromString, + ) + self.GetResult = channel.unary_unary( + "/api.storage.DB/GetResult", + request_serializer=api__pb2.GetResultRequest.SerializeToString, + response_deserializer=api__pb2.GetResultReply.FromString, + ) + + +class DBServicer(object): + """Missing associated documentation comment in .proto file.""" + + def SaveResult(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def GetResult(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_DBServicer_to_server(servicer, server): + rpc_method_handlers = { + "SaveResult": grpc.unary_unary_rpc_method_handler( + servicer.SaveResult, + request_deserializer=api__pb2.SaveResultRequest.FromString, + response_serializer=api__pb2.SaveResultReply.SerializeToString, + ), + "GetResult": grpc.unary_unary_rpc_method_handler( + servicer.GetResult, + request_deserializer=api__pb2.GetResultRequest.FromString, + response_serializer=api__pb2.GetResultReply.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "api.storage.DB", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class DB(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def SaveResult( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/api.storage.DB/SaveResult", + api__pb2.SaveResultRequest.SerializeToString, + api__pb2.SaveResultReply.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def GetResult( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/api.storage.DB/GetResult", + api__pb2.GetResultRequest.SerializeToString, + api__pb2.GetResultReply.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/pkg/client_locust/image.jpg b/pkg/client_locust/image.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce4941853dca136bc4448c206ba3a561bbbd4d3f GIT binary patch literal 13007 zcmb7pRa6~ax9rAs!^Yi%ySrO(cXxN!;2zxF-Ccrj+=9DH2oMrn1DEeR=l{ofy|epm zjqcI2Mz2~`HQ(3X_W&SSDH$mM1Oxy8@o@p(-vIpLs*2FoeVnE&7PJ_JC62XsT=LP3xNAkiS8&>-Hy073xZe#j>u3A=83A;&&oLoX*rBZ2s=!wvZ)=-2ES8JcHETvj|ml} zhBzog;*i;$|5NPWrSs`EvC;5Wq(x^PbWf!Y$OWj;ujlx4;vDBj!d`>3y7PDK&;LNlag&KS>fWQVBV>VlZ0_(WCZUBNdq{u@T@DlHr{byAvxX?z&f zMiLP$6C0;4!%>W9a^5S07#%6u=B*Ha6En2G1351HZRuneK z)2z3&{qwXkJG`{CnSd(pOhT8c7X`DX^{>OVx8k|f#?cHvEaj1Zy7Cy(S+rqTWRRb7 z9^JJ~rQK#*pf}7{Iuhv6Crhbz@?TYobuZOc%9q?`(@%+$n#5UF(rF;81=(x*g(42G zbDgKavbp^Dq)D={dhlR%h zs&i9u@54Z~A&GsAUn%MDv9hv*lOL{%jYw3EBN`P0(L|v@4pUSmhe!C`Eb@jf%RTs+ z%M!1i>E|*^q)r0`yu8-Pva8dK(+PiO>al906D`|>qJ_Dn`o8uJoY~~I3fWG7UM>AT zig%NALKspx0~cXZJEtA8pvbWqt!p&b$k${jV!IqK38x@O{Nmp#)CSGJXBl#D+b@MC zW<`AK?B!E`(aqQ@*3=(q=~la;++f8hAl%}NdKOR}Rm5NA|G8D?N6FN0vUEBu#>}#TK*m%o1(^;cn755C&e1K81M}4RD#6b- zs|Ji{UfZIy*fHV9X~W^tezyPxefrcnMqxkU=Jc`u=|CUs@KF%Z|B%E#4Dqr5p#f+x z=on<=AWT*^Sc*^VSlFT*AA|t^!4nWLkRCJlmOS+BrAp)?LE?CV-_Q9^J*n~JGFL03 z{nEbAvBKz|a*ugm-x-Si$ktvo9XjK37FT!^Ci_x?BK;zC%h#Kbs+OPNGOA$R;{}H> z9Ot?^hy(J0Zt^l}qyM@tA=D&TPcx8By?z@L571l(<{RwiBA?WGtA4Yqa`0|m?oV~) zjJ{ZBm7LtPba5CAd9m7oO1hRiOnxw>0^Y992hWPSWY951w4ug|uA(gB8_;o^$8J7A zVH|%w$HpT-Cx;^2-W)2b<)fx$-~T0Ug(pn7&J}GDsJ=##r>!z$9yW zLGnjGq-aaU^U~M(d*}{7UPSPtn=egwF%wQZX#D&N;IXsX{E?tTkVdmV8jmPX4_%4k%}gIn&>;yFQ?pI>A$|}W@2pe=VB1P z#NE8y{1(U=9Z?BFOMPfLC+~E%4jM)6@6wLZiI+#WAH(rSd$pH>207A}J3YRvR9tDq zdhs|N-iMA)B2zWFFsiI3)h4wt|8j9|3;}0S4GKAr2d_KwfrI{W3O_~-pt@GMaxBIXlInDN3?lpp@dh5-) zS@>_NT=KP;zT5BEZWGrgqdLou=(2mg70;Q!>h^@Mra_r4RFX_uitCtxPoJNc6B}73 zb}3?Y2z3rtL@DUr0n~3BB%|f4NDQXR!sTLU>KwS7e$(A%<_!tv!d6$MvB6D`l>05Y7#;EsupOj8yYjmMFl4P@%H`9yja6D-vLq6PbcG;sOp(>{aI4!D7_@9 zNir(Bf0btKuxCA(I>gv|r^M=;=CP2F4Wcj}{S*2W`RAYH@|fl^264Q|sNL`buE0Mw zJ!tinTqW1vBc@p0$NSB9ZOkS=Bj5Sf;*ju6j*f(qX!COaIg-9qz`eHCn29+JKf`OT zK(0xqsCWJ5Q$Q=jsFSUfA*dqdk_KISMaQ?C=4!(iYHM#(!SgA@UArkiH-&PXGI+Vf z-$-;$^X56Nz5lsCUB3wOsrVhB@DA9F2qGt0&8{Z$o-~BN;0`74pb*adOg~ik#CH^4 z28zn_iDyl!y=LB6t|*`pYZ@dx5b$xQN^R28e+;0PJGAwLs$i9uZx)U#eFp#^zzXJl z>mEf#Rwm#U1Y0xq03%V6M{OuY1`ywXO#cK}sL4>O*I zxf)l?R04H<)NzUVj1za%GbiIY)1+mL(>4m(DY}2+MG%Vks@Gp(67KlnN%M$Mlsl3} zv$7=Q_){DXndan9&dFY-PXFjEiIho&o?*>Ps`}67;}q6SDhZ42Y#wve%G!i(p*s>A z6P9PmGDp_e{cWa&p9jJnOZ{fWx?AACbaT7|u+a|FD@JKQ-4gcPUbw@4?wtr%gEkxk zEB<8dho}B^ui&v7F3yV(_c9b9(5p51$#7%f@1JK{s+&!O)DjA-#Ph&@Y9mfT*Ixlg ztwWr=NFock`DK@rM$H#eeooxK?lb57$OYPkpEQE=Lt!V+en)bInY~tf>biSkF25^^L$D>6_&`am!7(NA?hl zX@^lMV+r3gwRs3@K;~GxzVLFv7Mpv3D~KPLoW1SA@Wl#CS$T@-_yMFpD8#pGY{@ZmFrAz}$D zn{v@T`SCQx1gocnDSRpLrD2d*_so(pSc3lvtRK#eBI^VEg_iN}K zrixX|*}J&oYdtZEl9DWnZ_w5A$(1U(e>GniNtjVKrr@`U1>&~08xCi5VM%0tI_ePC zR>KyEejHQ#jZ@ZA6#$l6*!vs5T^VPA)#nZFP?r#!Ct6=Q(kCusTjX-GO8C>%d}Hb( z5jfFf+JR3biQ<5ErgFy9BVk!aut=lUR?ajB$BfOpLk2URpHWdkPyUMjRWKrgl`3Jc zjSOG%gB{{E}c2R7vYVDpcx`mj|0TOZhzN&s&LzeLT=N@1(Oc$dGW9Xvp6hO7+KZhWKM63 z$is`7CiqM}jZG&l*g6Z$2^f79nBhmkF}J_Pb=)O6l+1;is3dJ#IOf;k`@S!t!IYHB z>XnVD9?aVvIIhDH!VSIVvD5pVro2gzg#D1f`6izrZiHHtq+3MRpmb|tSsPNU(#%@_ zNy1EQ|CX|uavs|_^Bq8x6wz)b={o)}y#EzmSD(x!>5I_ol(-&lX82343Ylxa8IIXL zHua0}NAnyY00a~y6dWWp?Ekj^fdd*DD+pZ$ijaiGp3VsA=AH&@?3Z ze>EnoFeEkMh}w~Cad4-KAOqyQ3t`>wWq1(P&lYd3?Gue1YG6hj$}4c^e9`7WU>v`m zD9^){w{y2V)^ewGU5rQTD2~!)2Dtp^IK9e3@707LB|T;h0;l<|U7Js%bpEJrA3+x^L;<*1vzlqAF!s;0PL)mv4yi(-at0|pUotVv)YpBM?2OC{d zT0~gyj`>An#YCNJs25(nFP8g^&X2~ps;5<@8NgrDs`?HXA#ZE847wyq!PQW%us9(~ zi@MbC4E7*XCktMoYgZz_bh^$zAwW>91FE+wRx5!TUDvFCq^%i9uBez4vx-(jpJiTI z54{8EGRM0^l$ukr7OF&#U3xCcI#sI{mq<}1n_bvu$}J0Mi>*x3Q7vR=bocW2wK^nl z-vMO{t!L3kPBLdjC0wN?Oc~Z#*3CE)6sXfe23YDQ+bp9dNY`Uq^;RgO)e&MC^Ga7? zw;ZAe`#1ae(%9y>&&?Erim*k?hV{c~W2^{eN0#AfxxI?j$I8cumSQL2+i1c})Gqa> zI0$JY{?=OP?)ubF{v%oqO6$BO+Tnf-vr^x1&Cgo+L{ze!0qedq?nFWwe;_8E8M~*_ z)J-VITjlcBEiI>yE;SWeL}? zG^3b_uP;}QNFAdbH#QR~L&y{dC!kL?t!LkHDVVTbRu!PI`8ou;K~_vcj9=&<{+tn; zxy-}YK%{QmqOqRtNB6WpgZaLGF<__jtg;2Dy(sjAWC`uECnddho@5M5&J~-z5>hA1 zG;A1Nv_)I<_ng?|4y-JuOlkeH04a>ozCTm^W$2gB`Vdyiy%*XtZ||z-3doEsv|B)D zOv6IoF;-ulfcCrxkp(Y~3>Se%j8ID?k`9Z}tvA&n$omrX%GI`m1;n0zfKf#Vm#aPL zN}N1(jpSd`aLH01icS2RLHj}>35U4P-o4&h6c!p>oY`L8ilhXupN1on=+A3`uA8rL z5=7SQ`_+$jT7VHT$nfE`?0!Lm%i&D}O7D#UDQ_xo*JIj34gqfR@PKIOcxvAk3lm1K z+H^r#g@%xZE{~mBgEuu|-dg>Jo(6@Vf{zC4ve&*VsMTGX{=!{M(F<#2S!&Gc9pFVS zgCe+k-1x#g;nZ4fNqnAj*=<7k9X2M$Id8IP-XK6xEb~u2*@Mvngmd*3@l>QmR-lOT zBm6HeChH9k{05cFwveD|Ey-uZ1B`7o*FT+oXP7l6k{S|7ska8Kbs4wasN2pYuC>3T zynH#4|E$D^1TjrEAR|%NDuug?+jq)V4YlirTGVL0s8&T(ce>DwMiqwYVjFgoEuyU1 zR@YsWE{e49YN%*nAm(I$#SAq(lZ4mhzo!>3o)>G!+{yTIMB@NwAxpRJquWD#rm$40 z#Bd_b9)psqD}^=6QioEV%17h+4)E4|9K{G{j#jWyAA6JQH(XQkma18DqScZ4ih}~@ z+v&L#j@St=$*W=RAUh5WtRTfKs%kE%c~mYrBRBX<%dN^(Zc%3B@(v((PU=ErIf5aa z4lka$!$e+T$qe3Ipv`~g>Wu>D(TaCx!FClQfWW*X~ zhT(-ZUm5^v!bXbge5qbwan!(u6R9<;GY$tIvyklcjU9n=q9JSnxv=HeDOo4+yE)hUETt$w( z!!Ag(+&7aRf0opJ2e2Pw0OO!{I?0(tOXqQq*BknjouMDGWYN@Cg!)IWJfuqyIS#QDfop<2Bvp zWbKOhb?iZu-N#H5aXhF38;ono18S-o60MWRvw>l1kO`r8VY=gso8%H*urv_aVSR9M zj0#BUx>H*D4i9CBrv0*3rm5~PTVncDw(EydVy%@q0C_3!4hS@cInNH0eN-}g8)(Z- zyYhXRloPB@y9YDRi@yVuGsp?IdqJLA>c ztB9MnUH-vAz^h)jThO8lOY;fpZg|e+VWbD&iE=WD@3QUw>D*~3F|7z z@OGbFY6}u>mTkz?!5Z=U`?oUDvB4+&RqkMjPoAioO2LM}r;%%AlAF&IsuND(_7VY1 z&q8!42$KEL>HvLsivD+yH|yx&IHMTyJoGR2UvI#we5FD6WW+73L)h* zM569nKuv7P4Cx=%ztzmOxgPVXe%+b_hnjG?SJS>McJmwrQFknPIUOIJj3Jns0Dsh7zmXuk@9&+VYL|$BC$=+lV&Lj!_9Is~-d&eFo95>*Pwy0pKp5=Lf7xSy`P-1m_ zWF*sO%S4Qb1qg8@SpHhi93>NDa=E6OY&(>+m8xU~fBBW`2 zU(3Yvko)H`M(g%e?jXHFi#?7p-z9_dCHkvwDCc(Ad4CkZnvE~(a|g?oyQdsE=AP}Z zWswABHd4#IqN>0KFE?KKwk5C-IS@W&ii2nh*7%b#lL8IwngiwP5hfzfT^#|F2JO#|4W7aR)I zJthnT351DrxJ~^;jp@Osj3m7hki9}v7UQP*gh1x2<01I{Gpp}b^`%jgSX5bze35JT zWL8Yj7F&$EWs&1SF%4K5>qwTob6FjKtxkWarr`?NlT4z1voXC0_cNWPAQj+j^v@bT@#EKrR=a}s*{48B_yAJ_&ExISGSlWOF0#Q4q#&ZX?fEG#GkMd3t0aG}TAzhjYk~Ntr=sjICF&7BNHyfb1THA;M0s^m?(;#y zI*0I__g>JiNnsk^gZP!zWRi|wh>s}G@Bw;fH#9plp(VmhXf+&u7CHgYM;nx4b*Nz= z4GXmb#{%@@g2sX)rhE4xzvPAWdQ&znW=FoRhk_v%&yT@WL(jd&?WhfvL|jNjy+EvURKL~i|?zAbM;ok_Vn{h zfuPt(tg;c*lun2xV0(m*eaKh{1zw{E>?PV35UCpS4hSKyiGt+@*~^QK^y&1G>+cLk zEoA1T8((c5duQR)QyE(vES;h05%nLahs-{ag|*KjtEdL~Q$?zt%c5PIl-nw5)BM=1 zGVV@;K$cIH!tbd*fV{b+TF7mp?b`F8NpfNQ!p&NuK8f{){v)rXP%UN0^en5=AR#3g zAJtp?sCO#uG~%A%2eN)Ih9ebVF>IZ7y`K<#qc3{lJdLHre&$4`zU84nff}7k!wgCN zIZE|;GO4S$jDh~l-J8lZbeduVo1`n#DIFz}#kOCD&@an1>*9=#NwL_usgJ5&aqz?m zh*NoS<|I!L^(4Yk05v}d6N{*Ot5>~e#?y6^ zh6ndMVcZh4xdYW+vyKF%Mb$S){jRDoW7MoNa9vrbJRc)7bEg4pRLz2b7^G#k<1=+Q zj(c;K=U@2?;;AKxqWagt*y;yj?|?GNZB{y@VR?R{%nXV4EUg1vy{rHfO4~DbrRW%& z_^3&CWG>6$2z?5swsP3@c=T%h+<}!2+6L;<<=l5bt>_&^qRvd9%Ch#3z5G_oleWXH z?z5U#i_q@(fUL8DzbT0Kp%_&qJkL^pfxXm)&9dy#AENHxVJ+aFm;of4Z9dN`G&-+U zCxT2vg>Ea-sI@XBs_+`;-8|v5x3&yi(GEo!KAW2zGBB8Fe?9OtJ*T;!(fk!mM5zfB z-Be)mB3*3jwtW^!>!Yil#KLdvV z1nonmehifUnLhtda`dlS1%1d>7t+ChW$HhH5h(2Hfxr(K&p~;B)ZS0|mwfTT;W?6G zlQcLpurxwfm>5IeqHo$g9ds|TU=$KC6*rv@m^5a|YAUgMY!qy70T4DRpc9TX5=I6O zh>YVM1#ur@irJ5*kfL5(~7%}m-5}bQM(ZS zd7A-h)LD!xi_Ut*vvL^18s<6}H-&fWI3P$rK8`@+_DSe~wvReGH=Lb=$+09=L}nV( zux9P`N1cNAocxtrmdiRi=3nNAk-&(Mz+4#4H{>WK*f()M7dn%T91Q2A-A5R@z8lMf zqS!b1VuBt(86rOG?_=88tOcolm&>1of^ByMz#)YH^(G37NtUf0!(l19o$7QbRC0)-Q>G_eY^?qYC)m%Fm!tj0E@fRl8 zB%NUX0H63@@IPXy|4#n@$0~qGRZN2a#r~gA3Un3H<@~y%mC*UenB7a^d5QAz0!HYA~F>7Q?8MIVP;aA^d~qcbXCqs5rT70Dz+SS z55mZn{m#37CszACpFk<>hp~J+ulqI2xp%^CkD+kG+6?wpCxe1dAnbumAf$Qj=Ey>* zmy$7#*0|&tpM4*vsArB?SLJTS zzcK3%%}z+xlYP zSTR|5j0e!~6ep&|YA0xec? ze}n7vZqW;6WPplV%WsFCW#SliO`>paLc^E09a;Fp^YvwsprjA_Kv?asw4PWU7%lpK?H2RJy>t%#n9^TLcNz z1B}Bwm}{=4D@2N;cNnoOES=G`gdFahD3?qRCv=w7vus*08{<1*LavTtA@(LyJ-K0E z2dC}1GUOV&6>2~hD9G$DE*x3km}nt4Te&-p6a;w-OXy;QC*OwO<54)8Wj^na!U#^ME(#FV5`{*{KS9^Vjc zZcMyx-yp8q9PK3sQi5Cs2MS?2zC>F`!YDL1A+&IQwSkE(I-_Es(Trn?diah7J&M4^ zfluNWDLcR|7i45-2jf~F-)EdOLL`vj@2cXR+M=%6VZXs8h+^VE^BwY>(C_b+HJ=+} zJb&&6(T?RBAN=*hNQn7pFU6xUh76pkDxJB%s4P(Omdt>Z=EB1nkiLY>aX*OsHz#&~;P9o{%;1k`7ht%%=B413KoH&ioB<&`i&defv=PNXR zlCD>0B4ZTzM@+qOmQ5&cDvi<~q|r;ce1phjXd+r*MZ(%qh;80J*8uj7;pL-2jeHEo z#zz$q6rWK+p`n$N%I=`W4a?a0J9$;f{{vAFz^bfai8#2B#&hHl9!6_h1oQnv zgfT;ig>>!i!gW8|+}?uRVkBF??ajOg{C*%Uv=k1J$zArA82k{`fmvpStgOkvli<-V9yoES=4Guxu zamnzan)c)towwt6fMxu>gD>Q_q%{V{LO^~2SVA-;J=5&=BysfLcQfZzf~X<+PJ9WAHL>>6K$Gp2Dqd#Fm9e^$u(M`vysV+2pW>~ zLEn`brtIXwe=+x3~_{`mWTooc%g{$&^cfgLWWK;|c!i z8mI30NqK{ZJQOL3Chj8Uh5$G~hzgSDBD{CoqVl_5tF%dvDB+7{mhq2cKzfvs=dor! z>Kwpf8}K)M+jpo4sl56-nBvg!rQ!ivRK`HB`+zozb9eKSy%nZue&~5v*{ED4z=Vbp zrxdfo$b`A?&OilX2SJHYlmP&0EDY9)^`(t+E`^O-h?&=%Oohy%fb}JMBqAn#MPVkv z?9Uv~k!5^fdmX;xhCjp_Z5cYjtw;N{)VYG?Rva8}lkG(~5uvL-48HRguIKCBu52J8g071kcYT?})-gsl-i zfR5#vJ&iWraI4ILcnqaCHu=X#Sk{P2l6%)^X34p_2Kn!R0+znGwj=1d<`a{_;^Cd@ zAwGxk>z|P-&D=@^veDRo0=3gy^(E3XmYl3_^nOG_kmAd(6V6$DS$k*K0VS&(lWb zIItYlCLH+9m-e06-W(~Q+IG(VC3)Zmuw3-^#;dY{hk}r*Z6gIhY@lu(ife~!BV?0K z8N9*!34N~Mfd`oXgKa40MIWe-TDonx+~Q;BgMX zsS+LNeDqpA0p+4PBOcG*KJ^Z7iPtQWG!8P196GArYm1Z{nyx5%c{DNg-U|DyfW zoz=X{?D{M5ye$K=K7-{+_nYh#1o7uiq{^mO(gv`H%SIu8n0|_73B60E3FeunC=AAQ z%~+P%nMTsvoU>MrZ}cyVB$Wf!k0BCG#|@LG=KPuO>1T=5ZNn(u47zcDTRn8+l84u+ z?wGN7og2%87{~PIvN4Yv$fV8iorqI*9m7Kb8g*G?gJ3Bk3%{gP;%fm^FpW7RYoLtL zHLJn%B$i7(dS|$ZX{SqI@^(J8#tKiIWAIj?6@@T6GK_R{NX)8`MfjGnmnj_6&{`4p zb<hRzTg-W%=kDL}U*mtqZaDwmN0c^j=)(YCO#zNAHk@%sRi~RjY_oSI8V>i&tdz^zPF3c||p^-Ep4M=|zB*8~I&#@p*ZPG0vJz zda*-G=B2I;QB44j5W`s`yQpkT7&dw(maRWS1>kYi5c7!xZ1iCFf~6Yl=8xZP7Yfn1 zmPXCOJFQyX6qk1|A=+S2*ct#ym+~C|sVe#lth~m1E;@qKE1%S;T>&zfs z+FQfICEAqWGGz2haOsymiYvs8^edgpScp;m)`G3}LTMzHL~{6yrZFDo zIX%>>{r-}B(HZ)dTQLyT^%pwqHbPoFy|`V5a}d2b!ES1Qm))Uy%!APcBD85nWEGgy zl|VFZD7gbc5Ejjhn2J7~OPU#jj0TiPA>XJ{Y#%#6W^EChhLNScZQeqwX=Jk+8pTLE zR2ZAa*BhE0#^x#-cm^kNYPQG^hHW)OCXsKd=rs(~I-?5qE4e}p z5<{)<%)idEpqVvJRE{aNOJ<{+UXXB!S(p(=%H67w;}GB0s1C17TTuaoyK7Bk%4bMA zgd#??A+yv~ZtP~B@v&{aiXDc``m>&d@Tgfo>@qy$2 z$}-_9xjndfS5^17njgrPt9Lq2MY^d^KH~L^hyg`T8(iosoh2nhRCn+#`AW7Rb%-Xz zjVbB~DJOvlmS8~&s#dzH>=|Rx@orXR3_EY3-)D$00)Q90wJ`p|^_v8+PpRs_lW|;| zg^w>u*?tDQjDt7q--a7PENm1ln@9GS7%}U9O^GPk)DCAJ{6E-qp`uXEpr=i+DhT zY$Y3sy~k8HEU3!jDWj-kCeOfPE7twAMCVXDdBin_qcph^?CPN(x4qa52*D8 z|Dj4!k@5-%1=tT8@Zq+NkENyyhN|2f@#lR9n9}Tu!JF1_KX+3~o;FnU6aP-~5VhX$ zx$eItABt6RKhs42M)ZX1`VliiDy(aQCrX+oNMD@B%s=fqlD!N~FSCl7pZz#-@p z&{g4ZB`5?p_7wEcm9Q7sORiXa%&8P%v+&X@V`pnSy`QjR#g;`NvWOPn1Dn`OMHD`X zA;s3_P~^YgzwyNiTuLE7D$o;fh|V4#J93Gu!-R@41Okd;X;{C;u^_RjNgz@WLSu!b zTs&Q&|JoIh!R`-I^WQN`XXmBT^JL7txuv?v~b{H`5S?6C>w8@%w#ZWyU$%h_2rN5+MvV zT>&WYQwCb`7Q?<4e?k1xjsZ`vQU_JYk8l|;Lx}*+`6&hrP2VpCzvNVJXM+E-imNK3xt+eWNhR?wAK}10%X$l&ag9+A^5?X$K|x(=>Egz z3KjEW5w2y<+yrIu$4gpJ1BF$Da?%~X8PN6`Jf zs-HfNLNr(IF@UZ(1aIH>pZ*YKDJyKh!ZsY8N-v!7@Cod*2-&RcME(7@)5hBNTc5U0 z;lhm~+_?WP8rZRE3|V6OyJpHK*?}f6r+e}S0A3K2lJ#99d4FZyAFmU>8w)wvuf9{^ zf-<=krIPOVo#AiB2EFCH=qFvwmmJ~o=h3nb_CBQai;>{Q92nLoR&+T@uzdytd|e0f zxSTAw^ZrFUw{&zB6J7_phWv}|=Gd+Eb^ZR()BxQwf1$9ww2Hu-VWjX@ZR=7uVFQsA zREI;~PE_ly!~WaIANK4J*+zFykYKm!(c#Y>AXDYequQ3@& z`7Ve#P+&1NahLod9!Hh7G@tmQj7k`69`8`&@SxgWa7e)r$*k7olnHzKrV&hx6TZobsGjrG1x1L;3?Cw5sC(35CXP2mhMj$%J5PBL9ZX(V!4n@XgG&z4|#62OB&* x&JFMKDJI;c65e|ADM(X5ek#7a;7PUf&n+pB7uQ2{i|}KSZt`)FU)B5i{{v;h<(>cl literal 0 HcmV?d00001 diff --git a/pkg/client_locust/invokust/__init__.py b/pkg/client_locust/invokust/__init__.py new file mode 100644 index 0000000..cdd89df --- /dev/null +++ b/pkg/client_locust/invokust/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +from .settings import create_settings +from .loadtest import LocustLoadTest diff --git a/pkg/client_locust/invokust/loadtest.py b/pkg/client_locust/invokust/loadtest.py new file mode 100644 index 0000000..023f405 --- /dev/null +++ b/pkg/client_locust/invokust/loadtest.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +import sys +import gevent +import json +import signal +import logging +import time +from locust import events +from locust.env import Environment +from locust.log import setup_logging +from locust.stats import stats_printer +from locust.util.timespan import parse_timespan +from .prometheus_exporter import metrics_export # import the event hook + +setup_logging("INFO", None) +logger = logging.getLogger(__name__) + + +def sig_term_handler(): + logger.info("Got SIGTERM signal") + sys.exit(0) + + +class LocustLoadTest(object): + """ + Runs a Locust load test and returns statistics + """ + + def __init__(self, settings): + self.settings = settings + self.start_time = None + self.end_time = None + self.web_ui = None + gevent.signal_handler(signal.SIGTERM, sig_term_handler) + + def stats(self): + """ + Returns the statistics from the load test in JSON + """ + statistics = { + "requests": {}, + "failures": {}, + "num_requests": self.env.runner.stats.num_requests, + "num_requests_fail": self.env.runner.stats.num_failures, + "start_time": self.start_time, + "end_time": self.end_time, + "fail_ratio": self.env.runner.stats.total.fail_ratio + } + + for name, value in self.env.runner.stats.entries.items(): + locust_task_name = "{0}_{1}".format(name[1], name[0]) + statistics["requests"][locust_task_name] = { + "request_type": name[1], + "num_requests": value.num_requests, + "min_response_time": value.min_response_time, + "median_response_time": value.median_response_time, + "avg_response_time": value.avg_response_time, + "max_response_time": value.max_response_time, + "response_times": value.response_times, + "response_time_percentiles": { + 55: value.get_response_time_percentile(0.55), + 65: value.get_response_time_percentile(0.65), + 75: value.get_response_time_percentile(0.75), + 85: value.get_response_time_percentile(0.85), + 95: value.get_response_time_percentile(0.95), + }, + "total_rps": value.total_rps, + "total_rpm": value.total_rps * 60, + } + + for id, error in self.env.runner.errors.items(): + error_dict = error.to_dict() + locust_task_name = "{0}_{1}".format( + error_dict["method"], error_dict["name"] + ) + statistics["failures"][locust_task_name] = error_dict + + return statistics + + def set_run_time_in_sec(self, run_time_str): + try: + self.run_time_in_sec = parse_timespan(run_time_str) + except ValueError: + logger.error( + "Invalid format for `run_time` parameter: '%s', " + "Valid formats are: 20s, 3m, 2h, 1h20m, 3h30m10s, etc." % run_time_str + ) + sys.exit(1) + except TypeError: + logger.error( + "`run_time` must be a string, not %s. Received value: % " + % (type(run_time_str), run_time_str) + ) + sys.exit(1) + + def run(self): + """ + Run the load test. + """ + + if self.settings.run_time: + self.set_run_time_in_sec(run_time_str=self.settings.run_time) + + logger.info("Run time limit set to %s seconds" % self.run_time_in_sec) + + def timelimit_stop(): + logger.info( + "Run time limit reached: %s seconds. Stopping Locust Runner." + % self.run_time_in_sec + ) + self.env.runner.quit() + self.end_time = time.time() + logger.info( + "Locust completed %s requests with %s errors" + % (self.env.runner.stats.num_requests, len(self.env.runner.errors)) + ) + logger.info(json.dumps(self.stats())) + + gevent.spawn_later(self.run_time_in_sec, timelimit_stop) + + try: + logger.info("Starting Locust with settings %s " % vars(self.settings)) + + self.env = Environment( + user_classes=self.settings.classes, + reset_stats=self.settings.reset_stats, + tags=self.settings.tags, + events=events, + exclude_tags=self.settings.exclude_tags, + stop_timeout=self.settings.stop_timeout, + ) + + self.env.create_local_runner() + # gevent.spawn(stats_printer(self.env.stats)) # need to be stopped properly + + if self.web_ui == None and self.settings.metrics_export: # reuse the exist web_ui + self.web_ui = self.env.create_web_ui() + + self.env.events.init.fire(environment=self.env, runner=self.env.runner, web_ui=self.web_ui) # fire event hooks + + self.env.runner.start( + user_count=self.settings.num_users, spawn_rate=self.settings.spawn_rate + ) + + self.start_time = time.time() + self.env.runner.greenlet.join() + + except Exception as e: + logger.error("Locust exception {0}".format(repr(e))) + + finally: + self.env.events.quitting.fire() diff --git a/pkg/client_locust/invokust/prometheus_exporter.py b/pkg/client_locust/invokust/prometheus_exporter.py new file mode 100644 index 0000000..ab195bc --- /dev/null +++ b/pkg/client_locust/invokust/prometheus_exporter.py @@ -0,0 +1,110 @@ +# coding: utf8 + +import six +from itertools import chain + +from flask import request, Response +from locust import stats as locust_stats, runners as locust_runners +from locust import User, task, events +from prometheus_client import Metric, REGISTRY, exposition + +# This locustfile adds an external web endpoint to the locust master, and makes it serve as a prometheus exporter. +# Runs it as a normal locustfile, then points prometheus to it. +# locust -f prometheus_exporter.py --master + +# Lots of code taken from [mbolek's locust_exporter](https://github.com/mbolek/locust_exporter), thx mbolek! + + +class LocustCollector(object): + registry = REGISTRY + + def __init__(self, environment, runner): + self.environment = environment + self.runner = runner + + def collect(self): + # collect metrics only when locust runner is spawning or running. + runner = self.runner + + if runner and runner.state in (locust_runners.STATE_SPAWNING, locust_runners.STATE_RUNNING): + stats = [] + for s in chain(locust_stats.sort_stats(runner.stats.entries), [runner.stats.total]): + stats.append({ + "method": s.method, + "name": s.name, + "num_requests": s.num_requests, + "num_failures": s.num_failures, + "avg_response_time": s.avg_response_time, + "min_response_time": s.min_response_time or 0, + "max_response_time": s.max_response_time, + "current_rps": s.current_rps, + "median_response_time": s.median_response_time, + "ninetieth_response_time": s.get_response_time_percentile(0.9), + # only total stats can use current_response_time, so sad. + #"current_response_time_percentile_95": s.get_current_response_time_percentile(0.95), + "avg_content_length": s.avg_content_length, + "current_fail_per_sec": s.current_fail_per_sec + }) + + # perhaps StatsError.parse_error in e.to_dict only works in python slave, take notices! + errors = [e.to_dict() for e in six.itervalues(runner.stats.errors)] + + metric = Metric('locust_user_count', 'Swarmed users', 'gauge') + metric.add_sample('locust_user_count', value=runner.user_count, labels={}) + yield metric + + metric = Metric('locust_errors', 'Locust requests errors', 'gauge') + for err in errors: + metric.add_sample('locust_errors', value=err['occurrences'], + labels={'path': err['name'], 'method': err['method'], + 'error': err['error']}) + yield metric + + is_distributed = isinstance(runner, locust_runners.MasterRunner) + if is_distributed: + metric = Metric('locust_slave_count', 'Locust number of slaves', 'gauge') + metric.add_sample('locust_slave_count', value=len(runner.clients.values()), labels={}) + yield metric + + metric = Metric('locust_fail_ratio', 'Locust failure ratio', 'gauge') + metric.add_sample('locust_fail_ratio', value=runner.stats.total.fail_ratio, labels={}) + yield metric + + metric = Metric('locust_state', 'State of the locust swarm', 'gauge') + metric.add_sample('locust_state', value=1, labels={'state': runner.state}) + yield metric + + stats_metrics = ['avg_content_length', 'avg_response_time', 'current_rps', 'current_fail_per_sec', + 'max_response_time', 'ninetieth_response_time', 'median_response_time', 'min_response_time', + 'num_failures', 'num_requests'] + + for mtr in stats_metrics: + mtype = 'gauge' + if mtr in ['num_requests', 'num_failures']: + mtype = 'counter' + metric = Metric('locust_stats_' + mtr, 'Locust stats ' + mtr, mtype) + for stat in stats: + # Aggregated stat's method label is None, so name it as Aggregated + # locust has changed name Total to Aggregated since 0.12.1 + if 'Aggregated' != stat['name']: + metric.add_sample('locust_stats_' + mtr, value=stat[mtr], + labels={'path': stat['name'], 'method': stat['method']}) + else: + metric.add_sample('locust_stats_' + mtr, value=stat[mtr], + labels={'path': stat['name'], 'method': 'Aggregated'}) + yield metric + + +@events.init.add_listener +def metrics_export(environment, runner, **kwargs): + print("locust init event received") + if environment.web_ui and runner: + @environment.web_ui.app.route("/metrics") + def prometheus_exporter(): + registry = REGISTRY + encoder, content_type = exposition.choose_encoder(request.headers.get('Accept')) + if 'name[]' in request.args: + registry = REGISTRY.restricted_registry(request.args.get('name[]')) + body = encoder(registry) + return Response(body, content_type=content_type) + REGISTRY.register(LocustCollector(environment, runner)) diff --git a/pkg/client_locust/invokust/settings.py b/pkg/client_locust/invokust/settings.py new file mode 100644 index 0000000..5052193 --- /dev/null +++ b/pkg/client_locust/invokust/settings.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +import os + +from locust.main import load_locustfile + +def create_settings( + from_environment=False, + locustfile=None, + classes=None, + num_users=None, + spawn_rate=None, + reset_stats=False, + run_time="3m", + loglevel="INFO", + metrics_export=False, +): + """ + Returns a settings object to configure the locust load test. + + Arguments + + from_environment: get settings from environment variables + locustfile: locustfile to use for loadtest + classes: locust classes to use for load test + num_users: number of users to simulate in load test + spawn_rate: number of users per second to start + reset_stats: Whether to reset stats after all users are hatched + run_time: The length of time to run the test for. Cannot exceed the duration limit set by lambda + + If from_environment is set to True then this function will attempt to set + the attributes from environment variables. The environment variables are + named LOCUST_ + attribute name in upper case. + """ + + settings = type("", (), {})() + + settings.from_environment = from_environment + settings.locustfile = locustfile + + # parameters needed to create the locust Environment object + settings.classes = classes + settings.tags = None + settings.exclude_tags = None + settings.reset_stats = reset_stats + settings.step_load = False + settings.stop_timeout = None + settings.metrics_export = metrics_export + + # parameters to configure test + settings.num_users = num_users + settings.run_time = run_time + settings.spawn_rate = spawn_rate + + if from_environment: + for attribute in [ + "locustfile", + "classes", + "run_time", + "num_users", + "spawn_rate", + "loglevel", + "metrics_export" + ]: + var_name = "LOCUST_{0}".format(attribute.upper()) + var_value = os.environ.get(var_name) + if var_value: + setattr(settings, attribute, var_value) + + if settings.locustfile is None and settings.classes is None: + raise Exception("One of locustfile or classes must be specified") + + if settings.locustfile and settings.classes: + raise Exception("Only one of locustfile or classes can be specified") + + if settings.locustfile: + docstring, classes, shape_class = load_locustfile(settings.locustfile) + settings.classes = [classes[n] for n in classes] + else: + if isinstance(settings.classes, str): + settings.classes = settings.classes.split(",") + for idx, val in enumerate(settings.classes): + # This needs fixing + settings.classes[idx] = eval(val) + + for attribute in ["classes", "num_users", "spawn_rate"]: + val = getattr(settings, attribute, None) + if not val: + raise Exception( + "configuration error, attribute not set: {0}".format(attribute) + ) + + if isinstance(val, str) and val.isdigit(): + setattr(settings, attribute, int(val)) + + return settings diff --git a/pkg/client_locust/jobtemplate.yaml b/pkg/client_locust/jobtemplate.yaml new file mode 100644 index 0000000..01ce4cf --- /dev/null +++ b/pkg/client_locust/jobtemplate.yaml @@ -0,0 +1,41 @@ +metadata: + name: "mobilenet-client" + namespace: morphling-system +spec: + template: + spec: + containers: + - name: pi + image: yukiozhu/morphling-client-plugin:demo + env: + - name: TF_CPP_MIN_LOG_LEVEL + value: "3" + - name: MODEL_NAME + value: "mobilenet" + - name: LOCUST_NUM_USERS + value: 10 + - name: LOCUST_SPAWN_RATE + value: 10 + - name: LOCUST_RUN_TIME + value: 20 + - name: LOCUST_LOCUSTFILE + value: "locustfile_grpcuser.py" + - name: LOCUST_METRICS_EXPORT + value: true + - name: FAIL_RATIO + value: 0.3 + - name: PRINTLOG + value: True + resources: + requests: + cpu: 800m + memory: "1800Mi" + limits: + cpu: 800m + memory: "1800Mi" + command: [ "python3" ] + args: ["morphling_client_locust.py"] + + imagePullPolicy: Always + restartPolicy: Never + backoffLimit: 1 diff --git a/pkg/client_locust/locust_grpc.py b/pkg/client_locust/locust_grpc.py new file mode 100644 index 0000000..10f39b9 --- /dev/null +++ b/pkg/client_locust/locust_grpc.py @@ -0,0 +1,86 @@ +# make sure you use grpc version 1.39.0 or later, +# because of https://github.com/grpc/grpc/issues/15880 that affected earlier versions +import grpc +from locust import events, User, task +from locust.exception import LocustError +from locust.user.task import LOCUST_STATE_STOPPING +import numpy as np +import gevent +import time +import os + +# patch grpc so that it uses gevent instead of asyncio +import grpc.experimental.gevent as grpc_gevent +grpc_gevent.init_gevent() +failratio_limit = float(os.getenv("FAIL_RATIO", 0.2)) + +class GrpcClient: + def __init__(self, environment, stub, output): + self.env = environment + self._stub_class = stub.__class__ + self._stub = stub + self._outputs = output + + def __getattr__(self, name): + func = self._stub_class.__getattribute__(self._stub, name) + + def wrapper(*args, **kwargs): + request_meta = { + "request_type": "grpc", + "name": name, + "start_time": time.time(), + "response_length": 0, + "exception": None, + "context": None, + "response": None, + } + start_perf_counter = time.perf_counter() + try: + result = func(*args, **kwargs) + response = np.array(result.outputs[self._outputs].float_val) + prediction = np.argmax(response) + print("Prediction:", prediction) + request_meta["response"] = result + except grpc.RpcError as e: + request_meta["exception"] = e + request_meta["response_time"] = (time.perf_counter() - start_perf_counter) * 1000 + self.env.events.request.fire(**request_meta) + return request_meta["response"] + + return wrapper + +class GrpcUser(User): + abstract = True + stub_class = None + + def __init__(self, environment): + super().__init__(environment) + for attr_value, attr_name in ((self.stub_class, "stub_class"),): + if attr_value is None: + raise LocustError(f"You must specify the {attr_name}.") + self._channel = grpc.insecure_channel(self.FLAGS.server) + self._channel_closed = False + stub = self.stub_class(self._channel) + self.client = GrpcClient(environment, stub, self.FLAGS.outputs) + + def stop(self, force=False): + self._channel_closed = True + time.sleep(1) + +# Add a FailRatio Listener +from locust.runners import STATE_STOPPING, STATE_STOPPED, STATE_CLEANUP, MasterRunner, LocalRunner +def checker(environment): + while not environment.runner.state in [STATE_STOPPING, STATE_STOPPED, STATE_CLEANUP]: + time.sleep(1) + if environment.runner.stats.total.fail_ratio > failratio_limit: + print(f"fail ratio was {environment.runner.stats.total.fail_ratio}, quitting") + environment.runner.quit() + if environment.web_ui != None: + environment.web_ui.stop() + return + +@events.init.add_listener +def on_locust_init(environment, **_kwargs): + # dont run this on workers, we only care about the aggregated numbers + if isinstance(environment.runner, MasterRunner) or isinstance(environment.runner, LocalRunner): + gevent.spawn(checker, environment) \ No newline at end of file diff --git a/pkg/client_locust/locustfile_grpcuser.py b/pkg/client_locust/locustfile_grpcuser.py new file mode 100644 index 0000000..0d9ebb7 --- /dev/null +++ b/pkg/client_locust/locustfile_grpcuser.py @@ -0,0 +1,103 @@ +# make sure you use grpc version 1.39.0 or later, +# because of https://github.com/grpc/grpc/issues/15880 that affected earlier versions +import os +import grpc +from locust import task + +import tensorflow as tf +from tensorflow_serving.apis import predict_pb2, prediction_service_pb2_grpc +from locust_grpc import GrpcUser + +reserved_models = { + "densenet121": [224, 224], + "densenet169": [224, 224], + "densenet201": [224, 224], + "efficientnetb0": [224, 224], + "efficientnetb1": [240, 240], + "efficientnetb2": [260, 260], + "efficientnetb3": [300, 300], + "efficientnetb4": [380, 380], + "efficientnetb5": [456, 456], + "efficientnetb6": [528, 528], + "efficientnetb7": [600, 600], + "inceptionresnetv2": [299, 299], + "inceptionv3": [299, 299], + "mobilenet": [224, 224], + "mobilenetv2": [224, 224], + "nasnetlarge": [331, 331], + "nasnetmobile": [224, 224], + "resnet101": [224, 224], + "resnet152": [224, 224], + "resnet50": [224, 224], + "resnet101v2": [224, 224], + "resnet152v2": [224, 224], + "resnet50v2": [224, 224], + "vgg16": [224, 224], + "vgg19": [224, 224], + "xception": [299, 299], +} + +def parse_flags(): + with tf.device("/cpu:0"): + tf.get_logger().setLevel("ERROR") + + tf.compat.v1.app.flags.DEFINE_integer( + "concurrency", 100000, "maximum number of concurrent inference requests" + ) + tf.compat.v1.app.flags.DEFINE_integer( + "num_tests", 3, "Number of test images per test" + ) + tf.compat.v1.app.flags.DEFINE_integer( + "batch_size", os.environ["BATCH_SIZE"], "Number of test images per query" + ) + tf.compat.v1.app.flags.DEFINE_string( + "server", os.environ["ServiceName"], "PredictionService host:port" + ) + tf.compat.v1.app.flags.DEFINE_string("image", "", "path to imxage in JPEG format") + tf.compat.v1.app.flags.DEFINE_string( + "model", os.environ["MODEL_NAME"], "model name" + ) + tf.compat.v1.app.flags.DEFINE_string( + "signature", "serving_default", "signature name" + ) + tf.compat.v1.app.flags.DEFINE_string("inputs", "inputs", "signatureDef for inputs") + tf.compat.v1.app.flags.DEFINE_string( + "outputs", "predictions", "signatureDef for outputs" + ) + tf.compat.v1.app.flags.DEFINE_enum( + "task", default="cv", enum_values=["cv", "nlp"], help="which type of task" + ) + tf.compat.v1.app.flags.DEFINE_bool( + "printLog", True, "whether to print temp results" + ) + return tf.compat.v1.app.flags.FLAGS + +class TensorflowGrpcUser(GrpcUser): + FLAGS = parse_flags() + assert FLAGS.num_tests <= 10000 + assert FLAGS.server != "" + stub_class = prediction_service_pb2_grpc.PredictionServiceStub + + @task + def predict(self): + if not self._channel_closed: + if self.FLAGS.task == "cv": + with open("./image.jpg", "rb") as f: + data = f.read() + data = tf.image.decode_jpeg(data) + data = tf.image.convert_image_dtype(data, dtype=tf.float32) + data = tf.image.resize(data, reserved_models[self.FLAGS.model]) + data = tf.expand_dims(data, axis=0) + elif self.FLAGS.task == "nlp": + data = tf.convert_to_tensor(["This is a test!"]) + data = tf.concat([data] * self.FLAGS.batch_size, axis=0) + request = predict_pb2.PredictRequest() + request.model_spec.name = self.FLAGS.model # 'resnet50' + request.model_spec.signature_name = self.FLAGS.signature + request.inputs[self.FLAGS.inputs].CopyFrom( + tf.make_tensor_proto(data, shape=list(data.shape)) + ) + timeout = 100 # second + self.client.Predict(request, timeout) + + diff --git a/pkg/client_locust/locustfile_httpuser.py b/pkg/client_locust/locustfile_httpuser.py new file mode 100644 index 0000000..580529c --- /dev/null +++ b/pkg/client_locust/locustfile_httpuser.py @@ -0,0 +1,14 @@ +import os +from locust import HttpUser, task + +default_host = os.environ["ServiceName"] +if not default_host.startswith('http://'): + default_host = 'http://' + default_host + +class MyHttpUser(HttpUser): + host = os.getenv("HTTP_HOST", default_host) + assert host != "" + + @task + def access(self): + self.client.get("/") \ No newline at end of file diff --git a/pkg/client_locust/morphling_client_locust.py b/pkg/client_locust/morphling_client_locust.py new file mode 100644 index 0000000..3c309db --- /dev/null +++ b/pkg/client_locust/morphling_client_locust.py @@ -0,0 +1,87 @@ +from __future__ import print_function + +import math +import os +import threading +import time +import numpy as np + +import grpc +import api_pb2 +import api_pb2_grpc + +import invokust + +# ResultDB Settings +db_name = "morphling-db-manager" +db_namespace = os.environ["DBNamespace"] +db_port = os.environ["DBPort"] +manager_server = "%s.%s:%s" % ( + db_name, + db_namespace, + db_port, +) # "morphling-db-manager.morphling-system:6799" +channel_manager = grpc.insecure_channel(manager_server) +timeout_in_seconds = 10 +rt_slo = 1.0 # currently slo is not guaranteed +batch_size = int(os.getenv("BATCH_SIZE", 1)) +failratio_limit = float(os.getenv("FAIL_RATIO", 0.2)) +printlog = os.getenv("PRINTLOG", 'False').lower() in ('true', '1', 't') + +# Locust Settings +settings = invokust.create_settings( + locustfile=os.getenv("LOCUST_LOCUSTFILE","locustfile.py"), + num_users=os.getenv("LOCUST_NUM_USERS", 10), + spawn_rate=os.getenv("LOCUST_SPAWN_RATE", 10), + run_time=os.getenv("LOCUST_RUN_TIME", 15), + metrics_export=os.getenv("LOCUST_METRICS_EXPORT", 'False').lower() in ('true', '1', 't'), + loglevel=os.getenv("LOCUST_LOGLEVEL", "INFO") + ) +loadtest = invokust.LocustLoadTest(settings) + +def do_inference(): + """Tests PredictionService with concurrent requests. + + Returns: + The QPS and classification error rate. + """ + loadtest.run() + stats = loadtest.stats() + mean_error_rate = stats['fail_ratio'] + stats = list(stats["requests"].values())[0] # extract the dict, only need the first item + if mean_error_rate > failratio_limit: + stats['total_rps'] = -1 # if the error rate is too high, then clear the result directly + else: + stats['total_rps'] *= batch_size # take batch_size into account + return mean_error_rate, stats['median_response_time'], stats['total_rps'] + +def main(): + error_rate, rt, qps_real = do_inference() # first try + while qps_real == -1 and loadtest.settings.num_users > 10 : # if num_users is too large + time.sleep(5) + loadtest.settings.num_users /= 2 # halve the value + error_rate, rt, qps_real = do_inference() + + if printlog: + print( + "\nQPS_real: %s, Inference error rate: %s%%, RT: %s" + % (qps_real, error_rate * 100, np.mean(rt)) + ) + mls = [] + ml = api_pb2.KeyValue(key="qps", value=str(qps_real)) + mls.append(ml) + + stub_ = api_pb2_grpc.DBStub(channel_manager) + result = stub_.SaveResult( + api_pb2.SaveResultRequest( + trial_name=os.environ["TrialName"], + namespace=os.environ["Namespace"], + results=mls, + ), + timeout=timeout_in_seconds, + ) + if printlog: + print(result) + +if __name__ == "__main__": + main() diff --git a/pkg/client_locust/requirements.txt b/pkg/client_locust/requirements.txt new file mode 100644 index 0000000..c6dfafa --- /dev/null +++ b/pkg/client_locust/requirements.txt @@ -0,0 +1,51 @@ +absl-py===0.9.0 +appier===1.20.5 +astor===0.8.1 +astunparse===1.6.3 +cachetools===4.1.1 +certifi===2020.6.20 +chardet===3.0.4 +gast===0.3.3 +google-api===0.1.12 +google-auth===1.23.0 +google-auth-oauthlib===0.4.2 +google-pasta===0.2.0 +grpcio===1.30.0 +h5py===2.10.0 +idna===2.10 +importlib-metadata===1.7.0 +joblib===0.16.0 +Keras-Applications===1.0.8 +Keras-Preprocessing===1.1.2 +Markdown===3.2.2 +numpy===1.18.5 +oauthlib===3.1.0 +opt-einsum===3.3.0 +pip===20.2.4 +protobuf===3.12.2 +pyasn1===0.4.8 +pyasn1-modules===0.2.8 +requests===2.24.0 +requests-oauthlib===1.3.0 +rfc3339===6.2 +rsa===4.6 +scikit-learn===0.23.1 +scipy===1.5.2 +setuptools===49.2.0 +six===1.15.0 +style===1.1.0 +tensorboard===2.3.0 +tensorboard-plugin-wit===1.7.0 +tensorflow===2.3.1 +tensorflow-estimator===2.3.0 +tensorflow-serving-api===2.3.0 +termcolor===1.1.0 +threadpoolctl===2.1.0 +update===0.0.1 +urllib3===1.25.11 +Werkzeug===2.0.0 +wheel===0.34.2 +wrapt===1.12.1 +zipp===3.1.0 +locust===2.8.4 +prometheus-client===0.14.1