From 995d8b60f3e84cf7fb1b582edeb2ddc4fea6561c Mon Sep 17 00:00:00 2001 From: Tibor Reiss <75096465+tibor-reiss@users.noreply.github.com> Date: Sun, 16 Jun 2024 15:19:17 +0200 Subject: [PATCH] feat(langchain): use callbacks (#1170) --- .../instrumentation/langchain/__init__.py | 72 +++--- .../langchain/callback_wrapper.py | 101 ++++++++ .../poetry.lock | 141 ++++++++++- .../pyproject.toml | 1 + .../test_chains/test_asequential_chain.yaml | 231 ++++++++++++++++++ .../test_sequential_chain.yaml | 141 +++++++++++ .../tests/conftest.py | 2 + .../tests/test_chains.py | 94 ++++++- .../tests/test_documents_chains.py | 55 +++++ 9 files changed, 802 insertions(+), 36 deletions(-) create mode 100644 packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_wrapper.py create mode 100644 packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_chains/test_asequential_chain.yaml create mode 100644 packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_documents_chains/test_sequential_chain.yaml create mode 100644 packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py index 54797fbe6..f91bef27b 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py @@ -30,6 +30,8 @@ from opentelemetry.semconv.ai import TraceloopSpanKindValues +from opentelemetry.instrumentation.langchain.callback_wrapper import callback_wrapper + logger = logging.getLogger(__name__) _instruments = ("langchain >= 0.0.346", "langchain-core > 0.1.0") @@ -37,29 +39,27 @@ WRAPPED_METHODS = [ { "package": "langchain.chains.base", - "object": "Chain", - "method": "__call__", - "wrapper": task_wrapper, + "class": "Chain", + "is_callback": True, + "kind": TraceloopSpanKindValues.TASK.value, }, { - "package": "langchain.chains.base", - "object": "Chain", - "method": "acall", - "wrapper": atask_wrapper, + "package": "langchain.chains.llm", + "class": "LLMChain", + "is_callback": True, + "kind": TraceloopSpanKindValues.TASK.value, }, { - "package": "langchain.chains", - "object": "SequentialChain", - "method": "__call__", - "span_name": "langchain.workflow", - "wrapper": workflow_wrapper, + "package": "langchain.chains.combine_documents.stuff", + "class": "StuffDocumentsChain", + "is_callback": True, + "kind": TraceloopSpanKindValues.TASK.value, }, { "package": "langchain.chains", - "object": "SequentialChain", - "method": "acall", - "span_name": "langchain.workflow", - "wrapper": aworkflow_wrapper, + "class": "SequentialChain", + "is_callback": True, + "kind": TraceloopSpanKindValues.WORKFLOW.value, }, { "package": "langchain.agents", @@ -173,21 +173,33 @@ def _instrument(self, **kwargs): tracer = get_tracer(__name__, __version__, tracer_provider) for wrapped_method in WRAPPED_METHODS: wrap_package = wrapped_method.get("package") - wrap_object = wrapped_method.get("object") - wrap_method = wrapped_method.get("method") - wrapper = wrapped_method.get("wrapper") - wrap_function_wrapper( - wrap_package, - f"{wrap_object}.{wrap_method}" if wrap_object else wrap_method, - wrapper(tracer, wrapped_method), - ) + if wrapped_method.get("is_callback"): + wrap_class = wrapped_method.get("class") + wrap_function_wrapper( + wrap_package, + f"{wrap_class}.__init__", + callback_wrapper(tracer, wrapped_method), + ) + else: + wrap_object = wrapped_method.get("object") + wrap_method = wrapped_method.get("method") + wrapper = wrapped_method.get("wrapper") + wrap_function_wrapper( + wrap_package, + f"{wrap_object}.{wrap_method}" if wrap_object else wrap_method, + wrapper(tracer, wrapped_method), + ) def _uninstrument(self, **kwargs): for wrapped_method in WRAPPED_METHODS: wrap_package = wrapped_method.get("package") - wrap_object = wrapped_method.get("object") - wrap_method = wrapped_method.get("method") - unwrap( - f"{wrap_package}.{wrap_object}" if wrap_object else wrap_package, - wrap_method, - ) + if wrapped_method.get("is_callback"): + wrap_class = wrapped_method.get("class") + unwrap(wrap_package, f"{wrap_class}.__init__") + else: + wrap_object = wrapped_method.get("object") + wrap_method = wrapped_method.get("method") + unwrap( + f"{wrap_package}.{wrap_object}" if wrap_object else wrap_package, + wrap_method, + ) diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_wrapper.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_wrapper.py new file mode 100644 index 000000000..1b69fa56b --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_wrapper.py @@ -0,0 +1,101 @@ +import json +from typing import Any, Dict + +from langchain_core.callbacks import BaseCallbackHandler +from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.semconv.ai import SpanAttributes +from opentelemetry.trace import set_span_in_context, Tracer +from opentelemetry.trace.span import Span + +from opentelemetry import context as context_api +from opentelemetry.instrumentation.langchain.utils import ( + _with_tracer_wrapper, +) + + +class CustomJsonEncode(json.JSONEncoder): + def default(self, o: Any) -> str: + try: + return super().default(o) + except TypeError: + return str(o) + + +def get_name(to_wrap, instance) -> str: + return f"{instance.get_name()}.langchain.{to_wrap.get('kind')}" + + +def get_kind(to_wrap) -> str: + return to_wrap.get("kind") + + +@_with_tracer_wrapper +def callback_wrapper(tracer, to_wrap, wrapped, instance, args, kwargs): + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return wrapped(*args, **kwargs) + kind = get_kind(to_wrap) + name = get_name(to_wrap, instance) + cb = SyncSpanCallbackHandler(tracer, name, kind) + if "callbacks" in kwargs: + if not any(isinstance(c, SyncSpanCallbackHandler) for c in kwargs["callbacks"]): + # Avoid adding the same callback twice, e.g. SequentialChain is also a Chain + kwargs["callbacks"].append(cb) + else: + kwargs["callbacks"] = [ + cb, + ] + return wrapped(*args, **kwargs) + + +class SyncSpanCallbackHandler(BaseCallbackHandler): + def __init__(self, tracer: Tracer, name: str, kind: str) -> None: + self.tracer = tracer + self.name = name + self.kind = kind + self.span: Span + + def _create_span(self) -> None: + self.span = self.tracer.start_span(self.name) + self.span.set_attribute(SpanAttributes.TRACELOOP_SPAN_KIND, self.kind) + self.span.set_attribute(SpanAttributes.TRACELOOP_ENTITY_NAME, self.name) + + current_context = set_span_in_context(self.span) + context_api.attach(current_context) + + def on_chain_start( + self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any + ) -> None: + """Run when chain starts running.""" + self._create_span() + self.span.set_attribute( + SpanAttributes.TRACELOOP_ENTITY_INPUT, + json.dumps({"inputs": inputs, "kwargs": kwargs}, cls=CustomJsonEncode), + ) + + def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None: + """Run when chain ends running.""" + self.span.set_attribute( + SpanAttributes.TRACELOOP_ENTITY_OUTPUT, + json.dumps({"outputs": outputs, "kwargs": kwargs}, cls=CustomJsonEncode), + ) + self.span.end() + + def on_tool_start( + self, serialized: Dict[str, Any], input_str: str, **kwargs: Any + ) -> None: + """Run when tool starts running.""" + self._create_span() + self.span.set_attribute( + SpanAttributes.TRACELOOP_ENTITY_INPUT, + json.dumps( + {"input_str": input_str, "kwargs": kwargs}, cls=CustomJsonEncode + ), + ) + + def on_tool_end(self, output: Any, **kwargs: Any) -> None: + """Run when tool ends running.""" + self.span.set_attribute( + SpanAttributes.TRACELOOP_ENTITY_OUTPUT, + json.dumps({"output": output, "kwargs": kwargs}, cls=CustomJsonEncode), + ) + self.span.end() diff --git a/packages/opentelemetry-instrumentation-langchain/poetry.lock b/packages/opentelemetry-instrumentation-langchain/poetry.lock index 713e04ed0..d08a0d9f2 100644 --- a/packages/opentelemetry-instrumentation-langchain/poetry.lock +++ b/packages/opentelemetry-instrumentation-langchain/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -363,6 +363,28 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "cohere" +version = "5.5.3" +description = "" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "cohere-5.5.3-py3-none-any.whl", hash = "sha256:99d20129713a6dae052368b4839773a214592a76bee345b94a4846d00f702da3"}, + {file = "cohere-5.5.3.tar.gz", hash = "sha256:8c7ebe2f5bf83fee8e55a24a0acdd4b0e94de274fd0ef32b285978289a03e930"}, +] + +[package.dependencies] +boto3 = ">=1.34.0,<2.0.0" +fastavro = ">=1.9.4,<2.0.0" +httpx = ">=0.21.2" +httpx-sse = ">=0.4.0,<0.5.0" +pydantic = ">=1.9.2" +requests = ">=2.0.0,<3.0.0" +tokenizers = ">=0.19,<0.20" +types-requests = ">=2.0.0,<3.0.0" +typing_extensions = ">=4.0.0" + [[package]] name = "colorama" version = "0.4.6" @@ -442,6 +464,52 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "fastavro" +version = "1.9.4" +description = "Fast read/write of AVRO files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastavro-1.9.4-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:60cb38f07462a7fb4e4440ed0de67d3d400ae6b3d780f81327bebde9aa55faef"}, + {file = "fastavro-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:063d01d197fc929c20adc09ca9f0ca86d33ac25ee0963ce0b438244eee8315ae"}, + {file = "fastavro-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87a9053fcfbc895f2a16a4303af22077e3a8fdcf1cd5d6ed47ff2ef22cbba2f0"}, + {file = "fastavro-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:02bf1276b7326397314adf41b34a4890f6ffa59cf7e0eb20b9e4ab0a143a1598"}, + {file = "fastavro-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56bed9eca435389a8861e6e2d631ec7f8f5dda5b23f93517ac710665bd34ca29"}, + {file = "fastavro-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:0cd2099c8c672b853e0b20c13e9b62a69d3fbf67ee7c59c7271ba5df1680310d"}, + {file = "fastavro-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:af8c6d8c43a02b5569c093fc5467469541ac408c79c36a5b0900d3dd0b3ba838"}, + {file = "fastavro-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4a138710bd61580324d23bc5e3df01f0b82aee0a76404d5dddae73d9e4c723f"}, + {file = "fastavro-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:903d97418120ca6b6a7f38a731166c1ccc2c4344ee5e0470d09eb1dc3687540a"}, + {file = "fastavro-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c443eeb99899d062dbf78c525e4614dd77e041a7688fa2710c224f4033f193ae"}, + {file = "fastavro-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ac26ab0774d1b2b7af6d8f4300ad20bbc4b5469e658a02931ad13ce23635152f"}, + {file = "fastavro-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:cf7247874c22be856ba7d1f46a0f6e0379a6025f1a48a7da640444cbac6f570b"}, + {file = "fastavro-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:68912f2020e1b3d70557260b27dd85fb49a4fc6bfab18d384926127452c1da4c"}, + {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6925ce137cdd78e109abdb0bc33aad55de6c9f2d2d3036b65453128f2f5f5b92"}, + {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b928cd294e36e35516d0deb9e104b45be922ba06940794260a4e5dbed6c192a"}, + {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:90c9838bc4c991ffff5dd9d88a0cc0030f938b3fdf038cdf6babde144b920246"}, + {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:eca6e54da571b06a3c5a72dbb7212073f56c92a6fbfbf847b91c347510f8a426"}, + {file = "fastavro-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a4b02839ac261100cefca2e2ad04cdfedc556cb66b5ec735e0db428e74b399de"}, + {file = "fastavro-1.9.4-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:4451ee9a305a73313a1558d471299f3130e4ecc10a88bf5742aa03fb37e042e6"}, + {file = "fastavro-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8524fccfb379565568c045d29b2ebf71e1f2c0dd484aeda9fe784ef5febe1a8"}, + {file = "fastavro-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33d0a00a6e09baa20f6f038d7a2ddcb7eef0e7a9980e947a018300cb047091b8"}, + {file = "fastavro-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:23d7e5b29c9bf6f26e8be754b2c8b919838e506f78ef724de7d22881696712fc"}, + {file = "fastavro-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e6ab3ee53944326460edf1125b2ad5be2fadd80f7211b13c45fa0c503b4cf8d"}, + {file = "fastavro-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:64d335ec2004204c501f8697c385d0a8f6b521ac82d5b30696f789ff5bc85f3c"}, + {file = "fastavro-1.9.4-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:7e05f44c493e89e73833bd3ff3790538726906d2856f59adc8103539f4a1b232"}, + {file = "fastavro-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:253c63993250bff4ee7b11fb46cf3a4622180a783bedc82a24c6fdcd1b10ca2a"}, + {file = "fastavro-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24d6942eb1db14640c2581e0ecd1bbe0afc8a83731fcd3064ae7f429d7880cb7"}, + {file = "fastavro-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d47bb66be6091cd48cfe026adcad11c8b11d7d815a2949a1e4ccf03df981ca65"}, + {file = "fastavro-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c293897f12f910e58a1024f9c77f565aa8e23b36aafda6ad8e7041accc57a57f"}, + {file = "fastavro-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:f05d2afcb10a92e2a9e580a3891f090589b3e567fdc5641f8a46a0b084f120c3"}, + {file = "fastavro-1.9.4.tar.gz", hash = "sha256:56b8363e360a1256c94562393dc7f8611f3baf2b3159f64fb2b9c6b87b14e876"}, +] + +[package.extras] +codecs = ["cramjam", "lz4", "zstandard"] +lz4 = ["lz4"] +snappy = ["cramjam"] +zstandard = ["zstandard"] + [[package]] name = "filelock" version = "3.14.0" @@ -725,6 +793,17 @@ cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +[[package]] +name = "httpx-sse" +version = "0.4.0" +description = "Consume Server-Sent Event (SSE) messages with HTTPX." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-sse-0.4.0.tar.gz", hash = "sha256:1e81a3a3070ce322add1d3529ed42eb5f70817f45ed6ec915ab753f961139721"}, + {file = "httpx_sse-0.4.0-py3-none-any.whl", hash = "sha256:f329af6eae57eaa2bdfd962b42524764af68075ea87370a2de920af5341e318f"}, +] + [[package]] name = "huggingface-hub" version = "0.23.1" @@ -963,6 +1042,21 @@ anthropic = ">=0.23.0,<1" defusedxml = ">=0.7.1,<0.8.0" langchain-core = ">=0.1.43,<0.3" +[[package]] +name = "langchain-cohere" +version = "0.1.5" +description = "An integration package connecting Cohere and LangChain" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_cohere-0.1.5-py3-none-any.whl", hash = "sha256:f07bd53fadbebf744b8de1eebf977353f340f2010156821623a0c6247032ab9b"}, + {file = "langchain_cohere-0.1.5.tar.gz", hash = "sha256:d0be4e76079a74c4259fe4db2bab535d690efe0efac5e9e2fbf486476c0a85c8"}, +] + +[package.dependencies] +cohere = ">=5.5,<6.0" +langchain-core = ">=0.1.42,<0.3" + [[package]] name = "langchain-community" version = "0.0.29" @@ -1301,7 +1395,7 @@ wrapt = ">=1.0.0,<2.0.0" [[package]] name = "opentelemetry-instrumentation-bedrock" -version = "0.21.5" +version = "0.22.1" description = "OpenTelemetry Bedrock instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1321,7 +1415,7 @@ url = "../opentelemetry-instrumentation-bedrock" [[package]] name = "opentelemetry-instrumentation-openai" -version = "0.21.5" +version = "0.22.1" description = "OpenTelemetry OpenAI instrumentation" optional = false python-versions = ">=3.9,<4" @@ -2243,6 +2337,45 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] +[[package]] +name = "types-requests" +version = "2.31.0.6" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.7" +files = [ + {file = "types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0"}, + {file = "types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9"}, +] + +[package.dependencies] +types-urllib3 = "*" + +[[package]] +name = "types-requests" +version = "2.32.0.20240602" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-requests-2.32.0.20240602.tar.gz", hash = "sha256:3f98d7bbd0dd94ebd10ff43a7fbe20c3b8528acace6d8efafef0b6a184793f06"}, + {file = "types_requests-2.32.0.20240602-py3-none-any.whl", hash = "sha256:ed3946063ea9fbc6b5fc0c44fa279188bae42d582cb63760be6cb4b9d06c3de8"}, +] + +[package.dependencies] +urllib3 = ">=2" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + [[package]] name = "typing-extensions" version = "4.12.0" @@ -2524,4 +2657,4 @@ instruments = [] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4" -content-hash = "76933aa0bb5a13ac8e355a62de66f6a8498fc15cc07b1dbd19deefb60f66ff10" +content-hash = "294ea1e323ff6cfb8ea3d089536575a1e86cea66b3dcd1faa1d6b9cfe2dddf03" diff --git a/packages/opentelemetry-instrumentation-langchain/pyproject.toml b/packages/opentelemetry-instrumentation-langchain/pyproject.toml index 01a369a95..38a3b7a0b 100644 --- a/packages/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/packages/opentelemetry-instrumentation-langchain/pyproject.toml @@ -53,6 +53,7 @@ anthropic = ">=0.23,<0.29" boto3 = "^1.34.120" langchain-anthropic = "^0.1.11" langchain-openai = "^0.1.6" +langchain-cohere = "0.1.5" pydantic = "^2.7.1" [build-system] diff --git a/packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_chains/test_asequential_chain.yaml b/packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_chains/test_asequential_chain.yaml new file mode 100644 index 000000000..ebd1bf378 --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_chains/test_asequential_chain.yaml @@ -0,0 +1,231 @@ +interactions: +- request: + body: '{"model": "gpt-3.5-turbo-instruct", "prompt": ["You are a playwright. Given + the title of play and the era it is set in, it is your job to write a synopsis + for that title.\n\n Title: Tragedy at sunset on the beach\n Era: Victorian + England\n Playwright: This is a synopsis for the above play:"], "frequency_penalty": + 0, "logit_bias": {}, "max_tokens": 256, "n": 1, "presence_penalty": 0, "temperature": + 0.7, "top_p": 1}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '426' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.12.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.12.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.1 + method: POST + uri: https://api.openai.com/v1/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xTTW8TMRC951eM9sJlE6UtaUtuIEACpPJVqCqCqok9u2vijLeecZpQ9b8je7ft + hYtle+Y9v3meuZ8AVM5WS6jMtvfT87vb62/XF7dvr67+Xnzpvm4+Hf94f7U+Pz24j5+rOmeH9R8y + mhFKe70xYdt7Uhd4CJtIqJQZj87mr+Yni7OTRQlsgyWfYW2v05PZYqoprsPUsWhMRkd0F5whqZbw + awIAcF9WGN7KYFjxij8waEewdVYUQlMOotEZBWQLfQw9RZBgHOkhJ/x0RkN0yPCOW49sa0BoY0h9 + jjbREVuBFrWjCGFg7wkNNcmDdCGS5EQEIeOTJQtrQtOBBrhDzZusILGQzuC1lCPtiB23kLgJ3koN + SiwusEB0QkWokImkAhgJIu0IPdkaEkfckfcZPAiJDRmFBg1aGut1EYRo67j1h6cM73YkTwIkMUih + ZwsW44ZJBCyJKdWmfiy0lFKDRmzJHoqRG5IaPOHuUcPgVROyygimQ27JzuCyiyG1XTEmusGkxqN0 + azSb4WHHShFNbpDsQUeAyTpiQ3DnNEuiZ780b1HBk4XUZ3u1cwINavmJR0szb4aYwEK3KZONwCZ4 + H+5mcOW0fMp20OTDjmpYk0Y8oK8H80t7oAfa92QUR4Wr6nI0AhW+ly997Ig32ahVBZb8jiRXFkYZ + eQT2TkcHurRFBkZNkf6vNSc5m0sJTC8ENCYCId/MoMxBaXnHlvbVEuZPNz60fQzrPB6cvH+6bxw7 + 6W4ioQTOUyIa+qpEHyYAv8toJcGWquU4UlUfw7bXGw0b4kx4ejTQVc8D/Rw8erkYoxoU/XPgeH46 + yY88TP4BAAD//wMARyqwekoEAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 85c050e1ba1a0e19-MXP + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 27 Feb 2024 12:08:57 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=tweHeZeMtVPJLUV7SZB26Lipwpw.BQ1.4X_Y6CSKRRc-1709035737-1.0-AT33S2bZA8bffZwwKiVZne+rlqct/seEe76WNfaaj+/EMZBFJwAE9pYDb1NwytfKNW3NYdRhWae+H0Ezuy6Pgf0=; + path=/; expires=Tue, 27-Feb-24 12:38:57 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=oa6mlrcQaWGFRnmczT1FQHoA3dVLo.ySSS881dJvSvM-1709035737088-0.0-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-3.5-turbo-instruct + openai-organization: + - traceloop + openai-processing-ms: + - '1553' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-limit-tokens: + - '250000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-remaining-tokens: + - '249682' + x-ratelimit-reset-requests: + - 20ms + x-ratelimit-reset-tokens: + - 76ms + x-request-id: + - req_c0e8c1f38e972bfbc4998f99c2421b3a + status: + code: 200 + message: OK +- request: + body: '{"model": "gpt-3.5-turbo-instruct", "prompt": ["You are a play critic from + the New York Times. Given the synopsis of play, it is your job to write a review + for that play.\n\n Play Synopsis:\n \n\nIn the midst of the strict and + proper society of Victorian England, a group of friends gather on the peaceful + shores of a secluded beach to watch the sunset. As the evening unfolds, tensions + rise and secrets are revealed, unravelling the perfect facade of their seemingly + perfect lives. As the sun sets and darkness descends upon the beach, tragedy + strikes, leaving the group forever changed. Through a series of flashbacks and + interactions, the audience witnesses the events that led up to this fateful + evening and the consequences that follow. With themes of love, betrayal, and + societal expectations, \"Tragedy at Sunset on the Beach\" delves into the complexities + of human nature and the consequences of hiding one''s true self. \n Review + from a New York Times play critic of the above play:"], "frequency_penalty": + 0, "logit_bias": {}, "max_tokens": 256, "n": 1, "presence_penalty": 0, "temperature": + 0.7, "top_p": 1}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1127' + content-type: + - application/json + cookie: + - __cf_bm=tweHeZeMtVPJLUV7SZB26Lipwpw.BQ1.4X_Y6CSKRRc-1709035737-1.0-AT33S2bZA8bffZwwKiVZne+rlqct/seEe76WNfaaj+/EMZBFJwAE9pYDb1NwytfKNW3NYdRhWae+H0Ezuy6Pgf0=; + _cfuvid=oa6mlrcQaWGFRnmczT1FQHoA3dVLo.ySSS881dJvSvM-1709035737088-0.0-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.12.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.12.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.1 + method: POST + uri: https://api.openai.com/v1/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xUS28kRQy+51dYc55EWcIqkCPSckFiEYmQlg1aearc3Sbucsfl6sloxX9Hrp6E + PXAZTdfD/vw96usFwI7z7g52aV7k8ofj86eHj8/r7zd4P2XkX75797Pd/zl+/ITf024fp/XwNyWP + G04v/iXpvAg5a9m2kxE6RcV3t9c/Xt+8v7257RuzZpK4Ni5+eXP1/tKbHfSSS3Vryc+3J+VEdXcH + ny8AAL72X9h6xeXH8lgedw+GI+UToMN9K5UctIBPBD8RpulxB1wBwSdt4+SXi+mqT1xGwJJhNF6W + +FgET+ATOjg+UY3vRMBbIeORcz+/mC5kUDUx+Ql0gD84uRpjgQ9lFCz5Ch4m2uolKk5WAU1byYAw + mrYlbg3GVHKF46Qwok9kARqhUpKWKcMhsIMrHNHjz0RQ+3B7ODQHrH2JViodveloVCvVPTiVyloq + GFfqoCslIw8YBEYroVDeQxPnGZ3kBEKYo4xrEGU4cuqlfWMkTVhG6h3ZQHilCoNGJbsKCX4TPB2N + x8nh8684019Qn1hkaCInOBLGedeR+pgI1dU6daIr7eFAbnhC2W9YO7MoQC8LJUfvo3QduELCxXlF + P8+cW4r9jfE0oWE6801wJJHLTCuJLrSJt+E3kq3qxEvt67NuRaNTXKWXRdQoR9tMiwf9Fu7pAhlT + 7RIK1umA6WkrwiE1djz1Cj6Eem+IwoGDWqIcFCctg2knN/DosQCXQgaZ5g5hgxrnKj03KmlrOPEm + Ur/l1ggqSXDL3TlnS3bFVpRGtXdSm9lPgAddCVAESCp12YK06ji+ZiGSk6nyWKKbv5o4GKkcuYYT + OdAwUHIO5VBEj2dIgC1zQI0JB02tvqZwexNeegsKqrWgQGYUHRuF/Eei89E3CTdJW6VXKFseMJIc + nGfTBTDnMFbf15UspkOftS4TGX07xB6O7FuK3LA8Nxbe4tuDRd4dVcnWTkbtLkV7CgbdsPprG282 + K8vGciuDSu6DBo//seookfwMVCrNByFIUSNTZMcqLHokG5p0TkrDEs5YyEKs+Kh7OBiX8ZxJ4YF6 + 9zkyO2CiqP2t38+O4eLGCdPZof/j984rG6SJZq4eOTyDD48uKAsG3CjY2T3bX/uD3N9eLpledndw + /bYiOi6mh3inSxN5Wx+4cJ2+GGHVEs+1UBl92vX9fy7+lYJCLLiULy1OTE9VsoKW7koFRfm5BSXx + JfnZqXkgIw0tjSAmKiEqF4SskakZVLYkvyQxByFhYmLBBbKllgsAAAD//wMAcIMQMtYGAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 85c050ed49520e19-MXP + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 27 Feb 2024 12:09:00 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-3.5-turbo-instruct + openai-organization: + - traceloop + openai-processing-ms: + - '3284' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3000' + x-ratelimit-limit-tokens: + - '250000' + x-ratelimit-remaining-requests: + - '2999' + x-ratelimit-remaining-tokens: + - '249507' + x-ratelimit-reset-requests: + - 20ms + x-ratelimit-reset-tokens: + - 118ms + x-request-id: + - req_2c7e25c90aa81889ede319a3fea4ca53 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_documents_chains/test_sequential_chain.yaml b/packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_documents_chains/test_sequential_chain.yaml new file mode 100644 index 000000000..5b2a40c3f --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/tests/cassettes/test_documents_chains/test_sequential_chain.yaml @@ -0,0 +1,141 @@ +interactions: +- request: + body: '{"message": "Write a concise summary of the following:\n\n\n\"Today, all + ridges and faces of the Matterhorn have been ascended in all seasons,\n and + mountain guides take a large number of people up the northeast H\u00f6rnli route\n each + summer. In total, up to 150 climbers attempt the Matterhorn each day during\n summer. + By modern standards, the climb is fairly difficult (AD Difficulty rating),\n but + not hard for skilled mountaineers according to French climbing grades. There\n are + fixed ropes on parts of the route to help. Still, it should be remembered that\n several + climbers may die on the mountain each year.\n The usual pattern of ascent + is to take the Schwarzsee cable car up from Zermatt,\n hike up to the H\u00f6rnli + Hut elev. 3,260 m (10,700 ft), a large stone building at the\n base of the + main ridge, and spend the night. The next day, climbers rise at 3:30 am\n so + as to reach the summit and descend before the regular afternoon clouds and storms\n come + in. The Solvay Hut located on the ridge at 4,003 m (13,133 ft) can be used only\n in + a case of emergency.\n Other popular routes on the mountain include the Italian + (Lion) ridge (AD+ Difficulty\n rating) and the Zmutt ridge (D Difficulty + rating). The four faces, as well as the\n Furggen ridge, constitute the most + challenging routes to the summit. The north face\n is amongst the six most + difficult faces of the Alps, as well as \u2018The Trilogy\u2019, the\n three + hardest of the six, along with the north faces of the Eiger and the Grandes\n Jorasses + (TD+ Difficulty rating).\"\n\n\nCONCISE SUMMARY:", "stream": false, "model": + "command", "chat_history": [], "temperature": 0.75}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1709' + content-type: + - application/json + host: + - api.cohere.com + user-agent: + - python-httpx/0.27.0 + x-client-name: + - langchain:partner + x-fern-language: + - Python + x-fern-sdk-name: + - cohere + x-fern-sdk-version: + - 5.5.3 + method: POST + uri: https://api.cohere.com/v1/chat + response: + body: + string: "{\"response_id\":\"aaef5a62-6db8-436b-b291-a560d69d5188\",\"text\":\"The + Matterhorn, a mountain in the Alps, is a popular climbing destination, with + up to 150 climbers attempting the summit each day during the summer months. + Several climbing routes are available, ranging from moderately difficult (AD + Difficulty rating) to challenging (D Difficulty rating) and requiring skill + and preparedness. Fixed ropes are in place on some routes to assist climbers. + While the climb is achievable for skilled mountaineers, it is important to + note that it is not without risk, and several climbers may die on the mountain + each year. Hikers typically spend the night at the H\xF6rnli Hut before ascending + to the summit the following day and descending before afternoon clouds and + storms roll in. The Matterhorn's north face is renowned for its difficulty + and is considered one of the six most challenging faces of the Alps. Overall, + the Matterhorn presents a range of climbing opportunities for experienced + and aspiring mountaineers.\",\"generation_id\":\"73f5de94-ec1a-4633-86a4-491494401978\",\"chat_history\":[{\"role\":\"USER\",\"message\":\"Write + a concise summary of the following:\\n\\n\\n\\\"Today, all ridges and faces + of the Matterhorn have been ascended in all seasons,\\n and mountain guides + take a large number of people up the northeast H\xF6rnli route\\n each + summer. In total, up to 150 climbers attempt the Matterhorn each day during\\n + \ summer. By modern standards, the climb is fairly difficult (AD Difficulty + rating),\\n but not hard for skilled mountaineers according to French climbing + grades. There\\n are fixed ropes on parts of the route to help. Still, + it should be remembered that\\n several climbers may die on the mountain + each year.\\n The usual pattern of ascent is to take the Schwarzsee cable + car up from Zermatt,\\n hike up to the H\xF6rnli Hut elev. 3,260 m (10,700 + ft), a large stone building at the\\n base of the main ridge, and spend + the night. The next day, climbers rise at 3:30 am\\n so as to reach the + summit and descend before the regular afternoon clouds and storms\\n come + in. The Solvay Hut located on the ridge at 4,003 m (13,133 ft) can be used + only\\n in a case of emergency.\\n Other popular routes on the mountain + include the Italian (Lion) ridge (AD+ Difficulty\\n rating) and the Zmutt + ridge (D Difficulty rating). The four faces, as well as the\\n Furggen + ridge, constitute the most challenging routes to the summit. The north face\\n + \ is amongst the six most difficult faces of the Alps, as well as \u2018The + Trilogy\u2019, the\\n three hardest of the six, along with the north faces + of the Eiger and the Grandes\\n Jorasses (TD+ Difficulty rating).\\\"\\n\\n\\nCONCISE + SUMMARY:\"},{\"role\":\"CHATBOT\",\"message\":\"The Matterhorn, a mountain + in the Alps, is a popular climbing destination, with up to 150 climbers attempting + the summit each day during the summer months. Several climbing routes are + available, ranging from moderately difficult (AD Difficulty rating) to challenging + (D Difficulty rating) and requiring skill and preparedness. Fixed ropes are + in place on some routes to assist climbers. While the climb is achievable + for skilled mountaineers, it is important to note that it is not without risk, + and several climbers may die on the mountain each year. Hikers typically spend + the night at the H\xF6rnli Hut before ascending to the summit the following + day and descending before afternoon clouds and storms roll in. The Matterhorn's + north face is renowned for its difficulty and is considered one of the six + most challenging faces of the Alps. Overall, the Matterhorn presents a range + of climbing opportunities for experienced and aspiring mountaineers.\"}],\"finish_reason\":\"COMPLETE\",\"meta\":{\"api_version\":{\"version\":\"1\"},\"billed_units\":{\"input_tokens\":420,\"output_tokens\":180},\"tokens\":{\"input_tokens\":431,\"output_tokens\":181}}}" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Via: + - 1.1 google + access-control-expose-headers: + - X-Debug-Trace-ID + cache-control: + - no-cache, no-store, no-transform, must-revalidate, private, max-age=0 + content-type: + - application/json + date: + - Sat, 15 Jun 2024 20:16:51 GMT + expires: + - Thu, 01 Jan 1970 00:00:00 UTC + num_chars: + - '1863' + num_tokens: + - '600' + pragma: + - no-cache + server: + - envoy + transfer-encoding: + - chunked + vary: + - Origin + x-accel-expires: + - '0' + x-debug-trace-id: + - b4c98b43bda3a122914a234994c993a0 + x-endpoint-monthly-call-limit: + - '1000' + x-envoy-upstream-service-time: + - '6063' + x-trial-endpoint-call-limit: + - '10' + x-trial-endpoint-call-remaining: + - '9' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-langchain/tests/conftest.py b/packages/opentelemetry-instrumentation-langchain/tests/conftest.py index 05f6db0d5..1e8055ec0 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/conftest.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -40,6 +40,8 @@ def environment(): os.environ["OPENAI_API_KEY"] = "test_api_key" if not os.environ.get("ANTHROPIC_API_KEY"): os.environ["ANTHROPIC_API_KEY"] = "test" + if not os.environ.get("COHERE_API_KEY"): + os.environ["COHERE_API_KEY"] = "test" @pytest.fixture(scope="module") diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py b/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py index ab1009d84..a2db228a9 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_chains.py @@ -1,7 +1,10 @@ +import json + import pytest -from langchain.prompts import PromptTemplate from langchain.chains import SequentialChain, LLMChain +from langchain.prompts import PromptTemplate from langchain_openai import OpenAI +from opentelemetry.semconv.ai import SpanAttributes @pytest.mark.vcr @@ -45,6 +48,93 @@ def test_sequential_chain(exporter): "LLMChain.langchain.task", "openai.completion", "LLMChain.langchain.task", - "SequentialChain.langchain.task", "SequentialChain.langchain.workflow", ] == [span.name for span in spans] + + synopsis_span, review_span = [span for span in spans if span.name == "LLMChain.langchain.task"] + + data = json.loads(synopsis_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"] == {'title': 'Tragedy at sunset on the beach', 'era': 'Victorian England'} + assert data["kwargs"]["name"] == "LLMChain" + data = json.loads(synopsis_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"synopsis", } + + data = json.loads(review_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"].keys() == {"title", "era", "synopsis"} + assert data["kwargs"]["name"] == "LLMChain" + data = json.loads(review_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"review", } + + overall_span = next(span for span in spans if span.name == "SequentialChain.langchain.workflow") + data = json.loads(overall_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"] == {'title': 'Tragedy at sunset on the beach', 'era': 'Victorian England'} + assert data["kwargs"]["name"] == "SequentialChain" + data = json.loads(overall_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"synopsis", "review"} + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_asequential_chain(exporter): + llm = OpenAI(temperature=0.7) + synopsis_template = """You are a playwright. Given the title of play and the era it is set in, it is your job to write a synopsis for that title. + + Title: {title} + Era: {era} + Playwright: This is a synopsis for the above play:""" # noqa: E501 + synopsis_prompt_template = PromptTemplate( + input_variables=["title", "era"], template=synopsis_template + ) + synopsis_chain = LLMChain( + llm=llm, prompt=synopsis_prompt_template, output_key="synopsis" + ) + + template = """You are a play critic from the New York Times. Given the synopsis of play, it is your job to write a review for that play. + + Play Synopsis: + {synopsis} + Review from a New York Times play critic of the above play:""" # noqa: E501 + prompt_template = PromptTemplate(input_variables=["synopsis"], template=template) + review_chain = LLMChain(llm=llm, prompt=prompt_template, output_key="review") + + overall_chain = SequentialChain( + chains=[synopsis_chain, review_chain], + input_variables=["era", "title"], + # Here we return multiple variables + output_variables=["synopsis", "review"], + verbose=True, + ) + await overall_chain.ainvoke( + {"title": "Tragedy at sunset on the beach", "era": "Victorian England"} + ) + + spans = exporter.get_finished_spans() + + assert [ + "openai.completion", + "LLMChain.langchain.task", + "openai.completion", + "LLMChain.langchain.task", + "SequentialChain.langchain.workflow", + ] == [span.name for span in spans] + + synopsis_span, review_span = [span for span in spans if span.name == "LLMChain.langchain.task"] + + data = json.loads(synopsis_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"] == {'title': 'Tragedy at sunset on the beach', 'era': 'Victorian England'} + assert data["kwargs"]["name"] == "LLMChain" + data = json.loads(synopsis_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"synopsis", } + + data = json.loads(review_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"].keys() == {"title", "era", "synopsis"} + assert data["kwargs"]["name"] == "LLMChain" + data = json.loads(review_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"review", } + + overall_span = next(span for span in spans if span.name == "SequentialChain.langchain.workflow") + data = json.loads(overall_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"] == {'title': 'Tragedy at sunset on the beach', 'era': 'Victorian England'} + assert data["kwargs"]["name"] == "SequentialChain" + data = json.loads(overall_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"synopsis", "review"} diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py b/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py new file mode 100644 index 000000000..b1adbf78a --- /dev/null +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_documents_chains.py @@ -0,0 +1,55 @@ +import json + +import pytest +from langchain.chains.summarize import load_summarize_chain +from langchain.text_splitter import CharacterTextSplitter +from langchain_cohere import ChatCohere +from opentelemetry.semconv.ai import SpanAttributes + + +# source: wikipedia +INPUT_TEXT = """ + Today, all ridges and faces of the Matterhorn have been ascended in all seasons, + and mountain guides take a large number of people up the northeast Hörnli route + each summer. In total, up to 150 climbers attempt the Matterhorn each day during + summer. By modern standards, the climb is fairly difficult (AD Difficulty rating), + but not hard for skilled mountaineers according to French climbing grades. There + are fixed ropes on parts of the route to help. Still, it should be remembered that + several climbers may die on the mountain each year. + The usual pattern of ascent is to take the Schwarzsee cable car up from Zermatt, + hike up to the Hörnli Hut elev. 3,260 m (10,700 ft), a large stone building at the + base of the main ridge, and spend the night. The next day, climbers rise at 3:30 am + so as to reach the summit and descend before the regular afternoon clouds and storms + come in. The Solvay Hut located on the ridge at 4,003 m (13,133 ft) can be used only + in a case of emergency. + Other popular routes on the mountain include the Italian (Lion) ridge (AD+ Difficulty + rating) and the Zmutt ridge (D Difficulty rating). The four faces, as well as the + Furggen ridge, constitute the most challenging routes to the summit. The north face + is amongst the six most difficult faces of the Alps, as well as ‘The Trilogy’, the + three hardest of the six, along with the north faces of the Eiger and the Grandes + Jorasses (TD+ Difficulty rating). +""" + + +@pytest.mark.vcr +def test_sequential_chain(exporter): + small_docs = CharacterTextSplitter().create_documents(texts=[INPUT_TEXT, ]) + llm = ChatCohere(model="command", temperature=0.75) + chain = load_summarize_chain(llm, chain_type="stuff") + chain.run(small_docs) + + spans = exporter.get_finished_spans() + + assert [ + "ChatCohere.langchain.task", + "LLMChain.langchain.task", + "StuffDocumentsChain.langchain.task", + ] == [span.name for span in spans] + + stuff_span = next(span for span in spans if span.name == "StuffDocumentsChain.langchain.task") + + data = json.loads(stuff_span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT]) + assert data["inputs"].keys() == {"input_documents"} + assert data["kwargs"]["name"] == "StuffDocumentsChain" + data = json.loads(stuff_span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT]) + assert data["outputs"].keys() == {"output_text"}