diff --git a/agenta-backend/agenta_backend/services/evaluators_service.py b/agenta-backend/agenta_backend/services/evaluators_service.py index 45deb54738..e707dc7e20 100644 --- a/agenta-backend/agenta_backend/services/evaluators_service.py +++ b/agenta-backend/agenta_backend/services/evaluators_service.py @@ -65,9 +65,25 @@ async def map( """ mapping_outputs = {} - trace = process_distributed_trace_into_trace_tree(mapping_input.inputs["trace"]) + mapping_inputs = mapping_input.inputs + response_version = mapping_input.inputs.get("version") + + trace = process_distributed_trace_into_trace_tree( + trace=( + mapping_inputs["tree"] + if response_version == "3.0" + else mapping_inputs["trace"] + if response_version == "2.0" + else {} + ), + version=mapping_input.inputs.get("version"), + ) for to_key, from_key in mapping_input.mapping.items(): - mapping_outputs[to_key] = get_field_value_from_trace_tree(trace, from_key) + mapping_outputs[to_key] = get_field_value_from_trace_tree( + trace, + from_key, + version=mapping_input.inputs.get("version"), + ) return {"outputs": mapping_outputs} diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index e6b1064fe3..ea963b87b5 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -14,26 +14,92 @@ logger.setLevel(logging.DEBUG) -def extract_result_from_response(response): +def extract_result_from_response(response: dict): + def get_nested_value(d: dict, keys: list, default=None): + """ + Helper function to safely retrieve nested values. + """ + try: + for key in keys: + if isinstance(d, dict): + d = d.get(key, default) + else: + return default + return d + except Exception as e: + print(f"Error accessing nested value: {e}") + return default + + # Initialize default values value = None latency = None cost = None - if response.get("version", None) == "2.0": - value = response + try: + # Validate input + if not isinstance(response, dict): + raise ValueError("The response must be a dictionary.") + + # Handle version 3.0 response + if response.get("version") == "3.0": + value = response + # Ensure 'data' is a dictionary or convert it to a string + if not isinstance(value.get("data"), dict): + value["data"] = str(value.get("data")) + + if "tree" in response: + trace_tree = ( + response["tree"][0] + if isinstance(response.get("tree"), list) + else {} + ) + latency = ( + get_nested_value(trace_tree, ["time", "span"]) * 1_000_000 + if trace_tree + else None + ) + cost = get_nested_value( + trace_tree, ["metrics", "acc", "costs", "total"] + ) - if not isinstance(value["data"], dict): - value["data"] = str(value["data"]) + # Handle version 2.0 response + elif response.get("version") == "2.0": + value = response + if not isinstance(value.get("data"), dict): + value["data"] = str(value.get("data")) - if "trace" in response: - latency = response["trace"].get("latency", None) - cost = response["trace"].get("cost", None) - else: - value = {"data": str(response["message"])} - latency = response.get("latency", None) - cost = response.get("cost", None) + if "trace" in response: + latency = response["trace"].get("latency") + cost = response["trace"].get("cost") - kind = "text" if isinstance(value, str) else "object" + # Handle generic response (neither 2.0 nor 3.0) + else: + value = {"data": str(response.get("message", ""))} + latency = response.get("latency") + cost = response.get("cost") + + # Determine the type of 'value' (either 'text' or 'object') + kind = "text" if isinstance(value, str) else "object" + + except ValueError as ve: + print(f"Input validation error: {ve}") + value = {"error": str(ve)} + kind = "error" + + except KeyError as ke: + print(f"Missing key: {ke}") + value = {"error": f"Missing key: {ke}"} + kind = "error" + + except TypeError as te: + print(f"Type error: {te}") + value = {"error": f"Type error: {te}"} + kind = "error" + + except Exception as e: + print(f"Unexpected error: {e}") + value = {"error": f"Unexpected error: {e}"} + kind = "error" return value, kind, cost, latency diff --git a/agenta-backend/agenta_backend/tests/unit/test_evaluators.py b/agenta-backend/agenta_backend/tests/unit/test_evaluators.py index 0b4f65a00e..ecb818cb29 100644 --- a/agenta-backend/agenta_backend/tests/unit/test_evaluators.py +++ b/agenta-backend/agenta_backend/tests/unit/test_evaluators.py @@ -1,7 +1,10 @@ import os import pytest -from agenta_backend.tests.unit.test_traces import simple_rag_trace +from agenta_backend.tests.unit.test_traces import ( + simple_rag_trace, + simple_rag_trace_for_baseresponse_v3, +) from agenta_backend.services.evaluators_service import ( auto_levenshtein_distance, auto_ai_critique, @@ -535,3 +538,99 @@ async def test_rag_context_relevancy_evaluator( # - raised by evaluator (agenta) -> TypeError assert not isinstance(result.value, float) or not isinstance(result.value, int) assert result.error.message == "Error during RAG Context Relevancy evaluation" + + +@pytest.mark.parametrize( + "settings_values, expected_min, openai_api_key, expected_max", + [ + ( + { + "question_key": "rag.retriever.internals.prompt", + "answer_key": "rag.reporter.outputs.report", + "contexts_key": "rag.retriever.outputs.movies", + }, + os.environ.get("OPENAI_API_KEY"), + 0.0, + 1.0, + ), + ( + { + "question_key": "rag.retriever.internals.prompt", + "answer_key": "rag.reporter.outputs.report", + "contexts_key": "rag.retriever.outputs.movies", + }, + None, + None, + None, + ), + # add more use cases + ], +) +@pytest.mark.asyncio +async def test_rag_faithfulness_evaluator_for_baseresponse_v3( + settings_values, expected_min, openai_api_key, expected_max +): + result = await rag_faithfulness( + {}, + simple_rag_trace_for_baseresponse_v3, + {}, + {}, + settings_values, + {"OPENAI_API_KEY": openai_api_key}, + ) + + try: + assert expected_min <= round(result.value, 1) <= expected_max + except TypeError as error: + # exceptions + # - raised by evaluator (agenta) -> TypeError + assert not isinstance(result.value, float) or not isinstance(result.value, int) + + +@pytest.mark.parametrize( + "settings_values, expected_min, openai_api_key, expected_max", + [ + ( + { + "question_key": "rag.retriever.internals.prompt", + "answer_key": "rag.reporter.outputs.report", + "contexts_key": "rag.retriever.outputs.movies", + }, + os.environ.get("OPENAI_API_KEY"), + 0.0, + 1.0, + ), + ( + { + "question_key": "rag.retriever.internals.prompt", + "answer_key": "rag.reporter.outputs.report", + "contexts_key": "rag.retriever.outputs.movies", + }, + None, + None, + None, + ), + # add more use cases + ], +) +@pytest.mark.asyncio +async def test_rag_context_relevancy_evaluator_for_baseresponse_v3( + settings_values, expected_min, openai_api_key, expected_max +): + result = await rag_context_relevancy( + {}, + simple_rag_trace_for_baseresponse_v3, + {}, + {}, + settings_values, + {"OPENAI_API_KEY": openai_api_key}, + ) + + try: + assert expected_min <= round(result.value, 1) <= expected_max + except TypeError as error: + # exceptions + # - raised by autoevals -> ValueError (caught already and then passed as a stacktrace to the result) + # - raised by evaluator (agenta) -> TypeError + assert not isinstance(result.value, float) or not isinstance(result.value, int) + assert result.error.message == "Error during RAG Context Relevancy evaluation" diff --git a/agenta-backend/agenta_backend/tests/unit/test_traces.py b/agenta-backend/agenta_backend/tests/unit/test_traces.py index 664eb4a4bb..150c119bfa 100644 --- a/agenta-backend/agenta_backend/tests/unit/test_traces.py +++ b/agenta-backend/agenta_backend/tests/unit/test_traces.py @@ -1,4 +1,5 @@ simple_rag_trace = { + "version": "2.0", "data": {}, "trace": { "trace_id": "66a2862603cbee93a25914ad", @@ -72,6 +73,7 @@ simple_finance_assisstant_trace = { + "version": "2.0", "data": {}, "trace": { "trace_id": "66a61777a1e481ab498bc7b5", @@ -137,3 +139,283 @@ ], }, } + + +simple_rag_trace_for_baseresponse_v3 = { + "version": "3.0", + "data": {}, + "tree": { + "nodes": [ + { + "lifecycle": { + "created_at": "2024-11-16T19:21:25.839557", + "updated_at": None, + "updated_by_id": None, + "updated_by": None, + }, + "root": {"id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57"}, + "tree": {"id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57", "type": None}, + "node": { + "id": "007a51b2-60cf-eb57-69c0-c57f7e38fec6", + "name": "rag", + "type": "workflow", + }, + "parent": None, + "time": { + "start": "2024-11-16T19:21:18.838642", + "end": "2024-11-16T19:21:25.735744", + "span": 6897102, + }, + "status": {"code": "UNSET", "message": None}, + "exception": None, + "data": { + "inputs": { + "topic": "psychology", + "genre": "action, suspense", + "count": 6, + }, + "outputs": "Sure! Here is a list of 6 movies about psychology in the genre of action and suspense:\n\n1. Blood River (2009): A psychological thriller exploring the destruction of a young couple's seemingly perfect marriage.\n\n2. Alex Cross (2012): A homicide detective pushed to his moral and physical limits as he tangles with a skilled serial killer who specializes in torture and pain.\n\n3. Repeaters (2010): A gritty mind-bending thriller about three twenty-somethings trapped in an impossible time labyrinth.\n\n4. Blue Steel (1989): A female rookie in the police force engages in a cat and mouse game with a pistol-wielding psychopath obsessed with her.\n\n5. Donovan's Echo (2011): A man re-examines his tragic past, memory, and future after experiencing a series of uncanny déjà vu events.\n\n6. House of Last Things (2013): A mind-bending thriller set in Portland, Oregon, exploring the effects of an unspoken tragedy on a house and its inhabitants.", + }, + "metrics": { + "some-scope": {"count": 6}, + "acc": { + "costs": {"total": 0.0028775}, + "tokens": {"prompt": 920, "completion": 207, "total": 1127}, + }, + }, + "meta": { + "configuration": { + "retriever_prompt": "Movies about {topic} in the genre of {genre}.", + "retriever_multiplier": 3, + "generator_context_prompt": "Given the following list of suggested movies:\n\n{movies}", + "generator_instructions_prompt": "Provide a list of {count} movies about {topic} in the genre of {genre}.", + "generator_model": "gpt-3.5-turbo", + "generator_temperature": 0.8, + "summarizer_context_prompt": "Act as a professional cinema critic.\nBe concise and factual.\nUse one intro sentence, and one sentence per movie.", + "summarizer_instructions_prompt": "Summarize the following recommendations about {topic} in the genre of {genre}:\n\n{report}", + "summarizer_model": "gpt-4o-mini", + "summarizer_temperature": 0.2, + }, + "some-scope": {"topic": "psychology", "genre": "action, suspense"}, + }, + "refs": { + "application": {"id": "01933668-782a-7bde-9d9e-5808ec9a9901"}, + "variant": {"id": "40ce591a246e44bf82809dd9c748f02e"}, + }, + "links": None, + "otel": { + "kind": "SPAN_KIND_SERVER", + "attributes": None, + "events": [], + "links": None, + }, + "nodes": { + "retriever": { + "lifecycle": { + "created_at": "2024-11-16T19:21:25.839161", + "updated_at": None, + "updated_by_id": None, + "updated_by": None, + }, + "root": {"id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57"}, + "tree": { + "id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57", + "type": None, + }, + "node": { + "id": "007a51b2-60cf-eb57-1cf9-6860802d9008", + "name": "retriever", + "type": "task", + }, + "parent": {"id": "007a51b2-60cf-eb57-69c0-c57f7e38fec6"}, + "time": { + "start": "2024-11-16T19:21:18.839026", + "end": "2024-11-16T19:21:22.468042", + "span": 3629016, + }, + "status": {"code": "UNSET", "message": None}, + "exception": None, + "data": { + "internals": { + "prompt": "Movies about psychology in the genre of action, suspense." + }, + "outputs": { + "movies": [ + "Blood River (2009) in ['Horror', 'Thriller', 'Western']: A psychological thriller, which explores the destruction of a young couple's seemingly perfect marriage.", + "Alex Cross (2012) in ['Action', 'Crime', 'Mystery']: A homicide detective is pushed to the brink of his moral and physical limits as he tangles with a ferociously skilled serial killer who specializes in torture and pain.", + "Silver Medallist (2009) in ['Action', 'Adventure', 'Comedy']: An action-adventure story focused on the lives of express deliverymen, traffic cops and lonely beauties.", + "Kill Me Again (1989) in ['Action', 'Crime', 'Drama']: A young detective becomes involved with a beautiful woman on the run from the mob and her psychopath boyfriend.", + "Repeaters (2010) in ['Action', 'Crime', 'Drama']: A gritty mind-bending thriller about three twenty-somethings who find themselves in an impossible time labyrinth, where each day they awaken to the same terrifying day as the preceding one.", + "The Tesseract (2003) in ['Action', 'Crime', 'Drama']: A psychologist, an Englishman, a bellboy and a wounded female assasin have their fates crossed at a sleazy Bangkok hotel.", + "Abduction (2011) in ['Action', 'Mystery', 'Thriller']: A thriller centered on a young man who sets out to uncover the truth about his life after finding his baby photo on a missing persons website.", + "Blue Steel (1989) in ['Action', 'Crime', 'Drama']: A female rookie in the police force engages in a cat and mouse game with a pistol wielding psychopath who becomes obsessed with her.", + "Half Past Dead (2002) in ['Action', 'Crime', 'Thriller']: This movie tells the story of a man who goes undercover in a hi-tech prison to find out information to help prosecute those who killed his wife. While there he stumbles onto a plot involving a death-row inmate and his $200 million stash of gold.", + "Painted Skin (2008) in ['Action', 'Drama', 'Thriller']: An action-thriller centered on a vampire-like woman who eats the skins and hearts of her lovers.", + "Donovan's Echo (2011) in ['Drama', 'Fantasy', 'Mystery']: A series of uncanny dèjè vu events force a man to re-examine his tragic past, memory, instinct, and future.", + "Once a Thief (1991) in ['Action', 'Comedy', 'Crime']: A romantic and action packed story of three best friends, a group of high end art thieves, who come into trouble when a love-triangle forms between them.", + "Arrambam (2013) in ['Action', 'Drama', 'Mystery']: A mysterious man along with a young computer hacker tries to unveil a major government conspiracy which resulted in several bloody deaths.", + "House of Last Things (2013) in ['Fantasy', 'Thriller']: A mind-bending thriller set in Portland, Oregon about an unspoken tragedy and its effects on a house, its temporary caretakers and the owners, a classical music critic and his wife on a recuperative trip to Italy.", + "Traffickers (2012) in ['Action', 'Crime', 'Drama']: A thriller about the passengers with different objectives on board a cruiser headed for China, being chased over and over again and unexpected happening of things.", + "Shattered Image (1998) in ['Crime', 'Drama', 'Fantasy']: Confusing realities surface in this paranoid film dealing with the fragile nature of a young woman (Anne Parillaud) recovering from rape and an apparent attempted suicide. In one reality, ...", + "The 6th Day (2000) in ['Action', 'Mystery', 'Sci-Fi']: Futuristic action about a man who meets a clone of himself and stumbles into a grand conspiracy about clones taking over the world.", + "Angel of Death (2009) in ['Action', 'Crime', 'Thriller']: A career assassin becomes haunted by one of her victims following a near fatal injury to her brain. Becoming a rogue assassin settling the score with her former mob employers, chaos and power struggles ensue.", + ] + }, + }, + "metrics": { + "acc": { + "costs": {"total": 0.0011}, + "tokens": {"prompt": 11, "total": 11}, + } + }, + "meta": None, + "refs": { + "application": { + "id": "01933668-782a-7bde-9d9e-5808ec9a9901" + } + }, + "links": None, + "otel": { + "kind": "SPAN_KIND_INTERNAL", + "attributes": None, + "events": [], + "links": None, + }, + "nodes": None, + }, + "reporter": { + "lifecycle": { + "created_at": "2024-11-16T19:21:25.839450", + "updated_at": None, + "updated_by_id": None, + "updated_by": None, + }, + "root": {"id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57"}, + "tree": { + "id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57", + "type": None, + }, + "node": { + "id": "007a51b2-60cf-eb57-a6ea-b15a330bbf84", + "name": "reporter", + "type": "task", + }, + "parent": {"id": "007a51b2-60cf-eb57-69c0-c57f7e38fec6"}, + "time": { + "start": "2024-11-16T19:21:22.468268", + "end": "2024-11-16T19:21:25.735240", + "span": 3266972, + }, + "status": {"code": "UNSET", "message": None}, + "exception": None, + "data": { + "outputs": { + "report": "Sure! Here is a list of 6 movies about psychology in the genre of action and suspense:\n\n1. Blood River (2009): A psychological thriller exploring the destruction of a young couple's seemingly perfect marriage.\n\n2. Alex Cross (2012): A homicide detective pushed to his moral and physical limits as he tangles with a skilled serial killer who specializes in torture and pain.\n\n3. Repeaters (2010): A gritty mind-bending thriller about three twenty-somethings trapped in an impossible time labyrinth.\n\n4. Blue Steel (1989): A female rookie in the police force engages in a cat and mouse game with a pistol-wielding psychopath obsessed with her.\n\n5. Donovan's Echo (2011): A man re-examines his tragic past, memory, and future after experiencing a series of uncanny déjà vu events.\n\n6. House of Last Things (2013): A mind-bending thriller set in Portland, Oregon, exploring the effects of an unspoken tragedy on a house and its inhabitants." + } + }, + "metrics": { + "acc": { + "costs": {"total": 0.0017775}, + "tokens": { + "prompt": 909, + "completion": 207, + "total": 1116, + }, + } + }, + "meta": None, + "refs": { + "application": { + "id": "01933668-782a-7bde-9d9e-5808ec9a9901" + } + }, + "links": None, + "otel": { + "kind": "SPAN_KIND_INTERNAL", + "attributes": None, + "events": [], + "links": None, + }, + "nodes": { + "chat": { + "lifecycle": { + "created_at": "2024-11-16T19:21:25.839295", + "updated_at": None, + "updated_by_id": None, + "updated_by": None, + }, + "root": {"id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57"}, + "tree": { + "id": "7f8d1a05-f4ac-4145-007a-51b260cfeb57", + "type": None, + }, + "node": { + "id": "007a51b2-60cf-eb57-5cbb-1351f06949c7", + "name": "chat", + "type": "completion", + }, + "parent": { + "id": "007a51b2-60cf-eb57-a6ea-b15a330bbf84" + }, + "time": { + "start": "2024-11-16T19:21:22.468632", + "end": "2024-11-16T19:21:25.735134", + "span": 3266502, + }, + "status": {"code": "UNSET", "message": None}, + "exception": None, + "data": { + "inputs": { + "prompts": { + "system": "Given the following list of suggested movies:\n\nBlood River (2009) in ['Horror', 'Thriller', 'Western']: A psychological thriller, which explores the destruction of a young couple's seemingly perfect marriage.\nAlex Cross (2012) in ['Action', 'Crime', 'Mystery']: A homicide detective is pushed to the brink of his moral and physical limits as he tangles with a ferociously skilled serial killer who specializes in torture and pain.\nSilver Medallist (2009) in ['Action', 'Adventure', 'Comedy']: An action-adventure story focused on the lives of express deliverymen, traffic cops and lonely beauties.\nKill Me Again (1989) in ['Action', 'Crime', 'Drama']: A young detective becomes involved with a beautiful woman on the run from the mob and her psychopath boyfriend.\nRepeaters (2010) in ['Action', 'Crime', 'Drama']: A gritty mind-bending thriller about three twenty-somethings who find themselves in an impossible time labyrinth, where each day they awaken to the same terrifying day as the preceding one.\nThe Tesseract (2003) in ['Action', 'Crime', 'Drama']: A psychologist, an Englishman, a bellboy and a wounded female assasin have their fates crossed at a sleazy Bangkok hotel.\nAbduction (2011) in ['Action', 'Mystery', 'Thriller']: A thriller centered on a young man who sets out to uncover the truth about his life after finding his baby photo on a missing persons website.\nBlue Steel (1989) in ['Action', 'Crime', 'Drama']: A female rookie in the police force engages in a cat and mouse game with a pistol wielding psychopath who becomes obsessed with her.\nHalf Past Dead (2002) in ['Action', 'Crime', 'Thriller']: This movie tells the story of a man who goes undercover in a hi-tech prison to find out information to help prosecute those who killed his wife. While there he stumbles onto a plot involving a death-row inmate and his $200 million stash of gold.\nPainted Skin (2008) in ['Action', 'Drama', 'Thriller']: An action-thriller centered on a vampire-like woman who eats the skins and hearts of her lovers.\nDonovan's Echo (2011) in ['Drama', 'Fantasy', 'Mystery']: A series of uncanny dèjè vu events force a man to re-examine his tragic past, memory, instinct, and future.\nOnce a Thief (1991) in ['Action', 'Comedy', 'Crime']: A romantic and action packed story of three best friends, a group of high end art thieves, who come into trouble when a love-triangle forms between them.\nArrambam (2013) in ['Action', 'Drama', 'Mystery']: A mysterious man along with a young computer hacker tries to unveil a major government conspiracy which resulted in several bloody deaths.\nHouse of Last Things (2013) in ['Fantasy', 'Thriller']: A mind-bending thriller set in Portland, Oregon about an unspoken tragedy and its effects on a house, its temporary caretakers and the owners, a classical music critic and his wife on a recuperative trip to Italy.\nTraffickers (2012) in ['Action', 'Crime', 'Drama']: A thriller about the passengers with different objectives on board a cruiser headed for China, being chased over and over again and unexpected happening of things.\nShattered Image (1998) in ['Crime', 'Drama', 'Fantasy']: Confusing realities surface in this paranoid film dealing with the fragile nature of a young woman (Anne Parillaud) recovering from rape and an apparent attempted suicide. In one reality, ...\nThe 6th Day (2000) in ['Action', 'Mystery', 'Sci-Fi']: Futuristic action about a man who meets a clone of himself and stumbles into a grand conspiracy about clones taking over the world.\nAngel of Death (2009) in ['Action', 'Crime', 'Thriller']: A career assassin becomes haunted by one of her victims following a near fatal injury to her brain. Becoming a rogue assassin settling the score with her former mob employers, chaos and power struggles ensue.", + "user": "Provide a list of 6 movies about psychology in the genre of action, suspense.", + }, + "opts": { + "model": "gpt-3.5-turbo", + "temperature": 0.8, + }, + }, + "outputs": "Sure! Here is a list of 6 movies about psychology in the genre of action and suspense:\n\n1. Blood River (2009): A psychological thriller exploring the destruction of a young couple's seemingly perfect marriage.\n\n2. Alex Cross (2012): A homicide detective pushed to his moral and physical limits as he tangles with a skilled serial killer who specializes in torture and pain.\n\n3. Repeaters (2010): A gritty mind-bending thriller about three twenty-somethings trapped in an impossible time labyrinth.\n\n4. Blue Steel (1989): A female rookie in the police force engages in a cat and mouse game with a pistol-wielding psychopath obsessed with her.\n\n5. Donovan's Echo (2011): A man re-examines his tragic past, memory, and future after experiencing a series of uncanny déjà vu events.\n\n6. House of Last Things (2013): A mind-bending thriller set in Portland, Oregon, exploring the effects of an unspoken tragedy on a house and its inhabitants.", + }, + "metrics": { + "unit": { + "costs": {"total": 0.0017775}, + "tokens": { + "prompt": 909, + "completion": 207, + "total": 1116, + }, + }, + "acc": { + "costs": {"total": 0.0017775}, + "tokens": { + "prompt": 909, + "completion": 207, + "total": 1116, + }, + }, + }, + "meta": None, + "refs": { + "application": { + "id": "01933668-782a-7bde-9d9e-5808ec9a9901" + } + }, + "links": None, + "otel": { + "kind": "SPAN_KIND_CLIENT", + "attributes": None, + "events": [], + "links": None, + }, + "nodes": None, + } + }, + }, + }, + } + ], + "version": "1.0.0", + "count": None, + }, +} diff --git a/agenta-backend/agenta_backend/utils/traces.py b/agenta-backend/agenta_backend/utils/traces.py index a5433d1d5e..ea72929a31 100644 --- a/agenta-backend/agenta_backend/utils/traces.py +++ b/agenta-backend/agenta_backend/utils/traces.py @@ -1,9 +1,9 @@ import logging import traceback -from collections import OrderedDict from copy import deepcopy - from typing import Any, Dict +from collections import OrderedDict + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -30,10 +30,10 @@ def _make_spans_id_tree(trace): index = {} def push(span) -> None: - if span["parent_span_id"] is None: + if span.get("parent_span_id") is None: tree[span["id"]] = OrderedDict() index[span["id"]] = tree[span["id"]] - elif span["parent_span_id"] in index: + elif span.get("parent_span_id") in index: index[span["parent_span_id"]][span["id"]] = OrderedDict() index[span["id"]] = index[span["parent_span_id"]][span["id"]] else: @@ -45,6 +45,126 @@ def push(span) -> None: return tree +def _make_nested_nodes_tree(tree: dict): + """ + Creates a nested tree structure from a flat list of nodes. + + Args: + tree: {tree: {nodes: List[Node]}} + + Returns: + tree: {[node_id]: tree(node_id)} # recursive + e.g. {node_id_0: {node_id_0_0: {}, node_id_0_1: {}}} + for: + node_id_0 + node_id_0_0 + node_id_0_1 + """ + + ordered_tree = OrderedDict() + + def add_node(node: dict, parent_tree: dict): + """ + Recursively adds a node and its children to the parent tree. + """ + + node_id = node["node"]["id"] + parent_tree[node_id] = OrderedDict() + + # If there are child nodes, recursively add them + if "nodes" in node and node["nodes"] is not None: + for child_key, child_node in node["nodes"].items(): + add_node(child_node, parent_tree[node_id]) + + # Process the top-level nodes + for node in tree["nodes"]: + add_node(node, ordered_tree) + + return ordered_tree + + +def _make_nodes_ids(ordered_dict: OrderedDict): + """ + Recursively converts an OrderedDict to a plain dict. + + Args: + ordered_dict (OrderedDict): The OrderedDict to convert. + + Returns: + dict: A plain dictionary representation of the OrderedDict. + """ + + if isinstance(ordered_dict, OrderedDict): + return {key: _make_nodes_ids(value) for key, value in ordered_dict.items()} + return ordered_dict + + +def _build_nodes_tree(nodes_id: dict, tree_nodes: list): + """ + Recursively builds a dictionary of node keys from a dictionary of nodes. + + Args: + nodes_id (dict): The dictionary representing the nodes. + tree_nodes (list): List[Node] + + Returns: + List[dict]: A list of dictionary of unique node keys with their corresponding details from trace_tree. + """ + + def gather_nodes(nodes: list): + result = {} + stack = nodes[:] + while stack: + current = stack.pop() + node_id = current["node"]["id"] + result[node_id] = current + if "nodes" in current and current["nodes"] is not None: + stack.extend(current["nodes"].values()) + return result + + def extract_node_details(node_id: str, nodes: dict): + """ + Helper function to extract relevant details for a node. + """ + + node_data = nodes.get(node_id, {}) + return { + "lifecycle": node_data.get("lifecycle", {}), + "root": node_data.get("root", {}), + "tree": node_data.get("tree", {}), + "node": node_data.get("node", {}), + "parent": node_data.get("parent", None), + "time": node_data.get("time", {}), + "status": node_data.get("status"), + "exception": node_data.get("exception"), + "data": node_data.get("data"), + "metrics": node_data.get("metrics"), + "meta": node_data.get("meta"), + "refs": node_data.get("refs"), + "links": node_data.get("links"), + "otel": node_data.get("otel"), + } + + def recursive_flatten(current_nodes_id: dict, result: dict, nodes: dict): + """ + Recursive function to flatten nodes into an ordered dictionary. + """ + + for node_id, child_nodes in current_nodes_id.items(): + # Add the current node details to the result + result[node_id] = extract_node_details(node_id, nodes) + + # Recursively process child nodes + if child_nodes: + recursive_flatten(child_nodes, result, nodes) + + # Initialize the ordered dictionary and start the recursion + ordered_result = dict() + nodes = gather_nodes(nodes=tree_nodes) + recursive_flatten(current_nodes_id=nodes_id, result=ordered_result, nodes=nodes) + return list(ordered_result.values()) + + INCLUDED_KEYS = ["start_time", "end_time", "inputs", "internals", "outputs"] TRACE_DEFAULT_KEY = "__default__" @@ -60,9 +180,11 @@ def _make_spans_tree(spans_id_tree, spans_index): Args: spans_id_tree: {[span_id]: spans_id_tree(span_id)} # recursive (ids only) index: {[span_id]: span} + Returns: spans_tree: {[span_id]: spans_tree(span_id)} # recursive (full span) """ + spans_tree = dict() count = dict() @@ -78,7 +200,7 @@ def _make_spans_tree(spans_id_tree, spans_index): key = spans_index[id]["name"] - span = {k: spans_index[id][k] for k in INCLUDED_KEYS} + span = {k: spans_index[id].get(k, None) for k in INCLUDED_KEYS} if TRACE_DEFAULT_KEY in span["outputs"]: span["outputs"] = span["outputs"][TRACE_DEFAULT_KEY] @@ -97,7 +219,7 @@ def _make_spans_tree(spans_id_tree, spans_index): return spans_tree -def process_distributed_trace_into_trace_tree(trace): +def process_distributed_trace_into_trace_tree(trace: Any, version: str): """ Creates trace tree from flat trace @@ -108,12 +230,24 @@ def process_distributed_trace_into_trace_tree(trace): trace: {trace_id: str, spans: spans_tree} """ - spans_id_tree = _make_spans_id_tree(trace) - spans_index = {span["id"]: span for span in trace["spans"]} - trace = { - "trace_id": trace["trace_id"], - "spans": _make_spans_tree(deepcopy(spans_id_tree), spans_index), - } + if version == "3.0": + tree = trace # swap trace name to tree + trace_id = tree.get("nodes", [{}])[0].get("root", {}).get("id") + spans_id_tree = _make_nested_nodes_tree(tree=tree) + nodes_ids = _make_nodes_ids(ordered_dict=spans_id_tree) + spans = _build_nodes_tree(nodes_id=nodes_ids, tree_nodes=tree["nodes"]) + + elif version == "2.0": + trace_id = trace["trace_id"] + spans_id_tree = _make_spans_id_tree(trace) + spans_index = {span["id"]: span for span in trace["spans"]} + spans = _make_spans_tree(deepcopy(spans_id_tree), spans_index) + + else: + trace_id = None + spans = [] + + trace = {"trace_id": trace_id, "spans": spans} return trace @@ -139,7 +273,27 @@ def _parse_field_part(part): return key, idx -def get_field_value_from_trace_tree(tree: Dict[str, Any], field: str) -> Dict[str, Any]: +# --------------------------------------------------------------- # +# ------- HELPER FUNCTIONS TO GET FIELD VALUE FROM TRACE ------- # +# --------------------------------------------------------------- # + + +def get_field_value_from_trace_tree( + tree: Dict[str, Any], field: str, version: str +) -> Dict[str, Any]: + if version == "2.0": + return get_field_value_from_trace_tree_v2(tree=tree, field=field) + + elif version == "3.0": + return get_field_value_from_trace_tree_v3(trace_data=tree, key=field) + + return None + + +def get_field_value_from_trace_tree_v2( + tree: Dict[str, Any], + field: str, +): """ Retrieve the value of the key from the trace tree. @@ -153,6 +307,7 @@ def get_field_value_from_trace_tree(tree: Dict[str, Any], field: str) -> Dict[st Returns: Dict[str, Any]: The retrieved value or None if the key does not exist or an error occurs. """ + separate_by_spans_key = True parts = field.split(".") @@ -183,3 +338,80 @@ def get_field_value_from_trace_tree(tree: Dict[str, Any], field: str) -> Dict[st except Exception as e: logger.error(f"Error retrieving trace value from key: {traceback.format_exc()}") return None + + +def get_field_value_from_trace_tree_v3(trace_data: Dict[str, any], key: str): + """ + Retrieves a nested value from the trace data based on a hierarchical key. + + Args: + trace_data (dict): A dictionary container the trace_id and a list of node dictionaries in the trace data. + key (str): The hierarchical key (e.g., "rag.retriever.internals.prompt"). + + Returns: + The value associated with the specified key, or None if not found. + """ + + try: + # Parse the hierarchical key + key_parts = key.split(".") + + # Start with the root node name + current_name = key_parts.pop(0) + + # Find the root node + current_node = next( + ( + node + for node in trace_data["spans"] + if node["node"]["name"] == current_name + ), + None, + ) + if not current_node: + return None + + # Traverse the hierarchy + for part in key_parts: + if part in current_node: # If the part is a direct key in the current node + current_node = current_node[part] + elif ( + "data" in current_node and part in current_node["data"] + ): # Check inside "data" + current_node = current_node["data"][part] + elif ( + "metrics" in current_node and part in current_node["metrics"] + ): # Check inside "metrics" + current_node = current_node["metrics"][part] + elif ( + "meta" in current_node + and current_node["meta"] + and part in current_node["meta"] + ): # Check inside "meta" + current_node = current_node["meta"][part] + else: # Traverse to child node if it matches the "name" + child_node = next( + ( + node + for node in trace_data["spans"] + if node["node"]["name"] == part + and node["parent"] + and node["parent"]["id"] == current_node["node"]["id"] + ), + None, + ) + if not child_node: + return None + + current_node = child_node + + return current_node + + except Exception as e: + logger.error(f"Error retrieving trace value from key: {traceback.format_exc()}") + return None + + +# ---------------------------------------------------------------------- # +# ------- END OF HELPER FUNCTIONS TO GET FIELD VALUE FROM TRACE ------- # +# ---------------------------------------------------------------------- # diff --git a/agenta-cli/agenta/client/backend/__init__.py b/agenta-cli/agenta/client/backend/__init__.py index fa222989b6..8907b11d29 100644 --- a/agenta-cli/agenta/client/backend/__init__.py +++ b/agenta-cli/agenta/client/backend/__init__.py @@ -1,6 +1,13 @@ # This file was auto-generated by Fern from our API Definition. from .types import ( + AgentaNodeDto, + AgentaNodeDtoNodesValue, + AgentaNodesResponse, + AgentaRootDto, + AgentaRootsResponse, + AgentaTreeDto, + AgentaTreesResponse, AggregatedResult, AggregatedResultEvaluatorConfig, App, @@ -8,6 +15,7 @@ AppVariantRevision, BaseOutput, BodyImportTestset, + CollectStatusResponse, ConfigDb, ConfigDto, ConfigResponseModel, @@ -32,6 +40,7 @@ EvaluatorConfig, EvaluatorMappingOutputInterface, EvaluatorOutputInterface, + ExceptionDto, GetConfigResponse, HttpValidationError, HumanEvaluation, @@ -43,30 +52,50 @@ Image, InviteRequest, LifecycleDto, + LinkDto, ListApiKeysResponse, LlmRunRateLimit, LlmTokens, LmProvidersEnum, NewHumanEvaluation, NewTestset, + NodeDto, + NodeType, + OTelContextDto, + OTelEventDto, + OTelExtraDto, + OTelLinkDto, + OTelSpanDto, + OTelSpanKind, + OTelSpansResponse, + OTelStatusCode, Organization, OrganizationOutput, Outputs, + ParentDto, Permission, ReferenceDto, ReferenceRequestModel, Result, + RootDto, Score, SimpleEvaluationOutput, Span, SpanDetail, + SpanDto, + SpanDtoNodesValue, SpanStatusCode, SpanVariant, + StatusCode, + StatusDto, Template, TemplateImageInfo, TestSetOutputResponse, TestSetSimpleResponse, + TimeDto, TraceDetail, + TreeDto, + TreeType, UpdateAppOutput, Uri, ValidationError, @@ -90,16 +119,25 @@ evaluations, evaluators, observability, + observability_v_1, testsets, variants, ) from .client import AgentaApi, AsyncAgentaApi from .containers import ContainerTemplatesResponse +from .observability_v_1 import Format, QueryTracesResponse from .variants import AddVariantFromBaseAndConfigResponse __all__ = [ "AddVariantFromBaseAndConfigResponse", "AgentaApi", + "AgentaNodeDto", + "AgentaNodeDtoNodesValue", + "AgentaNodesResponse", + "AgentaRootDto", + "AgentaRootsResponse", + "AgentaTreeDto", + "AgentaTreesResponse", "AggregatedResult", "AggregatedResultEvaluatorConfig", "App", @@ -108,6 +146,7 @@ "AsyncAgentaApi", "BaseOutput", "BodyImportTestset", + "CollectStatusResponse", "ConfigDb", "ConfigDto", "ConfigResponseModel", @@ -133,6 +172,8 @@ "EvaluatorConfig", "EvaluatorMappingOutputInterface", "EvaluatorOutputInterface", + "ExceptionDto", + "Format", "GetConfigResponse", "HttpValidationError", "HumanEvaluation", @@ -144,30 +185,51 @@ "Image", "InviteRequest", "LifecycleDto", + "LinkDto", "ListApiKeysResponse", "LlmRunRateLimit", "LlmTokens", "LmProvidersEnum", "NewHumanEvaluation", "NewTestset", + "NodeDto", + "NodeType", + "OTelContextDto", + "OTelEventDto", + "OTelExtraDto", + "OTelLinkDto", + "OTelSpanDto", + "OTelSpanKind", + "OTelSpansResponse", + "OTelStatusCode", "Organization", "OrganizationOutput", "Outputs", + "ParentDto", "Permission", + "QueryTracesResponse", "ReferenceDto", "ReferenceRequestModel", "Result", + "RootDto", "Score", "SimpleEvaluationOutput", "Span", "SpanDetail", + "SpanDto", + "SpanDtoNodesValue", "SpanStatusCode", "SpanVariant", + "StatusCode", + "StatusDto", "Template", "TemplateImageInfo", "TestSetOutputResponse", "TestSetSimpleResponse", + "TimeDto", "TraceDetail", + "TreeDto", + "TreeType", "UnprocessableEntityError", "UpdateAppOutput", "Uri", @@ -189,6 +251,7 @@ "evaluations", "evaluators", "observability", + "observability_v_1", "testsets", "variants", ] diff --git a/agenta-cli/agenta/client/backend/client.py b/agenta-cli/agenta/client/backend/client.py index be6e35e3b1..9fc9784d8a 100644 --- a/agenta-cli/agenta/client/backend/client.py +++ b/agenta-cli/agenta/client/backend/client.py @@ -13,6 +13,7 @@ from .environments.client import EnvironmentsClient from .bases.client import BasesClient from .configs.client import ConfigsClient +from .observability_v_1.client import ObservabilityV1Client from .core.request_options import RequestOptions from .types.list_api_keys_response import ListApiKeysResponse from .core.pydantic_utilities import parse_obj_as @@ -40,6 +41,7 @@ from .environments.client import AsyncEnvironmentsClient from .bases.client import AsyncBasesClient from .configs.client import AsyncConfigsClient +from .observability_v_1.client import AsyncObservabilityV1Client # this is used as the default value for optional parameters OMIT = typing.cast(typing.Any, ...) @@ -89,17 +91,13 @@ def __init__( self._client_wrapper = SyncClientWrapper( base_url=base_url, api_key=api_key, - httpx_client=( - httpx_client - if httpx_client is not None - else ( - httpx.Client( - timeout=_defaulted_timeout, follow_redirects=follow_redirects - ) - if follow_redirects is not None - else httpx.Client(timeout=_defaulted_timeout) - ) - ), + httpx_client=httpx_client + if httpx_client is not None + else httpx.Client( + timeout=_defaulted_timeout, follow_redirects=follow_redirects + ) + if follow_redirects is not None + else httpx.Client(timeout=_defaulted_timeout), timeout=_defaulted_timeout, ) self.observability = ObservabilityClient(client_wrapper=self._client_wrapper) @@ -112,6 +110,9 @@ def __init__( self.environments = EnvironmentsClient(client_wrapper=self._client_wrapper) self.bases = BasesClient(client_wrapper=self._client_wrapper) self.configs = ConfigsClient(client_wrapper=self._client_wrapper) + self.observability_v_1 = ObservabilityV1Client( + client_wrapper=self._client_wrapper + ) def list_api_keys( self, *, request_options: typing.Optional[RequestOptions] = None @@ -1619,17 +1620,13 @@ def __init__( self._client_wrapper = AsyncClientWrapper( base_url=base_url, api_key=api_key, - httpx_client=( - httpx_client - if httpx_client is not None - else ( - httpx.AsyncClient( - timeout=_defaulted_timeout, follow_redirects=follow_redirects - ) - if follow_redirects is not None - else httpx.AsyncClient(timeout=_defaulted_timeout) - ) - ), + httpx_client=httpx_client + if httpx_client is not None + else httpx.AsyncClient( + timeout=_defaulted_timeout, follow_redirects=follow_redirects + ) + if follow_redirects is not None + else httpx.AsyncClient(timeout=_defaulted_timeout), timeout=_defaulted_timeout, ) self.observability = AsyncObservabilityClient( @@ -1644,6 +1641,9 @@ def __init__( self.environments = AsyncEnvironmentsClient(client_wrapper=self._client_wrapper) self.bases = AsyncBasesClient(client_wrapper=self._client_wrapper) self.configs = AsyncConfigsClient(client_wrapper=self._client_wrapper) + self.observability_v_1 = AsyncObservabilityV1Client( + client_wrapper=self._client_wrapper + ) async def list_api_keys( self, *, request_options: typing.Optional[RequestOptions] = None diff --git a/agenta-cli/agenta/client/backend/core/http_client.py b/agenta-cli/agenta/client/backend/core/http_client.py index d4c43911a8..c9f1d7a59b 100644 --- a/agenta-cli/agenta/client/backend/core/http_client.py +++ b/agenta-cli/agenta/client/backend/core/http_client.py @@ -148,9 +148,9 @@ def get_request_body( json_body = maybe_filter_request_body(json, request_options, omit) # If you have an empty JSON body, you should just send None - return (json_body if json_body != {} else None), ( - data_body if data_body != {} else None - ) + return ( + json_body if json_body != {} else None + ), data_body if data_body != {} else None class HttpClient: @@ -250,7 +250,9 @@ def request( data=data_body, content=content, files=( - convert_file_dict_to_httpx_tuples(remove_none_from_dict(files)) + convert_file_dict_to_httpx_tuples( + remove_omit_from_dict(remove_none_from_dict(files), omit) + ) if (files is not None and files is not omit) else None ), @@ -351,7 +353,9 @@ def stream( data=data_body, content=content, files=( - convert_file_dict_to_httpx_tuples(remove_none_from_dict(files)) + convert_file_dict_to_httpx_tuples( + remove_omit_from_dict(remove_none_from_dict(files), omit) + ) if (files is not None and files is not omit) else None ), @@ -458,7 +462,9 @@ async def request( data=data_body, content=content, files=( - convert_file_dict_to_httpx_tuples(remove_none_from_dict(files)) + convert_file_dict_to_httpx_tuples( + remove_omit_from_dict(remove_none_from_dict(files), omit) + ) if files is not None else None ), @@ -558,7 +564,9 @@ async def stream( data=data_body, content=content, files=( - convert_file_dict_to_httpx_tuples(remove_none_from_dict(files)) + convert_file_dict_to_httpx_tuples( + remove_omit_from_dict(remove_none_from_dict(files), omit) + ) if files is not None else None ), diff --git a/agenta-cli/agenta/client/backend/observability/client.py b/agenta-cli/agenta/client/backend/observability/client.py index daf18288d6..aebe134924 100644 --- a/agenta-cli/agenta/client/backend/observability/client.py +++ b/agenta-cli/agenta/client/backend/observability/client.py @@ -280,7 +280,7 @@ def get_traces( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) - def delete_traces( + def delete_traces_legacy( self, *, request: typing.Sequence[str], @@ -307,7 +307,7 @@ def delete_traces( api_key="YOUR_API_KEY", base_url="https://yourhost.com/path/to/api", ) - client.observability.delete_traces( + client.observability.delete_traces_legacy( request=["string"], ) """ @@ -901,7 +901,7 @@ async def main() -> None: raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) - async def delete_traces( + async def delete_traces_legacy( self, *, request: typing.Sequence[str], @@ -933,7 +933,7 @@ async def delete_traces( async def main() -> None: - await client.observability.delete_traces( + await client.observability.delete_traces_legacy( request=["string"], ) diff --git a/agenta-cli/agenta/client/backend/observability_v_1/__init__.py b/agenta-cli/agenta/client/backend/observability_v_1/__init__.py new file mode 100644 index 0000000000..aceeca0c75 --- /dev/null +++ b/agenta-cli/agenta/client/backend/observability_v_1/__init__.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +from .types import Format, QueryTracesResponse + +__all__ = ["Format", "QueryTracesResponse"] diff --git a/agenta-cli/agenta/client/backend/observability_v_1/client.py b/agenta-cli/agenta/client/backend/observability_v_1/client.py new file mode 100644 index 0000000000..5fa38f8c3c --- /dev/null +++ b/agenta-cli/agenta/client/backend/observability_v_1/client.py @@ -0,0 +1,560 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.client_wrapper import SyncClientWrapper +import typing +from ..core.request_options import RequestOptions +from ..types.collect_status_response import CollectStatusResponse +from ..core.pydantic_utilities import parse_obj_as +from json.decoder import JSONDecodeError +from ..core.api_error import ApiError +from .types.format import Format +from .types.query_traces_response import QueryTracesResponse +from ..errors.unprocessable_entity_error import UnprocessableEntityError +from ..types.http_validation_error import HttpValidationError +from ..core.client_wrapper import AsyncClientWrapper + + +class ObservabilityV1Client: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def otlp_status( + self, *, request_options: typing.Optional[RequestOptions] = None + ) -> CollectStatusResponse: + """ + Status of OTLP endpoint. + + Parameters + ---------- + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + CollectStatusResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + client.observability_v_1.otlp_status() + """ + _response = self._client_wrapper.httpx_client.request( + "observability/v1/otlp/traces", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + CollectStatusResponse, + parse_obj_as( + type_=CollectStatusResponse, # type: ignore + object_=_response.json(), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + def otlp_receiver( + self, *, request_options: typing.Optional[RequestOptions] = None + ) -> CollectStatusResponse: + """ + Receive traces via OTLP. + + Parameters + ---------- + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + CollectStatusResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + client.observability_v_1.otlp_receiver() + """ + _response = self._client_wrapper.httpx_client.request( + "observability/v1/otlp/traces", + method="POST", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + CollectStatusResponse, + parse_obj_as( + type_=CollectStatusResponse, # type: ignore + object_=_response.json(), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + def query_traces( + self, + *, + format: typing.Optional[Format] = None, + focus: typing.Optional[str] = None, + oldest: typing.Optional[str] = None, + newest: typing.Optional[str] = None, + filtering: typing.Optional[str] = None, + page: typing.Optional[int] = None, + size: typing.Optional[int] = None, + next: typing.Optional[str] = None, + stop: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> QueryTracesResponse: + """ + Query traces, with optional grouping, windowing, filtering, and pagination. + + Parameters + ---------- + format : typing.Optional[Format] + + focus : typing.Optional[str] + + oldest : typing.Optional[str] + + newest : typing.Optional[str] + + filtering : typing.Optional[str] + + page : typing.Optional[int] + + size : typing.Optional[int] + + next : typing.Optional[str] + + stop : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + QueryTracesResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + client.observability_v_1.query_traces() + """ + _response = self._client_wrapper.httpx_client.request( + "observability/v1/traces", + method="GET", + params={ + "format": format, + "focus": focus, + "oldest": oldest, + "newest": newest, + "filtering": filtering, + "page": page, + "size": size, + "next": next, + "stop": stop, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + QueryTracesResponse, + parse_obj_as( + type_=QueryTracesResponse, # type: ignore + object_=_response.json(), + ), + ) + if _response.status_code == 422: + raise UnprocessableEntityError( + typing.cast( + HttpValidationError, + parse_obj_as( + type_=HttpValidationError, # type: ignore + object_=_response.json(), + ), + ) + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + def delete_traces( + self, + *, + node_id: typing.Optional[str] = None, + node_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> CollectStatusResponse: + """ + Delete trace. + + Parameters + ---------- + node_id : typing.Optional[str] + + node_ids : typing.Optional[typing.Union[str, typing.Sequence[str]]] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + CollectStatusResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + client.observability_v_1.delete_traces() + """ + _response = self._client_wrapper.httpx_client.request( + "observability/v1/traces", + method="DELETE", + params={ + "node_id": node_id, + "node_ids": node_ids, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + CollectStatusResponse, + parse_obj_as( + type_=CollectStatusResponse, # type: ignore + object_=_response.json(), + ), + ) + if _response.status_code == 422: + raise UnprocessableEntityError( + typing.cast( + HttpValidationError, + parse_obj_as( + type_=HttpValidationError, # type: ignore + object_=_response.json(), + ), + ) + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + +class AsyncObservabilityV1Client: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def otlp_status( + self, *, request_options: typing.Optional[RequestOptions] = None + ) -> CollectStatusResponse: + """ + Status of OTLP endpoint. + + Parameters + ---------- + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + CollectStatusResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.observability_v_1.otlp_status() + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "observability/v1/otlp/traces", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + CollectStatusResponse, + parse_obj_as( + type_=CollectStatusResponse, # type: ignore + object_=_response.json(), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + async def otlp_receiver( + self, *, request_options: typing.Optional[RequestOptions] = None + ) -> CollectStatusResponse: + """ + Receive traces via OTLP. + + Parameters + ---------- + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + CollectStatusResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.observability_v_1.otlp_receiver() + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "observability/v1/otlp/traces", + method="POST", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + CollectStatusResponse, + parse_obj_as( + type_=CollectStatusResponse, # type: ignore + object_=_response.json(), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + async def query_traces( + self, + *, + format: typing.Optional[Format] = None, + focus: typing.Optional[str] = None, + oldest: typing.Optional[str] = None, + newest: typing.Optional[str] = None, + filtering: typing.Optional[str] = None, + page: typing.Optional[int] = None, + size: typing.Optional[int] = None, + next: typing.Optional[str] = None, + stop: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> QueryTracesResponse: + """ + Query traces, with optional grouping, windowing, filtering, and pagination. + + Parameters + ---------- + format : typing.Optional[Format] + + focus : typing.Optional[str] + + oldest : typing.Optional[str] + + newest : typing.Optional[str] + + filtering : typing.Optional[str] + + page : typing.Optional[int] + + size : typing.Optional[int] + + next : typing.Optional[str] + + stop : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + QueryTracesResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.observability_v_1.query_traces() + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "observability/v1/traces", + method="GET", + params={ + "format": format, + "focus": focus, + "oldest": oldest, + "newest": newest, + "filtering": filtering, + "page": page, + "size": size, + "next": next, + "stop": stop, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + QueryTracesResponse, + parse_obj_as( + type_=QueryTracesResponse, # type: ignore + object_=_response.json(), + ), + ) + if _response.status_code == 422: + raise UnprocessableEntityError( + typing.cast( + HttpValidationError, + parse_obj_as( + type_=HttpValidationError, # type: ignore + object_=_response.json(), + ), + ) + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + async def delete_traces( + self, + *, + node_id: typing.Optional[str] = None, + node_ids: typing.Optional[typing.Union[str, typing.Sequence[str]]] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> CollectStatusResponse: + """ + Delete trace. + + Parameters + ---------- + node_id : typing.Optional[str] + + node_ids : typing.Optional[typing.Union[str, typing.Sequence[str]]] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + CollectStatusResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.observability_v_1.delete_traces() + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "observability/v1/traces", + method="DELETE", + params={ + "node_id": node_id, + "node_ids": node_ids, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return typing.cast( + CollectStatusResponse, + parse_obj_as( + type_=CollectStatusResponse, # type: ignore + object_=_response.json(), + ), + ) + if _response.status_code == 422: + raise UnprocessableEntityError( + typing.cast( + HttpValidationError, + parse_obj_as( + type_=HttpValidationError, # type: ignore + object_=_response.json(), + ), + ) + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) diff --git a/agenta-cli/agenta/client/backend/observability_v_1/types/__init__.py b/agenta-cli/agenta/client/backend/observability_v_1/types/__init__.py new file mode 100644 index 0000000000..7303a90f08 --- /dev/null +++ b/agenta-cli/agenta/client/backend/observability_v_1/types/__init__.py @@ -0,0 +1,6 @@ +# This file was auto-generated by Fern from our API Definition. + +from .format import Format +from .query_traces_response import QueryTracesResponse + +__all__ = ["Format", "QueryTracesResponse"] diff --git a/agenta-cli/agenta/client/backend/observability_v_1/types/format.py b/agenta-cli/agenta/client/backend/observability_v_1/types/format.py new file mode 100644 index 0000000000..ed6f7db216 --- /dev/null +++ b/agenta-cli/agenta/client/backend/observability_v_1/types/format.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +Format = typing.Union[typing.Literal["opentelemetry", "agenta"], typing.Any] diff --git a/agenta-cli/agenta/client/backend/observability_v_1/types/query_traces_response.py b/agenta-cli/agenta/client/backend/observability_v_1/types/query_traces_response.py new file mode 100644 index 0000000000..4219a5b7e9 --- /dev/null +++ b/agenta-cli/agenta/client/backend/observability_v_1/types/query_traces_response.py @@ -0,0 +1,11 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from ...types.o_tel_spans_response import OTelSpansResponse +from ...types.agenta_nodes_response import AgentaNodesResponse +from ...types.agenta_trees_response import AgentaTreesResponse +from ...types.agenta_roots_response import AgentaRootsResponse + +QueryTracesResponse = typing.Union[ + OTelSpansResponse, AgentaNodesResponse, AgentaTreesResponse, AgentaRootsResponse +] diff --git a/agenta-cli/agenta/client/backend/types/__init__.py b/agenta-cli/agenta/client/backend/types/__init__.py index b20bd1e2ab..b10c09c61b 100644 --- a/agenta-cli/agenta/client/backend/types/__init__.py +++ b/agenta-cli/agenta/client/backend/types/__init__.py @@ -1,5 +1,12 @@ # This file was auto-generated by Fern from our API Definition. +from .agenta_node_dto import AgentaNodeDto +from .agenta_node_dto_nodes_value import AgentaNodeDtoNodesValue +from .agenta_nodes_response import AgentaNodesResponse +from .agenta_root_dto import AgentaRootDto +from .agenta_roots_response import AgentaRootsResponse +from .agenta_tree_dto import AgentaTreeDto +from .agenta_trees_response import AgentaTreesResponse from .aggregated_result import AggregatedResult from .aggregated_result_evaluator_config import AggregatedResultEvaluatorConfig from .app import App @@ -7,6 +14,7 @@ from .app_variant_revision import AppVariantRevision from .base_output import BaseOutput from .body_import_testset import BodyImportTestset +from .collect_status_response import CollectStatusResponse from .config_db import ConfigDb from .config_dto import ConfigDto from .config_response_model import ConfigResponseModel @@ -31,6 +39,7 @@ from .evaluator_config import EvaluatorConfig from .evaluator_mapping_output_interface import EvaluatorMappingOutputInterface from .evaluator_output_interface import EvaluatorOutputInterface +from .exception_dto import ExceptionDto from .get_config_response import GetConfigResponse from .http_validation_error import HttpValidationError from .human_evaluation import HumanEvaluation @@ -42,30 +51,50 @@ from .image import Image from .invite_request import InviteRequest from .lifecycle_dto import LifecycleDto +from .link_dto import LinkDto from .list_api_keys_response import ListApiKeysResponse from .llm_run_rate_limit import LlmRunRateLimit from .llm_tokens import LlmTokens from .lm_providers_enum import LmProvidersEnum from .new_human_evaluation import NewHumanEvaluation from .new_testset import NewTestset +from .node_dto import NodeDto +from .node_type import NodeType +from .o_tel_context_dto import OTelContextDto +from .o_tel_event_dto import OTelEventDto +from .o_tel_extra_dto import OTelExtraDto +from .o_tel_link_dto import OTelLinkDto +from .o_tel_span_dto import OTelSpanDto +from .o_tel_span_kind import OTelSpanKind +from .o_tel_spans_response import OTelSpansResponse +from .o_tel_status_code import OTelStatusCode from .organization import Organization from .organization_output import OrganizationOutput from .outputs import Outputs +from .parent_dto import ParentDto from .permission import Permission from .reference_dto import ReferenceDto from .reference_request_model import ReferenceRequestModel from .result import Result +from .root_dto import RootDto from .score import Score from .simple_evaluation_output import SimpleEvaluationOutput from .span import Span from .span_detail import SpanDetail +from .span_dto import SpanDto +from .span_dto_nodes_value import SpanDtoNodesValue from .span_status_code import SpanStatusCode from .span_variant import SpanVariant +from .status_code import StatusCode +from .status_dto import StatusDto from .template import Template from .template_image_info import TemplateImageInfo from .test_set_output_response import TestSetOutputResponse from .test_set_simple_response import TestSetSimpleResponse +from .time_dto import TimeDto from .trace_detail import TraceDetail +from .tree_dto import TreeDto +from .tree_type import TreeType from .update_app_output import UpdateAppOutput from .uri import Uri from .validation_error import ValidationError @@ -80,6 +109,13 @@ from .workspace_role_response import WorkspaceRoleResponse __all__ = [ + "AgentaNodeDto", + "AgentaNodeDtoNodesValue", + "AgentaNodesResponse", + "AgentaRootDto", + "AgentaRootsResponse", + "AgentaTreeDto", + "AgentaTreesResponse", "AggregatedResult", "AggregatedResultEvaluatorConfig", "App", @@ -87,6 +123,7 @@ "AppVariantRevision", "BaseOutput", "BodyImportTestset", + "CollectStatusResponse", "ConfigDb", "ConfigDto", "ConfigResponseModel", @@ -111,6 +148,7 @@ "EvaluatorConfig", "EvaluatorMappingOutputInterface", "EvaluatorOutputInterface", + "ExceptionDto", "GetConfigResponse", "HttpValidationError", "HumanEvaluation", @@ -122,30 +160,50 @@ "Image", "InviteRequest", "LifecycleDto", + "LinkDto", "ListApiKeysResponse", "LlmRunRateLimit", "LlmTokens", "LmProvidersEnum", "NewHumanEvaluation", "NewTestset", + "NodeDto", + "NodeType", + "OTelContextDto", + "OTelEventDto", + "OTelExtraDto", + "OTelLinkDto", + "OTelSpanDto", + "OTelSpanKind", + "OTelSpansResponse", + "OTelStatusCode", "Organization", "OrganizationOutput", "Outputs", + "ParentDto", "Permission", "ReferenceDto", "ReferenceRequestModel", "Result", + "RootDto", "Score", "SimpleEvaluationOutput", "Span", "SpanDetail", + "SpanDto", + "SpanDtoNodesValue", "SpanStatusCode", "SpanVariant", + "StatusCode", + "StatusDto", "Template", "TemplateImageInfo", "TestSetOutputResponse", "TestSetSimpleResponse", + "TimeDto", "TraceDetail", + "TreeDto", + "TreeType", "UpdateAppOutput", "Uri", "ValidationError", diff --git a/agenta-cli/agenta/client/backend/types/agenta_node_dto.py b/agenta-cli/agenta/client/backend/types/agenta_node_dto.py new file mode 100644 index 0000000000..8f8c933eac --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_node_dto.py @@ -0,0 +1,48 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from .lifecycle_dto import LifecycleDto +from .root_dto import RootDto +from .tree_dto import TreeDto +from .node_dto import NodeDto +from .parent_dto import ParentDto +from .time_dto import TimeDto +from .status_dto import StatusDto +from .exception_dto import ExceptionDto +from .link_dto import LinkDto +from .o_tel_extra_dto import OTelExtraDto +from .agenta_node_dto_nodes_value import AgentaNodeDtoNodesValue +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class AgentaNodeDto(UniversalBaseModel): + lifecycle: typing.Optional[LifecycleDto] = None + root: RootDto + tree: TreeDto + node: NodeDto + parent: typing.Optional[ParentDto] = None + time: TimeDto + status: StatusDto + exception: typing.Optional[ExceptionDto] = None + data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + metrics: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + meta: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + refs: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + links: typing.Optional[typing.List[LinkDto]] = None + otel: typing.Optional[OTelExtraDto] = None + nodes: typing.Optional[ + typing.Dict[str, typing.Optional[AgentaNodeDtoNodesValue]] + ] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/agenta_node_dto_nodes_value.py b/agenta-cli/agenta/client/backend/types/agenta_node_dto_nodes_value.py new file mode 100644 index 0000000000..771c4f8e9f --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_node_dto_nodes_value.py @@ -0,0 +1,6 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from .span_dto import SpanDto + +AgentaNodeDtoNodesValue = typing.Union[SpanDto, typing.List[SpanDto]] diff --git a/agenta-cli/agenta/client/backend/types/agenta_nodes_response.py b/agenta-cli/agenta/client/backend/types/agenta_nodes_response.py new file mode 100644 index 0000000000..37b8fea8f7 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_nodes_response.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +from ..core.pydantic_utilities import UniversalBaseModel +from .span_dto import SpanDto +import typing +from .agenta_node_dto import AgentaNodeDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic +from ..core.pydantic_utilities import update_forward_refs + + +class AgentaNodesResponse(UniversalBaseModel): + nodes: typing.List[AgentaNodeDto] + version: str + count: typing.Optional[int] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + +update_forward_refs(SpanDto, AgentaNodesResponse=AgentaNodesResponse) diff --git a/agenta-cli/agenta/client/backend/types/agenta_root_dto.py b/agenta-cli/agenta/client/backend/types/agenta_root_dto.py new file mode 100644 index 0000000000..04e57ee7e3 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_root_dto.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +from ..core.pydantic_utilities import UniversalBaseModel +from .span_dto import SpanDto +from .root_dto import RootDto +import typing +from .agenta_tree_dto import AgentaTreeDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic +from ..core.pydantic_utilities import update_forward_refs + + +class AgentaRootDto(UniversalBaseModel): + root: RootDto + trees: typing.List[AgentaTreeDto] + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + +update_forward_refs(SpanDto, AgentaRootDto=AgentaRootDto) diff --git a/agenta-cli/agenta/client/backend/types/agenta_roots_response.py b/agenta-cli/agenta/client/backend/types/agenta_roots_response.py new file mode 100644 index 0000000000..df3d3ba50b --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_roots_response.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +from ..core.pydantic_utilities import UniversalBaseModel +from .span_dto import SpanDto +import typing +from .agenta_root_dto import AgentaRootDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic +from ..core.pydantic_utilities import update_forward_refs + + +class AgentaRootsResponse(UniversalBaseModel): + roots: typing.List[AgentaRootDto] + version: str + count: typing.Optional[int] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + +update_forward_refs(SpanDto, AgentaRootsResponse=AgentaRootsResponse) diff --git a/agenta-cli/agenta/client/backend/types/agenta_tree_dto.py b/agenta-cli/agenta/client/backend/types/agenta_tree_dto.py new file mode 100644 index 0000000000..9e5ced12b5 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_tree_dto.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +from ..core.pydantic_utilities import UniversalBaseModel +from .span_dto import SpanDto +from .tree_dto import TreeDto +import typing +from .agenta_node_dto import AgentaNodeDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic +from ..core.pydantic_utilities import update_forward_refs + + +class AgentaTreeDto(UniversalBaseModel): + tree: TreeDto + nodes: typing.List[AgentaNodeDto] + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + +update_forward_refs(SpanDto, AgentaTreeDto=AgentaTreeDto) diff --git a/agenta-cli/agenta/client/backend/types/agenta_trees_response.py b/agenta-cli/agenta/client/backend/types/agenta_trees_response.py new file mode 100644 index 0000000000..f6e80b5b51 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/agenta_trees_response.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +from ..core.pydantic_utilities import UniversalBaseModel +from .span_dto import SpanDto +import typing +from .agenta_tree_dto import AgentaTreeDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic +from ..core.pydantic_utilities import update_forward_refs + + +class AgentaTreesResponse(UniversalBaseModel): + trees: typing.List[AgentaTreeDto] + version: str + count: typing.Optional[int] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + +update_forward_refs(SpanDto, AgentaTreesResponse=AgentaTreesResponse) diff --git a/agenta-cli/agenta/client/backend/types/collect_status_response.py b/agenta-cli/agenta/client/backend/types/collect_status_response.py new file mode 100644 index 0000000000..d52eed32ce --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/collect_status_response.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import typing +import pydantic + + +class CollectStatusResponse(UniversalBaseModel): + version: str + status: str + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/exception_dto.py b/agenta-cli/agenta/client/backend/types/exception_dto.py new file mode 100644 index 0000000000..a3e780d345 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/exception_dto.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import datetime as dt +import typing +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class ExceptionDto(UniversalBaseModel): + timestamp: dt.datetime + type: str + message: typing.Optional[str] = None + stacktrace: typing.Optional[str] = None + attributes: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/link_dto.py b/agenta-cli/agenta/client/backend/types/link_dto.py new file mode 100644 index 0000000000..91c76de759 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/link_dto.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from .tree_type import TreeType +import typing +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class LinkDto(UniversalBaseModel): + type: TreeType = "invocation" + id: str + tree_id: typing.Optional[str] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/node_dto.py b/agenta-cli/agenta/client/backend/types/node_dto.py new file mode 100644 index 0000000000..6caa131c32 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/node_dto.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from .node_type import NodeType +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class NodeDto(UniversalBaseModel): + id: str + name: str + type: typing.Optional[NodeType] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/node_type.py b/agenta-cli/agenta/client/backend/types/node_type.py new file mode 100644 index 0000000000..8abbe89309 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/node_type.py @@ -0,0 +1,19 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +NodeType = typing.Union[ + typing.Literal[ + "agent", + "workflow", + "chain", + "task", + "tool", + "embedding", + "query", + "completion", + "chat", + "rerank", + ], + typing.Any, +] diff --git a/agenta-cli/agenta/client/backend/types/o_tel_context_dto.py b/agenta-cli/agenta/client/backend/types/o_tel_context_dto.py new file mode 100644 index 0000000000..ab99bfac46 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_context_dto.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import typing +import pydantic + + +class OTelContextDto(UniversalBaseModel): + trace_id: str + span_id: str + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/o_tel_event_dto.py b/agenta-cli/agenta/client/backend/types/o_tel_event_dto.py new file mode 100644 index 0000000000..e5eed83822 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_event_dto.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class OTelEventDto(UniversalBaseModel): + name: str + timestamp: str + attributes: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/o_tel_extra_dto.py b/agenta-cli/agenta/client/backend/types/o_tel_extra_dto.py new file mode 100644 index 0000000000..c7e9294db3 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_extra_dto.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from .o_tel_event_dto import OTelEventDto +from .o_tel_link_dto import OTelLinkDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class OTelExtraDto(UniversalBaseModel): + kind: typing.Optional[str] = None + attributes: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + events: typing.Optional[typing.List[OTelEventDto]] = None + links: typing.Optional[typing.List[OTelLinkDto]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/o_tel_link_dto.py b/agenta-cli/agenta/client/backend/types/o_tel_link_dto.py new file mode 100644 index 0000000000..75ec3d1f1b --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_link_dto.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from .o_tel_context_dto import OTelContextDto +import typing +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class OTelLinkDto(UniversalBaseModel): + context: OTelContextDto + attributes: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/o_tel_span_dto.py b/agenta-cli/agenta/client/backend/types/o_tel_span_dto.py new file mode 100644 index 0000000000..66632172c9 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_span_dto.py @@ -0,0 +1,37 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from .o_tel_context_dto import OTelContextDto +import typing +from .o_tel_span_kind import OTelSpanKind +import datetime as dt +from .o_tel_status_code import OTelStatusCode +from .o_tel_event_dto import OTelEventDto +from .o_tel_link_dto import OTelLinkDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class OTelSpanDto(UniversalBaseModel): + context: OTelContextDto + name: str + kind: typing.Optional[OTelSpanKind] = None + start_time: dt.datetime + end_time: dt.datetime + status_code: typing.Optional[OTelStatusCode] = None + status_message: typing.Optional[str] = None + attributes: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + events: typing.Optional[typing.List[OTelEventDto]] = None + parent: typing.Optional[OTelContextDto] = None + links: typing.Optional[typing.List[OTelLinkDto]] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/o_tel_span_kind.py b/agenta-cli/agenta/client/backend/types/o_tel_span_kind.py new file mode 100644 index 0000000000..98ba7bf43c --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_span_kind.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +OTelSpanKind = typing.Union[ + typing.Literal[ + "SPAN_KIND_UNSPECIFIED", + "SPAN_KIND_INTERNAL", + "SPAN_KIND_SERVER", + "SPAN_KIND_CLIENT", + "SPAN_KIND_PRODUCER", + "SPAN_KIND_CONSUMER", + ], + typing.Any, +] diff --git a/agenta-cli/agenta/client/backend/types/o_tel_spans_response.py b/agenta-cli/agenta/client/backend/types/o_tel_spans_response.py new file mode 100644 index 0000000000..b9fb641427 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_spans_response.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from .o_tel_span_dto import OTelSpanDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class OTelSpansResponse(UniversalBaseModel): + version: str + count: typing.Optional[int] = None + spans: typing.List[OTelSpanDto] + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/o_tel_status_code.py b/agenta-cli/agenta/client/backend/types/o_tel_status_code.py new file mode 100644 index 0000000000..d5a60e6006 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/o_tel_status_code.py @@ -0,0 +1,8 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +OTelStatusCode = typing.Union[ + typing.Literal["STATUS_CODE_OK", "STATUS_CODE_ERROR", "STATUS_CODE_UNSET"], + typing.Any, +] diff --git a/agenta-cli/agenta/client/backend/types/parent_dto.py b/agenta-cli/agenta/client/backend/types/parent_dto.py new file mode 100644 index 0000000000..7bf3c33715 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/parent_dto.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import typing +import pydantic + + +class ParentDto(UniversalBaseModel): + id: str + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/root_dto.py b/agenta-cli/agenta/client/backend/types/root_dto.py new file mode 100644 index 0000000000..7b7e8f5aeb --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/root_dto.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import typing +import pydantic + + +class RootDto(UniversalBaseModel): + id: str + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/span_dto.py b/agenta-cli/agenta/client/backend/types/span_dto.py new file mode 100644 index 0000000000..80fdb2b0a3 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/span_dto.py @@ -0,0 +1,54 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from .lifecycle_dto import LifecycleDto +from .root_dto import RootDto +from .tree_dto import TreeDto +from .node_dto import NodeDto +from .parent_dto import ParentDto +from .time_dto import TimeDto +from .status_dto import StatusDto +from .exception_dto import ExceptionDto +from .link_dto import LinkDto +from .o_tel_extra_dto import OTelExtraDto +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic +from ..core.pydantic_utilities import update_forward_refs + + +class SpanDto(UniversalBaseModel): + lifecycle: typing.Optional[LifecycleDto] = None + root: RootDto + tree: TreeDto + node: NodeDto + parent: typing.Optional[ParentDto] = None + time: TimeDto + status: StatusDto + exception: typing.Optional[ExceptionDto] = None + data: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + metrics: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + meta: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + refs: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None + links: typing.Optional[typing.List[LinkDto]] = None + otel: typing.Optional[OTelExtraDto] = None + nodes: typing.Optional[ + typing.Dict[str, typing.Optional["SpanDtoNodesValue"]] + ] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow + + +from .span_dto_nodes_value import SpanDtoNodesValue # noqa: E402 + +update_forward_refs(SpanDto) diff --git a/agenta-cli/agenta/client/backend/types/span_dto_nodes_value.py b/agenta-cli/agenta/client/backend/types/span_dto_nodes_value.py new file mode 100644 index 0000000000..93e28b70de --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/span_dto_nodes_value.py @@ -0,0 +1,9 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations +import typing +import typing + +if typing.TYPE_CHECKING: + from .span_dto import SpanDto +SpanDtoNodesValue = typing.Union["SpanDto", typing.List["SpanDto"]] diff --git a/agenta-cli/agenta/client/backend/types/status_code.py b/agenta-cli/agenta/client/backend/types/status_code.py new file mode 100644 index 0000000000..ab7c307ab7 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/status_code.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +StatusCode = typing.Union[typing.Literal["UNSET", "OK", "ERROR"], typing.Any] diff --git a/agenta-cli/agenta/client/backend/types/status_dto.py b/agenta-cli/agenta/client/backend/types/status_dto.py new file mode 100644 index 0000000000..44f2ef907b --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/status_dto.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +from .status_code import StatusCode +import typing +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class StatusDto(UniversalBaseModel): + code: StatusCode + message: typing.Optional[str] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/time_dto.py b/agenta-cli/agenta/client/backend/types/time_dto.py new file mode 100644 index 0000000000..5def8ab023 --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/time_dto.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import datetime as dt +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import typing +import pydantic + + +class TimeDto(UniversalBaseModel): + start: dt.datetime + end: dt.datetime + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/tree_dto.py b/agenta-cli/agenta/client/backend/types/tree_dto.py new file mode 100644 index 0000000000..dfb98faaac --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/tree_dto.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +from ..core.pydantic_utilities import UniversalBaseModel +import typing +from .tree_type import TreeType +from ..core.pydantic_utilities import IS_PYDANTIC_V2 +import pydantic + + +class TreeDto(UniversalBaseModel): + id: str + type: typing.Optional[TreeType] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/agenta-cli/agenta/client/backend/types/tree_type.py b/agenta-cli/agenta/client/backend/types/tree_type.py new file mode 100644 index 0000000000..3be7057bec --- /dev/null +++ b/agenta-cli/agenta/client/backend/types/tree_type.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +TreeType = typing.Literal["invocation"] diff --git a/agenta-cli/agenta/client/backend/variants/client.py b/agenta-cli/agenta/client/backend/variants/client.py index 3e40a9d2f1..389a77f778 100644 --- a/agenta-cli/agenta/client/backend/variants/client.py +++ b/agenta-cli/agenta/client/backend/variants/client.py @@ -1104,14 +1104,14 @@ def configs_deploy( def configs_delete( self, *, - variant_ref: typing.Optional[ReferenceRequestModel] = OMIT, + variant_ref: ReferenceRequestModel, application_ref: typing.Optional[ReferenceRequestModel] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> int: """ Parameters ---------- - variant_ref : typing.Optional[ReferenceRequestModel] + variant_ref : ReferenceRequestModel application_ref : typing.Optional[ReferenceRequestModel] @@ -1125,13 +1125,15 @@ def configs_delete( Examples -------- - from agenta import AgentaApi + from agenta import AgentaApi, ReferenceRequestModel client = AgentaApi( api_key="YOUR_API_KEY", base_url="https://yourhost.com/path/to/api", ) - client.variants.configs_delete() + client.variants.configs_delete( + variant_ref=ReferenceRequestModel(), + ) """ _response = self._client_wrapper.httpx_client.request( "variants/configs/delete", @@ -1244,14 +1246,14 @@ def configs_list( def configs_history( self, *, - variant_ref: typing.Optional[ReferenceRequestModel] = OMIT, + variant_ref: ReferenceRequestModel, application_ref: typing.Optional[ReferenceRequestModel] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> typing.List[ConfigResponseModel]: """ Parameters ---------- - variant_ref : typing.Optional[ReferenceRequestModel] + variant_ref : ReferenceRequestModel application_ref : typing.Optional[ReferenceRequestModel] @@ -1265,13 +1267,15 @@ def configs_history( Examples -------- - from agenta import AgentaApi + from agenta import AgentaApi, ReferenceRequestModel client = AgentaApi( api_key="YOUR_API_KEY", base_url="https://yourhost.com/path/to/api", ) - client.variants.configs_history() + client.variants.configs_history( + variant_ref=ReferenceRequestModel(), + ) """ _response = self._client_wrapper.httpx_client.request( "variants/configs/history", @@ -2504,14 +2508,14 @@ async def main() -> None: async def configs_delete( self, *, - variant_ref: typing.Optional[ReferenceRequestModel] = OMIT, + variant_ref: ReferenceRequestModel, application_ref: typing.Optional[ReferenceRequestModel] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> int: """ Parameters ---------- - variant_ref : typing.Optional[ReferenceRequestModel] + variant_ref : ReferenceRequestModel application_ref : typing.Optional[ReferenceRequestModel] @@ -2527,7 +2531,7 @@ async def configs_delete( -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, ReferenceRequestModel client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2536,7 +2540,9 @@ async def configs_delete( async def main() -> None: - await client.variants.configs_delete() + await client.variants.configs_delete( + variant_ref=ReferenceRequestModel(), + ) asyncio.run(main()) @@ -2660,14 +2666,14 @@ async def main() -> None: async def configs_history( self, *, - variant_ref: typing.Optional[ReferenceRequestModel] = OMIT, + variant_ref: ReferenceRequestModel, application_ref: typing.Optional[ReferenceRequestModel] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> typing.List[ConfigResponseModel]: """ Parameters ---------- - variant_ref : typing.Optional[ReferenceRequestModel] + variant_ref : ReferenceRequestModel application_ref : typing.Optional[ReferenceRequestModel] @@ -2683,7 +2689,7 @@ async def configs_history( -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, ReferenceRequestModel client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2692,7 +2698,9 @@ async def configs_history( async def main() -> None: - await client.variants.configs_history() + await client.variants.configs_history( + variant_ref=ReferenceRequestModel(), + ) asyncio.run(main()) diff --git a/agenta-cli/agenta/sdk/__init__.py b/agenta-cli/agenta/sdk/__init__.py index 65886105db..daa653d1ce 100644 --- a/agenta-cli/agenta/sdk/__init__.py +++ b/agenta-cli/agenta/sdk/__init__.py @@ -17,6 +17,8 @@ FileInputURL, BinaryParam, Prompt, + AgentaNodeDto, + AgentaNodesResponse, ) from .tracing import Tracing, get_tracer diff --git a/agenta-cli/agenta/sdk/decorators/routing.py b/agenta-cli/agenta/sdk/decorators/routing.py index 0b5578c892..01270bef79 100644 --- a/agenta-cli/agenta/sdk/decorators/routing.py +++ b/agenta-cli/agenta/sdk/decorators/routing.py @@ -362,7 +362,7 @@ async def handle_success(self, result: Any, inline_trace: bool): log.info(f"Agenta SDK - exiting with success: 200") log.info(f"----------------------------------") - return BaseResponse(data=data, trace=trace) + return BaseResponse(data=data, tree=trace) def handle_failure(self, error: Exception): log.error("--------------------------------------------------") diff --git a/agenta-cli/agenta/sdk/tracing/inline.py b/agenta-cli/agenta/sdk/tracing/inline.py index 8c4165ea6f..a90525fc05 100644 --- a/agenta-cli/agenta/sdk/tracing/inline.py +++ b/agenta-cli/agenta/sdk/tracing/inline.py @@ -903,9 +903,9 @@ def parse_to_agenta_span_dto( if span_dto.data: span_dto.data = _unmarshal_attributes(span_dto.data) - # if "outputs" in span_dto.data: - # if "__default__" in span_dto.data["outputs"]: - # span_dto.data["outputs"] = span_dto.data["outputs"]["__default__"] + if "outputs" in span_dto.data: + if "__default__" in span_dto.data["outputs"]: + span_dto.data["outputs"] = span_dto.data["outputs"]["__default__"] # METRICS if span_dto.metrics: @@ -934,6 +934,17 @@ def parse_to_agenta_span_dto( else: parse_to_agenta_span_dto(v) + # MASK LINKS FOR NOW + span_dto.links = None + # ------------------ + + # MASK LIFECYCLE FOR NOW + # span_dto.lifecycle = None + if span_dto.lifecycle: + span_dto.lifecycle.updated_at = None + span_dto.lifecycle.updated_by_id = None + # ---------------------- + return span_dto @@ -945,6 +956,8 @@ def parse_to_agenta_span_dto( from litellm import cost_calculator from opentelemetry.sdk.trace import ReadableSpan +from agenta.sdk.types import AgentaNodeDto, AgentaNodesResponse + def parse_inline_trace( spans: Dict[str, ReadableSpan], @@ -992,51 +1005,19 @@ def parse_inline_trace( ### services.observability.service.query() ### ############################################## - LEGACY = True - inline_trace = None - - if LEGACY: - legacy_spans = [ - _parse_to_legacy_span(span_dto) for span_dto in span_idx.values() - ] - - root_span = agenta_span_dtos[0] - - trace_id = root_span.root.id.hex - latency = root_span.time.span / 1_000_000 - cost = root_span.metrics.get("acc", {}).get("costs", {}).get("total", 0.0) - tokens = { - "prompt_tokens": root_span.metrics.get("acc", {}) - .get("tokens", {}) - .get("prompt", 0), - "completion_tokens": root_span.metrics.get("acc", {}) - .get("tokens", {}) - .get("completion", 0), - "total_tokens": root_span.metrics.get("acc", {}) - .get("tokens", {}) - .get("total", 0), - } - - spans = [ - loads(span.model_dump_json(exclude_none=True)) for span in legacy_spans - ] - - inline_trace = { - "trace_id": trace_id, - "latency": latency, - "cost": cost, - "usage": tokens, - "spans": spans, - } - - else: - spans = [ - loads(span_dto.model_dump_json(exclude_none=True)) - for span_dto in agenta_span_dtos - ] - - inline_trace = spans # turn into Agenta Model ? - + spans = [ + loads( + span_dto.model_dump_json( + exclude_none=True, + exclude_defaults=True, + ) + ) + for span_dto in agenta_span_dtos + ] + inline_trace = AgentaNodesResponse( + version="1.0.0", + nodes=[AgentaNodeDto(**span) for span in spans], + ).model_dump(exclude_none=True, exclude_unset=True) return inline_trace @@ -1120,98 +1101,6 @@ class LlmTokens(BaseModel): total_tokens: Optional[int] = 0 -class CreateSpan(BaseModel): - id: str - app_id: str - variant_id: Optional[str] = None - variant_name: Optional[str] = None - inputs: Optional[Dict[str, Optional[Any]]] = None - internals: Optional[Dict[str, Optional[Any]]] = None - outputs: Optional[Union[str, Dict[str, Optional[Any]], List[Any]]] = None - config: Optional[Dict[str, Optional[Any]]] = None - environment: Optional[str] = None - tags: Optional[List[str]] = None - token_consumption: Optional[int] = None - name: str - parent_span_id: Optional[str] = None - attributes: Optional[Dict[str, Optional[Any]]] = None - spankind: str - status: str - user: Optional[str] = None - start_time: datetime - end_time: datetime - tokens: Optional[LlmTokens] = None - cost: Optional[float] = None - - -def _parse_to_legacy_span(span: SpanDTO) -> CreateSpan: - attributes = None - if span.otel: - attributes = span.otel.attributes or {} - - if span.otel.events: - for event in span.otel.events: - if event.name == "exception": - attributes.update(**event.attributes) - - legacy_span = CreateSpan( - id=span.node.id.hex[:24], - spankind=span.node.type, - name=span.node.name, - # - status=span.status.code.name, - # - start_time=span.time.start, - end_time=span.time.end, - # - parent_span_id=span.parent.id.hex[:24] if span.parent else None, - # - inputs=span.data.get("inputs") if span.data else {}, - internals=span.data.get("internals") if span.data else {}, - outputs=span.data.get("outputs") if span.data else {}, - # - environment=span.meta.get("environment") if span.meta else None, - config=span.meta.get("configuration") if span.meta else None, - # - tokens=( - LlmTokens( - prompt_tokens=span.metrics.get("acc", {}) - .get("tokens", {}) - .get("prompt", 0.0), - completion_tokens=span.metrics.get("acc", {}) - .get("tokens", {}) - .get("completion", 0.0), - total_tokens=span.metrics.get("acc", {}) - .get("tokens", {}) - .get("total", 0.0), - ) - if span.metrics - else None - ), - cost=( - span.metrics.get("acc", {}).get("costs", {}).get("total", 0.0) - if span.metrics - else None - ), - # - app_id=( - span.refs.get("application", {}).get("id", "missing-app-id") - if span.refs - else "missing-app-id" - ), - # - attributes=attributes, - # - variant_id=None, - variant_name=None, - tags=None, - token_consumption=None, - user=None, - ) - - return legacy_span - - TYPES_WITH_COSTS = [ "embedding", "query", diff --git a/agenta-cli/agenta/sdk/types.py b/agenta-cli/agenta/sdk/types.py index cab9fb4b2c..0784a8d3f6 100644 --- a/agenta-cli/agenta/sdk/types.py +++ b/agenta-cli/agenta/sdk/types.py @@ -4,6 +4,9 @@ from pydantic import ConfigDict, BaseModel, HttpUrl +from agenta.client.backend.types.agenta_node_dto import AgentaNodeDto +from agenta.client.backend.types.agenta_nodes_response import AgentaNodesResponse + @dataclass class MultipleChoice: @@ -23,9 +26,9 @@ class LLMTokenUsage(BaseModel): class BaseResponse(BaseModel): - version: Optional[str] = "2.0" + version: Optional[str] = "3.0" data: Optional[Union[str, Dict[str, Any]]] - trace: Optional[Dict[str, Any]] + tree: AgentaNodesResponse class DictInput(dict): diff --git a/agenta-cli/pyproject.toml b/agenta-cli/pyproject.toml index ca644f392e..9c3d731f41 100644 --- a/agenta-cli/pyproject.toml +++ b/agenta-cli/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "agenta" -version = "0.27.1" +version = "0.27.2a2" description = "The SDK for agenta is an open-source LLMOps platform." readme = "README.md" authors = ["Mahmoud Mabrouk "] diff --git a/agenta-web/src/components/Playground/Views/TestView.tsx b/agenta-web/src/components/Playground/Views/TestView.tsx index ae3305e6cf..c4cae71421 100644 --- a/agenta-web/src/components/Playground/Views/TestView.tsx +++ b/agenta-web/src/components/Playground/Views/TestView.tsx @@ -1,4 +1,4 @@ -import React, {useContext, useEffect, useRef, useState} from "react" +import React, {useContext, useEffect, useMemo, useRef, useState} from "react" import {Button, Input, Card, Row, Col, Space, Form, Modal} from "antd" import {CaretRightOutlined, CloseCircleOutlined, PlusOutlined} from "@ant-design/icons" import {callVariant} from "@/services/api" @@ -10,8 +10,9 @@ import { Parameter, Variant, StyleProps, - BaseResponseSpans, BaseResponse, + TraceDetailsV2, + TraceDetailsV3, } from "@/lib/Types" import {batchExecute, getStringOrJson, isDemo, randString, removeKeys} from "@/lib/helpers/utils" import LoadTestsModal from "../LoadTestsModal" @@ -33,11 +34,20 @@ import relativeTime from "dayjs/plugin/relativeTime" import duration from "dayjs/plugin/duration" import {useQueryParam} from "@/hooks/useQuery" import {formatCurrency, formatLatency, formatTokenUsage} from "@/lib/helpers/formatters" -import {dynamicComponent, dynamicService} from "@/lib/helpers/dynamic" +import {dynamicService} from "@/lib/helpers/dynamic" import {isBaseResponse, isFuncResponse} from "@/lib/helpers/playgroundResp" -import {fromBaseResponseToTraceSpanType} from "@/lib/transformers" +import {AgentaNodeDTO} from "@/services/observability/types" +import {isTraceDetailsV2, isTraceDetailsV3} from "@/lib/helpers/observability_helpers" +import GenericDrawer from "@/components/GenericDrawer" +import TraceHeader from "@/components/pages/observability/drawer/TraceHeader" +import TraceTree from "@/components/pages/observability/drawer/TraceTree" +import TraceContent from "@/components/pages/observability/drawer/TraceContent" +import { + buildNodeTree, + getNodeById, + observabilityTransformer, +} from "@/lib/helpers/observability_helpers" -const PlaygroundDrawer: any = dynamicComponent(`Playground/PlaygroundDrawer/PlaygroundDrawer`) const promptRevision: any = dynamicService("promptVersioning/api") dayjs.extend(relativeTime) @@ -174,7 +184,7 @@ interface BoxComponentProps { additionalData: { cost: number | null latency: number | null - usage: {completion_tokens: number; prompt_tokens: number; total_tokens: number} | null + usage: number | null } onInputParamChange: (paramName: string, newValue: any) => void onRun: () => void @@ -183,19 +193,7 @@ interface BoxComponentProps { isChatVariant?: boolean variant: Variant onCancel: () => void - traceSpans: - | { - trace_id: string - cost?: number - latency?: number - usage?: { - completion_tokens: number - prompt_tokens: number - total_tokens: number - } - spans?: BaseResponseSpans[] - } - | undefined + traceSpans: TraceDetailsV2 | TraceDetailsV3 | undefined } const BoxComponent: React.FC = ({ @@ -212,8 +210,31 @@ const BoxComponent: React.FC = ({ onCancel, traceSpans, }) => { + const [selectedTraceId, setSelectedTraceId] = useQueryParam("trace", "") + + const traces = useMemo(() => { + if (traceSpans && isTraceDetailsV3(traceSpans)) { + return traceSpans.nodes + .flatMap((node: AgentaNodeDTO) => buildNodeTree(node)) + .flatMap((item: any) => observabilityTransformer(item)) + } + }, [traceSpans]) + + const activeTrace = useMemo(() => (traces ? (traces[0] ?? null) : null), [traces]) + const [selected, setSelected] = useState("") + + useEffect(() => { + if (!selected) { + setSelected(activeTrace?.node.id ?? "") + } + }, [activeTrace, selected]) + + const selectedItem = useMemo( + () => (traces?.length ? getNodeById(traces, selected) : null), + [selected, traces], + ) + const {appTheme} = useAppTheme() - const [activeSpan, setActiveSpan] = useQueryParam("activeSpan") const classes = useStylesBox() const loading = result === LOADING_TEXT const [form] = Form.useForm() @@ -336,14 +357,14 @@ const BoxComponent: React.FC = ({ )} {additionalData?.cost || additionalData?.latency ? ( - Tokens: {formatTokenUsage(additionalData?.usage?.total_tokens)} + Tokens: {formatTokenUsage(additionalData?.usage)} Cost: {formatCurrency(additionalData?.cost)} Latency: {formatLatency(additionalData?.latency)} - {traceSpans?.spans?.length && isDemo() && ( + {traceSpans && isTraceDetailsV3(traceSpans) && ( @@ -353,15 +374,27 @@ const BoxComponent: React.FC = ({ "" )} - {traceSpans?.spans?.length && !!activeSpan && ( - setActiveSpan("")} - traceSpans={fromBaseResponseToTraceSpanType( - traceSpans.spans, - traceSpans.trace_id, - )} + {activeTrace && !!traces?.length && ( + setSelectedTraceId("")} + expandable + headerExtra={ + + } + mainContent={selectedItem ? : null} + sideContent={ + + } /> )} @@ -397,23 +430,10 @@ const App: React.FC = ({ Array<{ cost: number | null latency: number | null - usage: {completion_tokens: number; prompt_tokens: number; total_tokens: number} | null + usage: number | null }> >(testList.map(() => ({cost: null, latency: null, usage: null}))) - const [traceSpans, setTraceSpans] = useState< - | { - trace_id: string - cost?: number - latency?: number - usage?: { - completion_tokens: number - prompt_tokens: number - total_tokens: number - } - spans?: BaseResponseSpans[] - } - | undefined - >() + const [traceSpans, setTraceSpans] = useState() const [revisionNum] = useQueryParam("revision") useEffect(() => { @@ -567,32 +587,52 @@ const App: React.FC = ({ // Check result type // String, FuncResponse or BaseResponse if (typeof result === "string") { - res = {version: "2.0", data: result} as BaseResponse + res = {version: "3.0", data: result} as BaseResponse setResultForIndex(getStringOrJson(res.data), index) } else if (isFuncResponse(result)) { - res = {version: "2.0", data: result.message} as BaseResponse + const res = {version: "3.0", data: result.message} setResultForIndex(getStringOrJson(res.data), index) const {message, cost, latency, usage} = result + + // Set additional data setAdditionalDataList((prev) => { const newDataList = [...prev] - newDataList[index] = {cost, latency, usage} + newDataList[index] = { + cost, + latency, + usage: usage?.total_tokens, + } return newDataList }) } else if (isBaseResponse(result)) { res = result as BaseResponse setResultForIndex(getStringOrJson(res.data), index) - const {data, trace} = result + const {trace, version} = result + + // Main update logic setAdditionalDataList((prev) => { const newDataList = [...prev] - newDataList[index] = { - cost: trace?.cost || null, - latency: trace?.latency || null, - usage: trace?.usage || null, + if (version === "2.0" && isTraceDetailsV2(result.trace)) { + newDataList[index] = { + cost: result.trace?.cost ?? null, + latency: result.trace?.latency ?? null, + usage: result.trace?.usage?.total_tokens ?? null, + } + } else if (version === "3.0" && isTraceDetailsV3(result.trace)) { + const firstTraceNode = result.trace.nodes[0] + newDataList[index] = { + cost: firstTraceNode?.metrics?.acc?.costs?.total ?? null, + latency: firstTraceNode?.time?.span + ? firstTraceNode.time.span / 1_000_000 + : null, + usage: firstTraceNode?.metrics?.acc?.tokens?.total ?? null, + } } return newDataList }) + if (trace && isDemo()) { setTraceSpans(trace) } diff --git a/agenta-web/src/components/pages/observability/drawer/TraceContent.tsx b/agenta-web/src/components/pages/observability/drawer/TraceContent.tsx index f0c1387eaf..4ed1cab710 100644 --- a/agenta-web/src/components/pages/observability/drawer/TraceContent.tsx +++ b/agenta-web/src/components/pages/observability/drawer/TraceContent.tsx @@ -297,7 +297,11 @@ const TraceContent = ({activeTrace}: TraceContentProps) => { value1={
{" "} - {formatLatency(activeTrace?.metrics?.acc?.duration.total / 1000)} + {formatLatency( + activeTrace?.metrics?.acc?.duration.total + ? activeTrace?.metrics?.acc?.duration.total + : activeTrace?.metrics?.acc?.tokens.total / 1000, + )}
} /> diff --git a/agenta-web/src/components/pages/observability/drawer/TraceHeader.tsx b/agenta-web/src/components/pages/observability/drawer/TraceHeader.tsx index 95dd7be690..9b94100368 100644 --- a/agenta-web/src/components/pages/observability/drawer/TraceHeader.tsx +++ b/agenta-web/src/components/pages/observability/drawer/TraceHeader.tsx @@ -11,9 +11,9 @@ interface TraceHeaderProps { activeTrace: _AgentaRootsResponse traces: _AgentaRootsResponse[] setSelectedTraceId: (val: string) => void - activeTraceIndex: number - handleNextTrace: () => void - handlePrevTrace: () => void + activeTraceIndex?: number + handleNextTrace?: () => void + handlePrevTrace?: () => void } const useStyles = createUseStyles((theme: JSSTheme) => ({ diff --git a/agenta-web/src/components/pages/observability/drawer/TraceTree.tsx b/agenta-web/src/components/pages/observability/drawer/TraceTree.tsx index b3371c8f60..7d53ea2d06 100644 --- a/agenta-web/src/components/pages/observability/drawer/TraceTree.tsx +++ b/agenta-web/src/components/pages/observability/drawer/TraceTree.tsx @@ -96,7 +96,11 @@ const TreeContent = ({value}: {value: _AgentaRootsResponse}) => {
- {formatLatency(metrics?.acc?.duration.total / 1000)} + {formatLatency( + metrics?.acc?.duration.total + ? metrics?.acc?.duration.total + : metrics?.acc?.tokens.total / 1000, + )}
{metrics?.acc?.costs?.total && ( diff --git a/agenta-web/src/lib/Types.ts b/agenta-web/src/lib/Types.ts index e95d56bc41..2b2b6d2cea 100644 --- a/agenta-web/src/lib/Types.ts +++ b/agenta-web/src/lib/Types.ts @@ -1,6 +1,7 @@ import {StaticImageData} from "next/image" import {EvaluationFlow, EvaluationType} from "./enums" import {GlobalToken} from "antd" +import {AgentaNodeDTO} from "@/services/observability/types" export type JSSTheme = GlobalToken & {isDark: boolean; fontWeightMedium: number} @@ -493,7 +494,9 @@ export type ComparisonResultRow = { export type RequestMetadata = { cost: number latency: number - usage: {completion_tokens?: number; prompt_tokens?: number; total_tokens: number} + usage: + | {completion?: number; prompt?: number; total: number} + | {completion_tokens?: number; prompt_tokens?: number; total_tokens: number} } export type WithPagination = { @@ -544,16 +547,24 @@ export type FuncResponse = { usage: {completion_tokens: number; prompt_tokens: number; total_tokens: number} } -export type BaseResponse = { +export interface TraceDetailsV2 { + trace_id: string + cost?: number + latency?: number + usage: {completion_tokens: number; prompt_tokens: number; total_tokens: number} + spans?: BaseResponseSpans[] +} + +export interface TraceDetailsV3 { version: string + nodes: AgentaNodeDTO[] + count?: number | null +} + +export type BaseResponse = { + version?: string | null data: string | Record - trace?: { - trace_id: string - cost?: number - latency?: number - usage?: {completion_tokens: number; prompt_tokens: number; total_tokens: number} - spans?: BaseResponseSpans[] - } + trace: TraceDetailsV2 | TraceDetailsV3 } export type BaseResponseSpans = { diff --git a/agenta-web/src/lib/helpers/observability_helpers.ts b/agenta-web/src/lib/helpers/observability_helpers.ts index 82423f591d..368fad3a8d 100644 --- a/agenta-web/src/lib/helpers/observability_helpers.ts +++ b/agenta-web/src/lib/helpers/observability_helpers.ts @@ -1,4 +1,5 @@ import {_AgentaRootsResponse, AgentaNodeDTO, AgentaTreeDTO} from "@/services/observability/types" +import {BaseResponse, TraceDetailsV2, TraceDetailsV3} from "../Types" export const observabilityTransformer = ( item: AgentaTreeDTO | AgentaNodeDTO, @@ -74,3 +75,11 @@ export const getNodeById = ( } return null } + +export const isTraceDetailsV2 = (trace: BaseResponse["trace"]): trace is TraceDetailsV2 => { + return (trace as TraceDetailsV2)?.trace_id !== undefined +} + +export const isTraceDetailsV3 = (trace: BaseResponse["trace"]): trace is TraceDetailsV3 => { + return (trace as TraceDetailsV3)?.nodes !== undefined +} diff --git a/agenta-web/src/lib/helpers/playgroundResp.ts b/agenta-web/src/lib/helpers/playgroundResp.ts index 748b01870b..0ae2dcc430 100644 --- a/agenta-web/src/lib/helpers/playgroundResp.ts +++ b/agenta-web/src/lib/helpers/playgroundResp.ts @@ -5,5 +5,5 @@ export function isFuncResponse(res: any): res is FuncResponse { } export function isBaseResponse(res: any): res is BaseResponse { - return res && res?.version === "2.0" + return res && res?.version } diff --git a/agenta-web/src/services/observability/types/index.ts b/agenta-web/src/services/observability/types/index.ts index 081d761e1a..f2cb8034be 100644 --- a/agenta-web/src/services/observability/types/index.ts +++ b/agenta-web/src/services/observability/types/index.ts @@ -122,6 +122,7 @@ interface ParentContextDTO { interface NodeTimeDTO { start: string end: string + span?: number } export interface NodeStatusDTO {