From 3fca82acaa633a44ad78ff5c946229289f5f58bd Mon Sep 17 00:00:00 2001 From: Zamil Majdy Date: Thu, 7 Nov 2024 10:04:42 +0700 Subject: [PATCH 1/3] feat(backend): Enable json parsing with typing & conversion --- .../backend/backend/data/execution.py | 2 +- autogpt_platform/backend/backend/data/graph.py | 15 ++++++++------- autogpt_platform/backend/backend/util/json.py | 18 +++++++++++++++++- autogpt_platform/backend/backend/util/type.py | 16 +++++++++++++--- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py index 5b7d34a3b272..4573184e08b5 100644 --- a/autogpt_platform/backend/backend/data/execution.py +++ b/autogpt_platform/backend/backend/data/execution.py @@ -76,7 +76,7 @@ class ExecutionResult(BaseModel): def from_db(execution: AgentNodeExecution): if execution.executionData: # Execution that has been queued for execution will persist its data. - input_data = json.loads(execution.executionData) + input_data = json.loads(execution.executionData, target_type=dict[str, Any]) else: # For incomplete execution, executionData will not be yet available. input_data: BlockInput = defaultdict() diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.py index d490208262a3..4a285f649e88 100644 --- a/autogpt_platform/backend/backend/data/graph.py +++ b/autogpt_platform/backend/backend/data/graph.py @@ -55,8 +55,8 @@ def from_db(node: AgentNode): obj = Node( id=node.id, block_id=node.AgentBlock.id, - input_default=json.loads(node.constantInput), - metadata=json.loads(node.metadata), + input_default=json.loads(node.constantInput, target_type=dict[str, Any]), + metadata=json.loads(node.metadata, target_type=dict[str, Any]), ) obj.input_links = [Link.from_db(link) for link in node.Input or []] obj.output_links = [Link.from_db(link) for link in node.Output or []] @@ -79,10 +79,9 @@ def from_db(execution: AgentGraphExecution): duration = (end_time - start_time).total_seconds() total_run_time = duration - if execution.stats: - stats = json.loads(execution.stats) - duration = stats.get("walltime", duration) - total_run_time = stats.get("nodes_walltime", total_run_time) + stats = json.loads(execution.stats or "{}", target_type=dict[str, Any]) + duration = stats.get("walltime", duration) + total_run_time = stats.get("nodes_walltime", total_run_time) return GraphExecution( id=execution.id, @@ -290,7 +289,9 @@ def from_db(graph: AgentGraph, hide_credentials: bool = False): def _process_node(node: AgentNode, hide_credentials: bool) -> Node: node_dict = node.model_dump() if hide_credentials and "constantInput" in node_dict: - constant_input = json.loads(node_dict["constantInput"]) + constant_input = json.loads( + node_dict["constantInput"], target_type=dict[str, Any] + ) constant_input = Graph._hide_credentials_in_input(constant_input) node_dict["constantInput"] = json.dumps(constant_input) return Node.from_db(AgentNode(**node_dict)) diff --git a/autogpt_platform/backend/backend/util/json.py b/autogpt_platform/backend/backend/util/json.py index 64ab65291b29..fbfaed0e55d8 100644 --- a/autogpt_platform/backend/backend/util/json.py +++ b/autogpt_platform/backend/backend/util/json.py @@ -1,7 +1,10 @@ import json +from typing import Any, Type, TypeVar, overload from fastapi.encoders import jsonable_encoder +from .type import convert + def to_dict(data) -> dict: return jsonable_encoder(data) @@ -11,4 +14,17 @@ def dumps(data) -> str: return json.dumps(jsonable_encoder(data)) -loads = json.loads +T = TypeVar("T") + + +@overload +def loads(data: str, *args, target_type: Type[T], **kwargs) -> T: ... + + +@overload +def loads(data: str, *args, **kwargs) -> Any: ... + + +def loads(data: str, *args, target_type: Type[T] | None = None, **kwargs) -> Any: + parsed = json.loads(data, *args, **kwargs) + return convert(parsed, target_type) if target_type else parsed diff --git a/autogpt_platform/backend/backend/util/type.py b/autogpt_platform/backend/backend/util/type.py index 9c267aba2a71..a2ef9051a496 100644 --- a/autogpt_platform/backend/backend/util/type.py +++ b/autogpt_platform/backend/backend/util/type.py @@ -1,8 +1,8 @@ import json -from typing import Any, Type, TypeVar, get_args, get_origin +from typing import Any, Type, TypeVar, cast, get_args, get_origin -class ConversionError(Exception): +class ConversionError(ValueError): pass @@ -102,7 +102,7 @@ def __convert_bool(value: Any) -> bool: return bool(value) -def convert(value: Any, target_type: Type): +def _convert(value: Any, target_type: Type): origin = get_origin(target_type) args = get_args(target_type) if origin is None: @@ -175,3 +175,13 @@ def convert(value: Any, target_type: Type): return __convert_bool(value) else: return value + + +T = TypeVar("T") + + +def convert(value: Any, target_type: Type[T]) -> T: + try: + return cast(T, _convert(value, target_type)) + except Exception as e: + raise ConversionError(f"Failed to convert {value} to {target_type}") from e From 4f7e7d0cdcdceaaa7e90f6698bc752a5a2d93338 Mon Sep 17 00:00:00 2001 From: Zamil Majdy Date: Wed, 13 Nov 2024 15:16:34 +0700 Subject: [PATCH 2/3] Address comments --- autogpt_platform/backend/backend/data/graph.py | 6 +++++- autogpt_platform/backend/backend/util/json.py | 6 ++++-- autogpt_platform/backend/backend/util/type.py | 10 ++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.py index 7af2da9718ce..ba55ad978c07 100644 --- a/autogpt_platform/backend/backend/data/graph.py +++ b/autogpt_platform/backend/backend/data/graph.py @@ -80,7 +80,11 @@ def from_db(execution: AgentGraphExecution): duration = (end_time - start_time).total_seconds() total_run_time = duration - stats = json.loads(execution.stats or "{}", target_type=dict[str, Any]) + try: + stats = json.loads(execution.stats or "{}", target_type=dict[str, Any]) + except ValueError: + stats = {} + duration = stats.get("walltime", duration) total_run_time = stats.get("nodes_walltime", total_run_time) diff --git a/autogpt_platform/backend/backend/util/json.py b/autogpt_platform/backend/backend/util/json.py index fbfaed0e55d8..f8fb6f2fca04 100644 --- a/autogpt_platform/backend/backend/util/json.py +++ b/autogpt_platform/backend/backend/util/json.py @@ -3,7 +3,7 @@ from fastapi.encoders import jsonable_encoder -from .type import convert +from .type import type_match def to_dict(data) -> dict: @@ -27,4 +27,6 @@ def loads(data: str, *args, **kwargs) -> Any: ... def loads(data: str, *args, target_type: Type[T] | None = None, **kwargs) -> Any: parsed = json.loads(data, *args, **kwargs) - return convert(parsed, target_type) if target_type else parsed + if target_type: + return type_match(parsed, target_type) + return parsed diff --git a/autogpt_platform/backend/backend/util/type.py b/autogpt_platform/backend/backend/util/type.py index a2ef9051a496..f36d281ad129 100644 --- a/autogpt_platform/backend/backend/util/type.py +++ b/autogpt_platform/backend/backend/util/type.py @@ -102,7 +102,7 @@ def __convert_bool(value: Any) -> bool: return bool(value) -def _convert(value: Any, target_type: Type): +def _try_convert(value: Any, target_type: Type, raise_on_mismatch: bool) -> Any: origin = get_origin(target_type) args = get_args(target_type) if origin is None: @@ -133,6 +133,8 @@ def _convert(value: Any, target_type: Type): return {convert(v, args[0]) for v in value} else: return value + elif raise_on_mismatch: + raise ConversionError(f"Failed to convert {value} to {target_type}") else: # Need to convert value to the origin type if origin is list: @@ -180,8 +182,12 @@ def _convert(value: Any, target_type: Type): T = TypeVar("T") +def type_match(value: Any, target_type: Type[T]) -> T: + return cast(T, _try_convert(value, target_type, raise_on_mismatch=True)) + + def convert(value: Any, target_type: Type[T]) -> T: try: - return cast(T, _convert(value, target_type)) + return cast(T, _try_convert(value, target_type, raise_on_mismatch=False)) except Exception as e: raise ConversionError(f"Failed to convert {value} to {target_type}") from e From 0815d5a2f2a48294b0a1766d631a3ffcc31c1a98 Mon Sep 17 00:00:00 2001 From: Zamil Majdy Date: Fri, 15 Nov 2024 16:29:46 +0700 Subject: [PATCH 3/3] Update autogpt_platform/backend/backend/util/type.py Co-authored-by: Reinier van der Leer --- autogpt_platform/backend/backend/util/type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt_platform/backend/backend/util/type.py b/autogpt_platform/backend/backend/util/type.py index f36d281ad129..2c480b674d8f 100644 --- a/autogpt_platform/backend/backend/util/type.py +++ b/autogpt_platform/backend/backend/util/type.py @@ -134,7 +134,7 @@ def _try_convert(value: Any, target_type: Type, raise_on_mismatch: bool) -> Any: else: return value elif raise_on_mismatch: - raise ConversionError(f"Failed to convert {value} to {target_type}") + raise TypeError(f"Value {value} is not of expected type {target_type}") else: # Need to convert value to the origin type if origin is list: