From 704d45f131c0ddd15f4055f335b42c179abecfc1 Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Wed, 13 Dec 2023 12:56:04 -0600 Subject: [PATCH 1/5] Adding support for multion screenshots and video --- agentops/event.py | 11 +++++++---- agentops/session.py | 4 ++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/agentops/event.py b/agentops/event.py index 1bc0b2ab..c4468c6c 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -7,7 +7,7 @@ from .helpers import get_ISO_time, Models from typing import Optional, List from pydantic import Field - +from PIL import Image class Event: """ @@ -18,7 +18,7 @@ class Event: params (str, optional): The parameters passed to the operation. returns (str, optional): The output of the operation. result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate". Defaults to "Indeterminate". - action_type (str, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to 'action'. + action_type (str, optional): Type of action of the event e.g. 'action', 'llm', 'api', 'screenshot'. Defaults to 'action'. model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4). For models, see the types available in the Models enum. If a model is set but an action_type is not, the action_type will be coerced to 'llm'. @@ -27,6 +27,7 @@ class Event: tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"]. Defaults to None. init_timestamp (float, optional): The timestamp for when the event was initiated, represented as seconds since the epoch. Defaults to the end timestamp. + screenshot: A screenshot of the webapage at the time of the event Attributes: event_type (str): Type of the event. @@ -49,11 +50,12 @@ def __init__(self, event_type: str, pattern="^(Success|Fail|Indeterminate)$"), action_type: Optional[str] = Field("action", description="Type of action that the user is recording", - pattern="^(action|api|llm)$"), + pattern="^(action|api|llm|screenshot)$"), model: Optional[Models] = None, prompt: Optional[str] = None, tags: Optional[List[str]] = None, - init_timestamp: Optional[float] = None + init_timestamp: Optional[float] = None, + screenshot: Optional[Image.Image] = None ): self.event_type = event_type self.params = params @@ -65,3 +67,4 @@ def __init__(self, event_type: str, self.prompt = prompt self.end_timestamp = get_ISO_time() self.init_timestamp = init_timestamp if init_timestamp else self.end_timestamp + self.screenshot = screenshot \ No newline at end of file diff --git a/agentops/session.py b/agentops/session.py index e29c6cad..1ed99f38 100644 --- a/agentops/session.py +++ b/agentops/session.py @@ -27,6 +27,10 @@ def __init__(self, session_id: str, tags: Optional[List[str]] = None): self.session_id = session_id self.init_timestamp = get_ISO_time() self.tags = tags + self.video = None + + def add_video_recording(self, video): + self.video = video def end_session(self, end_state: str = "Indeterminate", rating: Optional[str] = None): """ From 82d4e7504c2d4770d7dd4089455ee913f45f6e23 Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Fri, 15 Dec 2023 12:39:55 -0500 Subject: [PATCH 2/5] Added support for screenshot images and screen recording videos --- agentops/client.py | 9 ++++++--- agentops/session.py | 4 ---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/agentops/client.py b/agentops/client.py index 4acb2055..cf206785 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -245,19 +245,22 @@ def start_session(self, tags: Optional[List[str]] = None): self.worker.start_session(self.session) def end_session(self, end_state: str = Field("Indeterminate", - description="End state of the session", - pattern="^(Success|Fail|Indeterminate)$"), - rating: Optional[str] = None): + description="End state of the session", + pattern="^(Success|Fail|Indeterminate)$"), + rating: Optional[str] = None, + video: Optional[str] = None): """ End the current session with the AgentOps service. Args: end_state (str, optional): The final state of the session. rating (str, optional): The rating for the session. + video (str, optional): The video screen recording of the session """ if not self.session.has_ended: self.session.end_session(end_state, rating) self.worker.end_session(self.session) + self.session.video = video else: logging.info("Warning: The session has already been ended.") diff --git a/agentops/session.py b/agentops/session.py index 1ed99f38..e29c6cad 100644 --- a/agentops/session.py +++ b/agentops/session.py @@ -27,10 +27,6 @@ def __init__(self, session_id: str, tags: Optional[List[str]] = None): self.session_id = session_id self.init_timestamp = get_ISO_time() self.tags = tags - self.video = None - - def add_video_recording(self, video): - self.video = video def end_session(self, end_state: str = "Indeterminate", rating: Optional[str] = None): """ From fd5bf9cf8ab4694193b082b214cbec0757c1232c Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Fri, 15 Dec 2023 13:18:19 -0500 Subject: [PATCH 3/5] Changed screenshot type from PIL Image to string (base64) --- agentops/event.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/agentops/event.py b/agentops/event.py index c4468c6c..fe608bf8 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -7,7 +7,6 @@ from .helpers import get_ISO_time, Models from typing import Optional, List from pydantic import Field -from PIL import Image class Event: """ @@ -27,7 +26,7 @@ class Event: tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"]. Defaults to None. init_timestamp (float, optional): The timestamp for when the event was initiated, represented as seconds since the epoch. Defaults to the end timestamp. - screenshot: A screenshot of the webapage at the time of the event + screenshot (str, optional): A screenshot of the webapage at the time of the event. Base64 string Attributes: event_type (str): Type of the event. @@ -55,7 +54,7 @@ def __init__(self, event_type: str, prompt: Optional[str] = None, tags: Optional[List[str]] = None, init_timestamp: Optional[float] = None, - screenshot: Optional[Image.Image] = None + screenshot: Optional[str] = None ): self.event_type = event_type self.params = params From 72ab84fe53067614c1624f5d888e0351492b580b Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Fri, 15 Dec 2023 13:34:51 -0500 Subject: [PATCH 4/5] Updated arg definition to reflect that screenshot can be base64 or url --- agentops/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentops/event.py b/agentops/event.py index fe608bf8..259c5944 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -26,7 +26,7 @@ class Event: tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"]. Defaults to None. init_timestamp (float, optional): The timestamp for when the event was initiated, represented as seconds since the epoch. Defaults to the end timestamp. - screenshot (str, optional): A screenshot of the webapage at the time of the event. Base64 string + screenshot (str, optional): A screenshot of the webapage at the time of the event. Base64 string or URL. Defaults to None. Attributes: event_type (str): Type of the event. From 3f01ae1009d63a58414248193b88806359e5bed5 Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Fri, 15 Dec 2023 14:24:23 -0500 Subject: [PATCH 5/5] typo --- agentops/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentops/event.py b/agentops/event.py index 259c5944..e33b76d6 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -26,7 +26,7 @@ class Event: tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"]. Defaults to None. init_timestamp (float, optional): The timestamp for when the event was initiated, represented as seconds since the epoch. Defaults to the end timestamp. - screenshot (str, optional): A screenshot of the webapage at the time of the event. Base64 string or URL. Defaults to None. + screenshot (str, optional): A screenshot of the webpage at the time of the event. Base64 string or URL. Defaults to None. Attributes: event_type (str): Type of the event.