From 0a9691a48567529be4b0d6ef4c0c6bd3d882b2c0 Mon Sep 17 00:00:00 2001
From: Kerem Yilmaz <kerem@skyvern.com>
Date: Mon, 26 Aug 2024 12:39:46 +0300
Subject: [PATCH] Add html elements tree in prompt (#731)

---
 skyvern-frontend/src/api/types.ts             |  1 +
 .../src/routes/tasks/detail/HTMLArtifact.tsx  | 66 +++++++++++++++++++
 .../src/routes/tasks/detail/StepArtifacts.tsx | 10 +--
 3 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 skyvern-frontend/src/routes/tasks/detail/HTMLArtifact.tsx

diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index c35b4356d..9bb79fe33 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -6,6 +6,7 @@ export const ArtifactType = {
   LLMResponseParsed: "llm_response_parsed",
   VisibleElementsTree: "visible_elements_tree",
   VisibleElementsTreeTrimmed: "visible_elements_tree_trimmed",
+  VisibleElementsTreeInPrompt: "visible_elements_tree_in_prompt",
   LLMPrompt: "llm_prompt",
   LLMRequest: "llm_request",
   HTMLScrape: "html_scrape",
diff --git a/skyvern-frontend/src/routes/tasks/detail/HTMLArtifact.tsx b/skyvern-frontend/src/routes/tasks/detail/HTMLArtifact.tsx
new file mode 100644
index 000000000..ce7d281de
--- /dev/null
+++ b/skyvern-frontend/src/routes/tasks/detail/HTMLArtifact.tsx
@@ -0,0 +1,66 @@
+import { artifactApiClient } from "@/api/AxiosClient";
+import { ArtifactApiResponse } from "@/api/types";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Textarea } from "@/components/ui/textarea";
+import { useQuery } from "@tanstack/react-query";
+import axios from "axios";
+
+// https://stackoverflow.com/a/60338028
+function format(html: string) {
+  const tab = "\t";
+  let result = "";
+  let indent = "";
+
+  html.split(/>\s*</).forEach(function (element) {
+    if (element.match(/^\/\w/)) {
+      indent = indent.substring(tab.length);
+    }
+
+    result += indent + "<" + element + ">\r\n";
+
+    if (element.match(/^<?\w[^>]*[^/]$/) && !element.startsWith("input")) {
+      indent += tab;
+    }
+  });
+
+  return result.substring(1, result.length - 3);
+}
+
+type Props = {
+  artifact: ArtifactApiResponse;
+};
+
+function HTMLArtifact({ artifact }: Props) {
+  const { data, isFetching, isError, error } = useQuery<string>({
+    queryKey: ["artifact", artifact.artifact_id],
+    queryFn: async () => {
+      if (artifact.uri.startsWith("file://")) {
+        return artifactApiClient
+          .get(`/artifact/text`, {
+            params: {
+              path: artifact.uri.slice(7),
+            },
+          })
+          .then((response) => response.data);
+      }
+      if (artifact.uri.startsWith("s3://") && artifact.signed_url) {
+        return axios.get(artifact.signed_url).then((response) => response.data);
+      }
+    },
+  });
+
+  if (isFetching) {
+    return <Skeleton className="h-48 w-full" />;
+  }
+
+  return (
+    <Textarea
+      className="w-full"
+      rows={15}
+      value={isError ? JSON.stringify(error) : format(data ?? "")}
+      readOnly
+    />
+  );
+}
+
+export { HTMLArtifact };
diff --git a/skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx b/skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx
index ea3457d44..64a54d490 100644
--- a/skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx
+++ b/skyvern-frontend/src/routes/tasks/detail/StepArtifacts.tsx
@@ -17,7 +17,7 @@ import { getImageURL } from "./artifactUtils";
 import { Input } from "@/components/ui/input";
 import { basicTimeFormat } from "@/util/timeFormat";
 import { useCredentialGetter } from "@/hooks/useCredentialGetter";
-
+import { HTMLArtifact } from "./HTMLArtifact";
 type Props = {
   id: string;
   stepProps: StepApiResponse;
@@ -61,9 +61,9 @@ function StepArtifacts({ id, stepProps }: Props) {
     (artifact) => artifact.artifact_type === ArtifactType.LLMRequest,
   );
 
-  const visibleElementsTreeTrimmed = artifacts?.filter(
+  const visibleElementsTreeInPrompt = artifacts?.find(
     (artifact) =>
-      artifact.artifact_type === ArtifactType.VisibleElementsTreeTrimmed,
+      artifact.artifact_type === ArtifactType.VisibleElementsTreeInPrompt,
   );
 
   const llmPrompt = artifacts?.find(
@@ -164,8 +164,8 @@ function StepArtifacts({ id, stepProps }: Props) {
         )}
       </TabsContent>
       <TabsContent value="element_tree_trimmed">
-        {visibleElementsTreeTrimmed ? (
-          <JSONArtifact artifacts={visibleElementsTreeTrimmed} />
+        {visibleElementsTreeInPrompt ? (
+          <HTMLArtifact artifact={visibleElementsTreeInPrompt} />
         ) : null}
       </TabsContent>
       <TabsContent value="html_element_tree">