diff --git a/next/package.json b/next/package.json index c493f60139..176586fb0d 100644 --- a/next/package.json +++ b/next/package.json @@ -48,14 +48,12 @@ "next-auth": "4.20.1", "next-i18next": "^13.2.2", "nextjs-google-analytics": "^2.3.3", - "pusher-js": "^8.2.0", "react": "18.2.0", "react-dom": "18.2.0", "react-i18next": "^12.3.1", "react-icons": "^4.8.0", "react-markdown": "^8.0.7", "react-type-animation": "^3.1.0", - "reactflow": "^11.7.4", "rehype-highlight": "^6.0.0", "remark-gfm": "^3.0.1", "superjson": "1.9.1", diff --git a/next/src/components/drawer/WorkflowSidebar.tsx b/next/src/components/drawer/WorkflowSidebar.tsx deleted file mode 100644 index 65baedf90d..0000000000 --- a/next/src/components/drawer/WorkflowSidebar.tsx +++ /dev/null @@ -1,186 +0,0 @@ -import type { FC } from "react"; -import { FaBars } from "react-icons/fa"; -import type { Edge, Node } from "reactflow"; - -import { SidebarTransition } from "./Sidebar"; -import type { createNodeType, updateNodeType } from "../../hooks/useWorkflow"; -import { findParents } from "../../services/graph-utils"; -import type { IOField, NodeBlockDefinition } from "../../services/workflow/node-block-definitions"; -import { - getNodeBlockDefinitionFromNode, - getNodeBlockDefinitions, -} from "../../services/workflow/node-block-definitions"; -import { useConfigStore } from "../../stores/configStore"; -import type { WorkflowEdge, WorkflowNode } from "../../types/workflow"; -import WorkflowSidebarInput from "../../ui/WorkflowSidebarInput"; - -type WorkflowControls = { - selectedNode: Node | undefined; - nodes: Node[]; - edges: Edge[]; - createNode: createNodeType; - updateNode: updateNodeType; -}; - -const WorkflowSidebar: FC = (controls) => { - const { layout, setLayout } = useConfigStore(); - - const setShow = (show: boolean) => { - setLayout({ showRightSidebar: show }); - }; - - return ( - -
-
- -
-
- -
- - ); -}; - -type InspectSectionProps = { - selectedNode: Node | undefined; - updateNode: updateNodeType; - nodes: Node[]; - edges: Edge[]; -}; - -const InspectSection = ({ selectedNode, updateNode, nodes, edges }: InspectSectionProps) => { - if (selectedNode == undefined) - return ( -
- No components selected. Click on a component to select it -
- ); - - const definition = getNodeBlockDefinitionFromNode(selectedNode); - - const handleValueChange = (name: string, value: string) => { - const updatedNode = { ...selectedNode }; - updatedNode.data.block.input[name] = value; - updateNode(updatedNode); - }; - - const outputFields = findParents(nodes, edges, selectedNode).flatMap((ancestorNode) => { - const definition = getNodeBlockDefinitionFromNode(ancestorNode); - if (definition == undefined) return []; - - const outputFields = definition.output_fields; - return outputFields.map((outputField) => ({ - key: `{{${ancestorNode.id}.${outputField.name}}}`, - value: `${definition.type}.${outputField.name}`, - })); - }); - - const handleAutocompleteClick = (inputField: IOField, field: { key: string; value: string }) => { - handleValueChange( - inputField.name, - `${selectedNode.data.block.input[inputField.name] || ""}{{${field.key}}}` - ); - }; - - return ( - <> -
-

{definition?.type}

-

{definition?.description}

-
-
-
Inputs
- {definition?.input_fields.map((inputField: IOField) => ( -
- handleValueChange(inputField.name, val)} - suggestions={outputFields} - /> -
- ))} - {definition?.input_fields.length == 0 && ( -

This node does not take any input.

- )} -
-
-
Outputs
-
- {definition?.output_fields.map((outputField: IOField) => ( -
-

- {outputField.name}:{" "} - - {outputField.type} - -

-

- {outputField.description} -

-
- ))} - {definition?.output_fields.length == 0 && ( -

This node does not have any output.

- )} -
-
- - ); -}; - -type CreateSectionProps = { - createNode: createNodeType; -}; - -const CreateSection = ({ createNode }: CreateSectionProps) => { - return ( - <> - {getNodeBlockDefinitions().map((nodeBlockDefinition) => ( - - ))} - - ); -}; - -type NodeBlockProps = { - definition: NodeBlockDefinition; - createNode: createNodeType; -}; -const NodeBlock = ({ definition, createNode }: NodeBlockProps) => { - return ( -
{ - const input: Record = {}; - for (const field of definition.input_fields) { - input[field.name] = ""; - } - - createNode({ input: input, type: definition.type }, { x: 0, y: 0 }); - }} - > -
- -

{definition.type}

-
-

{definition.description}

-
- ); -}; - -export default WorkflowSidebar; diff --git a/next/src/components/workflow/BasicEdge.tsx b/next/src/components/workflow/BasicEdge.tsx deleted file mode 100644 index 384d70ca24..0000000000 --- a/next/src/components/workflow/BasicEdge.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import React, { memo } from "react"; -import type { EdgeProps } from "reactflow"; -import { BaseEdge, getBezierPath } from "reactflow"; - -import type { WorkflowEdge } from "../../types/workflow"; - -const edgeColors = { - running: "yellow", - success: "green", - error: "red", -}; - -const CustomEdge = ({ - id, - sourceX, - sourceY, - targetX, - targetY, - sourcePosition, - targetPosition, - style = {}, - markerEnd, - ...props -}: EdgeProps) => { - const [edgePath, labelX, labelY] = getBezierPath({ - sourceX, - sourceY, - sourcePosition, - targetX, - targetY, - targetPosition, - }); - - return ( - - ); -}; - -export default memo(CustomEdge); diff --git a/next/src/components/workflow/BlockDialog.tsx b/next/src/components/workflow/BlockDialog.tsx deleted file mode 100644 index b38c4f2f80..0000000000 --- a/next/src/components/workflow/BlockDialog.tsx +++ /dev/null @@ -1,136 +0,0 @@ -import { Combobox, Dialog, Transition } from "@headlessui/react"; -import clsx from "clsx"; -import type { Dispatch, SetStateAction } from "react"; -import { Fragment, useState } from "react"; -import { FaSearch } from "react-icons/fa"; -import { NodeBlockDefinition } from "../../services/workflow/node-block-definitions"; - -interface Props { - openModel: [boolean, Dispatch>]; - items: NodeBlockDefinition[]; - onClick: (value: NodeBlockDefinition) => void; -} - -export default function BlockDialog({ items, openModel, onClick }: Props) { - const [query, setQuery] = useState(""); - - const filteredItems = - query === "" - ? items - : items.filter((item) => { - return item.name.toLowerCase().includes(query.toLowerCase()); - }); - - return ( - setQuery("")} appear> - - -
- - -
- - - { - onClick(t as NodeBlockDefinition); - openModel[1](false); - }} - > -
-
- - {filteredItems.length > 0 && ( - - {filteredItems.map((item) => ( - - clsx( - "flex cursor-default select-none rounded-xl p-3", - active && "bg-gray-100" - ) - } - > - {({ active }) => ( -
-
-
-
-

- {item.name} -

-

- {item.description} -

-
-
- )} - - ))} - - )} - - {query !== "" && filteredItems.length === 0 && ( -
- -

No results found

-

- No Blocks found for this search term. Please try again. -

-
- )} - - - -
-
- - ); -} diff --git a/next/src/components/workflow/CreateWorkflowDialog.tsx b/next/src/components/workflow/CreateWorkflowDialog.tsx deleted file mode 100644 index 0dabac5923..0000000000 --- a/next/src/components/workflow/CreateWorkflowDialog.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import React from "react"; - -import Dialog from "../../ui/dialog"; -import Input from "../../ui/input"; -import PrimaryButton from "../PrimaryButton"; -import TextButton from "../TextButton"; - -const CreateWorkflowDialog = ({ - createWorkflow, - showDialog, - setShowDialog, -}: { - createWorkflow: (name: string, description: string) => void; - showDialog: boolean; - setShowDialog: (boolean) => void; -}) => { - const [name, setName] = React.useState(""); - const [description, setDescription] = React.useState(""); - const [isError, setIsError] = React.useState(false); - - const handleCreate = () => { - if (name === "" || description === "") { - setIsError(true); - return; - } - - setIsError(false); - createWorkflow(name, description); - setShowDialog(false); - }; - - return ( - } - actions={ - <> - Create - setShowDialog(false)}>Close - - } - > -
- setName(e.target.value)} - /> - setDescription(e.target.value)} - /> - {isError && ( -

- Please provide a name and a description -

- )} -
- - ); -}; - -export default CreateWorkflowDialog; diff --git a/next/src/components/workflow/EmptyWorkflow.tsx b/next/src/components/workflow/EmptyWorkflow.tsx deleted file mode 100644 index 1f938bc373..0000000000 --- a/next/src/components/workflow/EmptyWorkflow.tsx +++ /dev/null @@ -1,30 +0,0 @@ -type Props = { - onClick: () => void; -}; -export default function EmptyWorkflowButton({ onClick }: Props) { - return ( - - ); -} diff --git a/next/src/components/workflow/Flowchart.tsx b/next/src/components/workflow/Flowchart.tsx deleted file mode 100644 index 49d1f17a29..0000000000 --- a/next/src/components/workflow/Flowchart.tsx +++ /dev/null @@ -1,209 +0,0 @@ -import type { ComponentProps, MouseEvent as ReactMouseEvent } from "react"; -import React, { forwardRef, useCallback, useImperativeHandle, useRef, useState } from "react"; -import type { - Connection, - EdgeChange, - FitViewOptions, - NodeChange, - OnConnectStartParams, -} from "reactflow"; -import ReactFlow, { - addEdge, - applyEdgeChanges, - applyNodeChanges, - Background, - BackgroundVariant, - Controls, - MiniMap, - ReactFlowProvider, - useReactFlow, - useStore, -} from "reactflow"; - -import "reactflow/dist/style.css"; - -import CustomEdge from "./BasicEdge"; -import { BasicNode, IfNode, TriggerNode } from "./nodes"; -import type { Position } from "../../hooks/useWorkflow"; -import type { EdgesModel, NodesModel } from "../../types/workflow"; - -const nodeTypes = { - if: IfNode, - custom: BasicNode, - trigger: TriggerNode, -}; - -const edgeTypes = { - custom: CustomEdge, -}; - -const fitViewOptions: FitViewOptions = { - padding: 0.2, -}; - -interface FlowChartProps extends ComponentProps { - onSave?: (e: ReactMouseEvent) => Promise; - controls?: boolean; - minimap?: boolean; - setOnConnectStartParams: (params: OnConnectStartParams | undefined) => void; // Specify which node to connect to on edge drag - onPaneDoubleClick: (clickPosition: Position) => void; - - // workflow: Workflow; - nodesModel: NodesModel; - edgesModel: EdgesModel; -} - -export interface FlowChartHandles { - fitView: () => void; -} - -const FlowChart = forwardRef( - ({ onSave, nodesModel, edgesModel, ...props }, ref) => { - const [nodes, setNodes] = [nodesModel.get() ?? [], nodesModel.set]; - const [edges, setEdges] = [edgesModel.get() ?? [], edgesModel.set]; - const flow = useReactFlow(); - const [lastClickTime, setLastClickTime] = useState(null); - const connectionDragging = useRef(false); - const transform = useStore((state) => state.transform); - - const { onPaneDoubleClick, setOnConnectStartParams, ...rest } = props; - - const getExactPosition = useCallback( - (event: MouseEvent | ReactMouseEvent | TouchEvent) => { - let x, y; - const rect = (event.target as Element).getBoundingClientRect(); - - if (!(event instanceof TouchEvent)) { - x = event.clientX - rect.left; - y = event.clientY - rect.top; - } else { - x = (event.touches[0] || { clientX: 0 }).clientX - rect.left; - y = (event.touches[0] || { clientY: 0 }).clientY - rect.top; - } - - // calculate exact position considering zoom level and pan offset of the pane - const exactX = (x - transform[0]) / transform[2]; - const exactY = (y - transform[1]) / transform[2]; - - return { x: exactX - 135 /* Half of the width of the node */, y: exactY }; - }, - [transform] - ); - - const handlePaneClick = (event: ReactMouseEvent) => { - // Check if it was a double click - const currentTime = new Date().getTime(); - const doubleClickDelay = 400; - if (lastClickTime && currentTime - lastClickTime < doubleClickDelay) { - setOnConnectStartParams(undefined); // Reset the on connect as we are not dragging anymore - onPaneDoubleClick(getExactPosition(event)); - } else { - setLastClickTime(currentTime); - } - }; - - const onNodesChange = useCallback( - (changes: NodeChange[]) => { - const currentNodes = nodesModel.get(); - const updatedNodes = applyNodeChanges(changes, currentNodes ?? []); - setNodes(updatedNodes); - }, - [setNodes, nodesModel] - ); - const onEdgesChange = useCallback( - (changes: EdgeChange[]) => { - const currentEdges = edgesModel.get(); - const updatedEdges = applyEdgeChanges(changes, currentEdges ?? []); - setEdges(updatedEdges); - }, - [setEdges, edgesModel] - ); - - const onConnectStart = useCallback( - (_, params: OnConnectStartParams) => { - setOnConnectStartParams(params); - connectionDragging.current = true; - }, - [setOnConnectStartParams, connectionDragging] - ); - - const onConnect = useCallback( - (connection: Connection) => { - if (connection.source === connection.target) return; - - const currentEdges = edgesModel.get(); - const updatedEdges = addEdge({ ...connection, animated: true }, currentEdges ?? []); - setEdges(updatedEdges); - - connectionDragging.current = false; - }, - [setEdges, edgesModel] - ); - - const onConnectEnd = useCallback( - (event: MouseEvent | TouchEvent) => { - if (!connectionDragging.current) return; - connectionDragging.current = false; - onPaneDoubleClick(getExactPosition(event)); - }, - [getExactPosition, onPaneDoubleClick, connectionDragging] - ); - - useImperativeHandle(ref, () => ({ - fitView: () => { - flow?.fitView(fitViewOptions); - }, - })); - - return ( - onConnectEnd(event)} - nodeTypes={nodeTypes} - edgeTypes={edgeTypes} - proOptions={{ hideAttribution: true }} - fitViewOptions={fitViewOptions} - fitView - zoomOnDoubleClick={false} - {...rest} - onPaneClick={handlePaneClick} - > - -
- - {props.minimap && } - {props.controls && } - - ); - } -); - -FlowChart.displayName = "FlowChart"; - -const WrappedFlowchart = forwardRef((props, ref) => { - return ( - - {/* @ts-ignore*/} - - - ); -}); - -WrappedFlowchart.displayName = "WrappedFlowchart"; -export default WrappedFlowchart; diff --git a/next/src/components/workflow/WorkflowCard.tsx b/next/src/components/workflow/WorkflowCard.tsx deleted file mode 100644 index 8e53d3c3e3..0000000000 --- a/next/src/components/workflow/WorkflowCard.tsx +++ /dev/null @@ -1,107 +0,0 @@ -import { motion } from "framer-motion"; - -import type { WorkflowMeta } from "../../services/workflow/workflowApi"; -import Input from "../../ui/input"; -import PrimaryButton from "../PrimaryButton"; -import TextButton from "../TextButton"; - -type Props = { - workflow: WorkflowMeta; - onClick: () => void; -}; - -export default function WorkflowCard({ workflow, onClick }: Props) { - return ( - - -

{workflow.name}

-

{workflow.description}

- {workflow.organization_id && ( -

Org: {workflow.organization_id}

- )} - - - - - - ); -} - -type WorkflowCardDialogProps = { - workflow: WorkflowMeta; - onEdit: () => void; - onDelete: () => void; - onClose: () => void; -}; - -// Note, this is currently not being used. -export function WorkflowCardDialog({ - workflow, - onEdit, - onDelete, - onClose, -}: WorkflowCardDialogProps) { - return ( - <> - - - - -

{workflow.name}

-

{workflow.description}

- - - - - - - - - Close - Delete - Edit - - - - - - - - ); -} diff --git a/next/src/components/workflow/WorkflowChat.tsx b/next/src/components/workflow/WorkflowChat.tsx deleted file mode 100644 index f31abd7128..0000000000 --- a/next/src/components/workflow/WorkflowChat.tsx +++ /dev/null @@ -1,120 +0,0 @@ -import React, { useState, FC } from 'react'; -import { motion } from "framer-motion"; -import { AiOutlineArrowUp } from "react-icons/ai"; -import { streamText } from "../../services/stream-utils"; -import MarkdownRenderer from '../console/MarkdownRenderer'; -import { useAuth } from '../../hooks/useAuth'; -import { useRouter } from "next/router"; - -interface Message { - id: number; - role: string; - content: string; -} - -const WorkflowChat: FC = () => { - const [messages, setMessages] = useState([]); - const [input, setInput] = useState(''); - const [isLoading, setIsLoading] = useState(false); - const { session } = useAuth(); - const accessToken = session?.accessToken || ""; - const router = useRouter(); - const workflowId = router.query.w as string; - - const handleInputChange = (event) => { - setInput(event.target.value); - }; - - const handleSubmit = async (e) => { - e.preventDefault(); - setMessages((prevMessages) => [ - ...prevMessages, - { id: prevMessages.length, role: "user", content: 'User: ' + input } - ]); - setInput(""); - - try { - let content = ""; - await streamText( - "/api/workflowchat/workflow_chat", - { - message: input, - model_settings: { language: "English", model: "gpt-3.5-turbo", temperature: 0.8, max_tokens: 400, custom_api_key: "" }, - workflow_id: workflowId, - }, - accessToken, - () => { - setIsLoading(true); - setMessages((prevMessages) => [ - ...prevMessages, - { id: prevMessages.length, role: "assistant", content: content } - ]); - }, - (text: string) => { - setMessages((prevMessages) => { - const lastMessage = prevMessages[prevMessages.length - 1]; - const newContent = lastMessage?.content ?? ''; - const updatedContent = newContent + text; - - return prevMessages.map((message, index) => { - if (index === prevMessages.length - 1) { - return { ...message, content: updatedContent }; - } else { - return message; - } - }); - }); - }, - () => false - ); - } catch (error) { - console.error('Error while fetching data:', error); - } finally { - setIsLoading(false); - } - }; - - return ( - <> -
-
-
- - {messages.length <= 0 && ( -
- Ask any question about your PDF -
- )} - - {messages.map(m => { - if (m.role === "user") return ( -
{m.content}
- ) - return ( -
- {m.content} -
- ) - })} -
- - -
- -
- -
-
- -
-
- - ); -}; - -export default WorkflowChat; \ No newline at end of file diff --git a/next/src/components/workflow/WorkflowChatDialog.tsx b/next/src/components/workflow/WorkflowChatDialog.tsx deleted file mode 100644 index ea89afa6f6..0000000000 --- a/next/src/components/workflow/WorkflowChatDialog.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import {Dialog, Transition } from "@headlessui/react"; -import type { Dispatch, SetStateAction } from "react"; -import { Fragment, useState } from "react"; -import WorkflowChat from "./WorkflowChat"; - -interface Props { - openModel: [boolean, Dispatch>]; -} - -export default function WorkflowChatDialog({ openModel }: Props) { - return ( - - - -
- -
- - - - - -
-
- - ); -} diff --git a/next/src/components/workflow/WorkflowDialog.tsx b/next/src/components/workflow/WorkflowDialog.tsx deleted file mode 100644 index 947e4515f2..0000000000 --- a/next/src/components/workflow/WorkflowDialog.tsx +++ /dev/null @@ -1,82 +0,0 @@ -import React, { useEffect } from "react"; - -import type { WorkflowMeta } from "../../services/workflow/workflowApi"; -import Dialog from "../../ui/dialog"; -import Input from "../../ui/input"; -import PrimaryButton from "../PrimaryButton"; -import TextButton from "../TextButton"; - -const WorkflowDialog = ({ - workflow, - openWorkflow, - deleteWorkflow, - showDialog, - setShowDialog, -}: { - workflow: WorkflowMeta | null; - openWorkflow: () => void; - deleteWorkflow: () => void; - showDialog: boolean; - setShowDialog: (boolean) => void; -}) => { - const [name, setName] = React.useState(""); - const [description, setDescription] = React.useState(""); - - useEffect(() => { - if (!workflow) return; - - setName(workflow.name); - setDescription(workflow.description); - }, [workflow]); - - return ( - } - actions={ - <> - openWorkflow()}>Open -
- { - deleteWorkflow(); - setShowDialog(false); - }} - > - Delete - - setShowDialog(false)}> - Close - -
- - } - > -
- setName(e.target.value)} - /> - setDescription(e.target.value)} - /> -
- - ); -}; - -export default WorkflowDialog; diff --git a/next/src/components/workflow/nodes/AbstractNode.tsx b/next/src/components/workflow/nodes/AbstractNode.tsx deleted file mode 100644 index e97e02742f..0000000000 --- a/next/src/components/workflow/nodes/AbstractNode.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import clsx from "clsx"; -import type { PropsWithChildren } from "react"; -import React, { memo } from "react"; -import { HiEllipsisHorizontal } from "react-icons/hi2"; -import type { HandleType, Position } from "reactflow"; -import { Handle } from "reactflow"; - -import type { NodeBlockDefinition } from "../../../services/workflow/node-block-definitions"; -import { useConfigStore } from "../../../stores/configStore"; - -interface Handle { - position: Position; - type: HandleType; - className?: string; - id?: string; - text?: string; -} - -interface NodeProps extends PropsWithChildren { - handles: Handle[]; - selected: boolean; - status?: string; -} - -const AbstractNode = (props: NodeProps) => ( -
- {props.children} - {props.handles.map(({ position, type, text, className, id }, i) => ( - - {text} - - ))} -
-); - -export default memo(AbstractNode); - -export const NodeTitle = ({ definition }: { definition?: NodeBlockDefinition }) => { - const setLayout = useConfigStore().setLayout; - if (!definition) return <>; - - return ( - <> -
- -
{definition?.name}
- -
-
{definition?.description}
- - ); -}; - -// background: linear-gradient(180deg, #FA4D62 0%, #C21026 100%); diff --git a/next/src/components/workflow/nodes/BasicNode.tsx b/next/src/components/workflow/nodes/BasicNode.tsx deleted file mode 100644 index 74c3a35531..0000000000 --- a/next/src/components/workflow/nodes/BasicNode.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import React, { memo } from "react"; -import { type NodeProps, Position } from "reactflow"; - -import AbstractNode, { NodeTitle } from "./AbstractNode"; -import { getNodeBlockDefinitions } from "../../../services/workflow/node-block-definitions"; -import type { WorkflowNode } from "../../../types/workflow"; - -function BasicNode({ data, selected }: NodeProps) { - const definition = getNodeBlockDefinitions().find((d) => d.type === data.block.type); - - return ( - - - - ); -} - -export default memo(BasicNode); diff --git a/next/src/components/workflow/nodes/IfNode.tsx b/next/src/components/workflow/nodes/IfNode.tsx deleted file mode 100644 index 393c0b7061..0000000000 --- a/next/src/components/workflow/nodes/IfNode.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import React, { memo } from "react"; -import { type NodeProps, Position } from "reactflow"; - -import AbstractNode, { NodeTitle } from "./AbstractNode"; -import { getNodeBlockDefinitions } from "../../../services/workflow/node-block-definitions"; -import type { WorkflowNode } from "../../../types/workflow"; - -function IfNode(props: NodeProps) { - const { data, selected } = props; - const definition = getNodeBlockDefinitions().find((d) => d.type === data.block.type); - - return ( - - - - ); -} - -export default memo(IfNode); diff --git a/next/src/components/workflow/nodes/TriggerNode.tsx b/next/src/components/workflow/nodes/TriggerNode.tsx deleted file mode 100644 index 0fc18c0456..0000000000 --- a/next/src/components/workflow/nodes/TriggerNode.tsx +++ /dev/null @@ -1,56 +0,0 @@ -import clsx from "clsx"; -import { useSession } from "next-auth/react"; -import React, { memo, useState } from "react"; -import { type NodeProps, Position } from "reactflow"; - -import AbstractNode, { NodeTitle } from "./AbstractNode"; -import { getNodeBlockDefinitions } from "../../../services/workflow/node-block-definitions"; -import WorkflowApi from "../../../services/workflow/workflowApi"; -import { useConfigStore } from "../../../stores/configStore"; -import { useWorkflowStore } from "../../../stores/workflowStore"; -import type { WorkflowNode } from "../../../types/workflow"; -import Button from "../../../ui/button"; - -function TriggerNode({ data, selected }: NodeProps) { - const { data: session } = useSession(); - const workflow = useWorkflowStore().workflow; - const { organization: org, setLayout } = useConfigStore(); - const api = new WorkflowApi(session?.accessToken, org?.id); - const [loading, setLoading] = useState(false); - - const definition = getNodeBlockDefinitions().find((d) => d.type === data.block.type); - - const handleButtonClick = async () => { - if (!workflow) return; - - setLoading(true); - await api.execute(workflow.id); - setLayout({ showLogSidebar: true }); - setLayout({ showRightSidebar: false }); - setTimeout(() => { - setLoading(false); - }, 2000); // Set the duration of the loader in milliseconds (2 seconds in this example) - }; - - return ( - - - - - ); -} - -export default memo(TriggerNode); diff --git a/next/src/components/workflow/nodes/index.ts b/next/src/components/workflow/nodes/index.ts deleted file mode 100644 index 1f8f7cc7bb..0000000000 --- a/next/src/components/workflow/nodes/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { default as TriggerNode } from "./TriggerNode"; -export { default as IfNode } from "./IfNode"; -export { default as BasicNode } from "./BasicNode"; diff --git a/next/src/hooks/useSocket.ts b/next/src/hooks/useSocket.ts deleted file mode 100644 index b6da33d5f8..0000000000 --- a/next/src/hooks/useSocket.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* eslint-disable react-hooks/exhaustive-deps */ -import Pusher from "pusher-js"; -import { useEffect, useState } from "react"; -import { z } from "zod"; - -import { env } from "../env/client.mjs"; - -const PresenceInfoSchema = z.object({ - name: z.string().nullish(), - email: z.string().nullish(), - image: z.string().nullish(), -}); - -const PresenceSubscriptionSucceededSchema = z.object({ - count: z.number(), - me: z.object({ - id: z.string(), - info: PresenceInfoSchema, - }), - members: z.record(PresenceInfoSchema), -}); - -const PresenceMemberEventSchema = z.object({ - id: z.string(), - info: PresenceInfoSchema, -}); - -type PresenceInfo = z.infer; - -export default function useSocket( - channelName: string | undefined, - accessToken: string | undefined, - callbacks: { - event: string; - callback: (data: unknown) => Promise | void; - }[], - options?: { - enabled?: boolean; - } -) { - const [members, setMembers] = useState>({}); - - useEffect(() => { - const app_key = env.NEXT_PUBLIC_PUSHER_APP_KEY; - if (!app_key || !accessToken || !channelName) return () => void 0; - - const pusher = new Pusher(app_key, { - cluster: "mt1", - channelAuthorization: { - transport: "ajax", - endpoint: `${env.NEXT_PUBLIC_BACKEND_URL}/api/auth/pusher`, - headers: { - Authorization: `Bearer ${accessToken || ""}`, - }, - }, - }); - - const channel = pusher.subscribe("presence-" + channelName); - callbacks.map(({ event, callback }) => { - channel.bind(event, async (data) => { - await callback(data); - }); - }); - - channel.bind("pusher:subscription_succeeded", async (data) => { - const event = await PresenceSubscriptionSucceededSchema.parseAsync(data); - setMembers(event.members); - }); - - channel.bind("pusher:member_added", async (data) => { - const event = await PresenceMemberEventSchema.parseAsync(data); - - setMembers((prev) => ({ - ...prev, - [event.id]: event.info, - })); - }); - - channel.bind("pusher:member_removed", async (data) => { - const event = await PresenceMemberEventSchema.parseAsync(data); - setMembers(({ [event.id]: _, ...rest }) => rest); - }); - - return () => { - pusher.unsubscribe(channel.name); - pusher.disconnect(); - setMembers({}); - }; - }, [accessToken, channelName]); - - return members; -} diff --git a/next/src/hooks/useWorkflow.ts b/next/src/hooks/useWorkflow.ts deleted file mode 100644 index 09f6d2af44..0000000000 --- a/next/src/hooks/useWorkflow.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { useMutation, useQuery } from "@tanstack/react-query"; -import { nanoid } from "nanoid"; -import type { Session } from "next-auth"; -import { useEffect, useState } from "react"; -import type { Edge, Node } from "reactflow"; -import { z } from "zod"; - -import useSocket from "./useSocket"; -import WorkflowApi from "../services/workflow/workflowApi"; -import { useWorkflowStore } from "../stores/workflowStore"; -import type { NodeBlock, Workflow, WorkflowEdge, WorkflowNode } from "../types/workflow"; -import { getNodeType, toReactFlowEdge, toReactFlowNode } from "../types/workflow"; - -const StatusEventSchema = z.object({ - nodeId: z.string(), - status: z.enum(["running", "success", "error"]), - remaining: z.number().optional(), -}); - -const SaveEventSchema = z.object({ - user_id: z.string(), -}); - -const updateNodeValue = < - DATA extends WorkflowNode, - KEY extends keyof DATA, - T extends DATA extends WorkflowEdge ? Edge : Node ->( - currentNodes: Node[], - setNodes: (nodes: Node[]) => void, - key: KEY, - value: DATA[KEY], - filter: (node?: T["data"]) => boolean = () => true -) => { - const updatedNodes = currentNodes.map((t: Node) => { - if (filter(t.data)) { - return { - ...t, - data: { - ...t.data, - [key]: value, - }, - }; - } - return t; - }); - - setNodes(updatedNodes); -}; - -const updateEdgeValue = < - DATA extends WorkflowEdge | WorkflowNode, - KEY extends keyof DATA, - T extends DATA extends WorkflowEdge ? Edge : Node ->( - currentEdges: Edge[], - setEdges: (edges: Edge[]) => void, - key: KEY, - value: DATA[KEY], - filter: (edge?: T["data"]) => boolean = () => true -) => { - const updatedEdges = currentEdges.map((t: Edge) => { - if (filter(t.data)) { - return { - ...t, - data: { - ...t.data, - [key]: value, - }, - }; - } - return t; - }) as Edge[]; - - setEdges(updatedEdges); -}; - -export const useWorkflow = ( - workflowId: string | undefined, - session: Session | null, - organizationId: string | undefined, - onLog?: (log: LogType) => void -) => { - const api = new WorkflowApi(session?.accessToken, organizationId); - const [selectedNode, setSelectedNode] = useState | undefined>(undefined); - - const { mutateAsync: updateWorkflow } = useMutation(async (data: Workflow) => { - if (!workflowId) return; - await api.update(workflowId, data); - }); - - const { - workflow, - setInputs, - updateWorkflow: updateWorkflowStore, - setWorkflow, - setNodes, - setEdges, - getNodes, - getEdges, - } = useWorkflowStore(); - - const nodesModel = { - get: getNodes, - set: setNodes, - }; - - const edgesModel = { - get: getEdges, - set: setEdges, - }; - - const { refetch: refetchWorkflow, isLoading } = useQuery( - ["workflow", workflowId], - async () => { - if (!workflowId) { - setNodes([]); - setEdges([]); - return; - } - - const workflow = await api.get(workflowId); - setWorkflow({ - id: workflow.id, - nodes: workflow.nodes.map(toReactFlowNode), - edges: workflow.edges.map(toReactFlowEdge), - }); - - return workflow; - }, - { - enabled: !!workflowId && !!session?.accessToken, - refetchOnWindowFocus: false, - refetchOnMount: false, - } - ); - - useEffect(() => { - if (!workflow?.nodes) return; // Early exit if nodes are not available - - const selectedNodes = workflow.nodes.filter((n) => n.selected); - - if (selectedNodes.length === 0) { - setSelectedNode(undefined); - } else { - setSelectedNode(selectedNodes[0]); - } - }, [workflow?.nodes]); - - const members = useSocket( - workflowId, - session?.accessToken, - [ - { - event: "workflow:node:status", - callback: async (data) => { - const { nodeId, status, remaining } = await StatusEventSchema.parseAsync(data); - - updateNodeValue(getNodes() ?? [], setNodes, "status", status, (n) => n?.id === nodeId); - updateEdgeValue(getEdges() ?? [], setEdges, "status", status, (e) => e?.id === nodeId); - - if (status === "error" || remaining === 0) { - setTimeout(() => { - updateNodeValue(getNodes() ?? [], setNodes, "status", undefined); - updateEdgeValue(getEdges() ?? [], setEdges, "status", undefined); - }, 1000); - } - }, - }, - { - event: "workflow:updated", - callback: async (data) => { - const { user_id } = await SaveEventSchema.parseAsync(data); - if (user_id !== session?.user?.id) await refetchWorkflow(); - }, - }, - { - event: "workflow:log", - callback: async (data) => { - const log = await LogSchema.parseAsync(data); - onLog?.({ ...log, date: log.date.substring(11, 19) }); - }, - }, - ], - { - enabled: !!workflowId && !!session?.accessToken, - } - ); - - const createNode: createNodeType = (block: NodeBlock, position: Position) => { - const ref = nanoid(11); - const node = { - id: ref, - type: getNodeType(block), - position, - data: { - id: undefined, - ref: ref, - pos_x: 0, - pos_y: 0, - block: block, - }, - }; - - const newNodes = [...(getNodes() ?? []), node]; - - setNodes(newNodes); - - return node; - }; - - const updateNode: updateNodeType = (nodeToUpdate: Node) => { - const nodes = (getNodes() ?? []).map((node) => { - if (node.id === nodeToUpdate.id) { - return { - ...node, - data: { - ...node.data, - ...nodeToUpdate.data, - }, - }; - } - return node; - }); - - updateWorkflowStore({ - nodes, - }); - }; - - const onSave = async () => { - if (!workflowId) return; - - const nodes = getNodes() ?? []; - const edges = getEdges() ?? []; - - await updateWorkflow({ - id: workflowId, - nodes: nodes.map((n) => ({ - id: n.data.id, - ref: n.data.ref, - pos_x: n.position.x, - pos_y: n.position.y, - block: n.data.block, - })), - edges: edges.map((e) => ({ - id: e.id, - source: e.source, - source_handle: e.sourceHandle || undefined, - target: e.target, - })), - }); - }; - - const onExecute = async () => { - if (!workflowId) return; - await api.execute(workflowId); - }; - - return { - nodesModel, - edgesModel, - selectedNode, - saveWorkflow: onSave, - executeWorkflow: onExecute, - createNode, - updateNode, - members, - isLoading, - }; -}; - -const LogSchema = z.object({ - // level: z.enum(["info", "error"]), - date: z.string().refine((date) => date.substring(0, 19)), // Get rid of milliseconds - msg: z.string(), -}); - -export type LogType = z.infer; -export type Position = { x: number; y: number }; -export type createNodeType = (block: NodeBlock, position: Position) => Node; -export type updateNodeType = (node: Node) => void; diff --git a/next/src/hooks/useWorkflows.ts b/next/src/hooks/useWorkflows.ts deleted file mode 100644 index b866e85a7e..0000000000 --- a/next/src/hooks/useWorkflows.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; - -import type { WorkflowMeta } from "../services/workflow/workflowApi"; -import WorkflowApi from "../services/workflow/workflowApi"; - -export default function useWorkflows(accessToken?: string, organizationId?: string) { - const api = new WorkflowApi(accessToken, organizationId); - - const queryClient = useQueryClient(); - const { data: workflows, refetch: refetchWorkflows } = useQuery( - ["workflows"], - async () => await api.getAll(), - { - enabled: !!accessToken && !!organizationId, - } - ); - - // Optimistic update when creating a workflow, we don't need to fetch again - const { mutateAsync: createWorkflow } = useMutation( - async (data: Omit) => { - const workflow = await api.create(data); - queryClient.setQueriesData(["workflows"], (oldData: WorkflowMeta[] | undefined) => [ - ...(oldData || []), - workflow, - ]); - return workflow; - } - ); - - const { mutateAsync: deleteWorkflow } = useMutation(async (id: string) => { - await api.delete(id); - queryClient.setQueriesData(["workflows"], (oldData: WorkflowMeta[] | undefined) => - (oldData || []).filter((workflow) => workflow.id !== id) - ); - }); - - return { - workflows, - refetchWorkflows, - createWorkflow, - deleteWorkflow, - }; -} diff --git a/next/src/pages/organization.tsx b/next/src/pages/organization.tsx deleted file mode 100644 index 4659fe0c49..0000000000 --- a/next/src/pages/organization.tsx +++ /dev/null @@ -1,67 +0,0 @@ -import DashboardLayout from "../layout/dashboard"; -import type { GetStaticProps } from "next"; -import { serverSideTranslations } from "next-i18next/serverSideTranslations"; -import React from "react"; -import nextI18NextConfig from "../../next-i18next.config.js"; -import { languages } from "../utils/languages"; -import PrimaryButton from "../components/PrimaryButton"; -import Input from "../ui/input"; - -const OrganizationManagement = () => { - return ( - -
-
-
-

- Organization -

- - Manage your Organization settings - -
-
-

Logo

-
- - Pick a logo for your Oraganization - -
-
-
-

General

-
- -
-
- -
- Update -
-
-
-
- - ); -}; - -export default OrganizationManagement; - -export const getStaticProps: GetStaticProps = async ({ locale = "en" }) => { - const supportedLocales = languages.map((language) => language.code); - const chosenLocale = supportedLocales.includes(locale) ? locale : "en"; - - return { - props: { - ...(await serverSideTranslations(chosenLocale, nextI18NextConfig.ns)), - }, - }; -}; diff --git a/next/src/pages/workflow/index.tsx b/next/src/pages/workflow/index.tsx deleted file mode 100644 index 020232bbc0..0000000000 --- a/next/src/pages/workflow/index.tsx +++ /dev/null @@ -1,434 +0,0 @@ -import clsx from "clsx"; -import { AnimatePresence, motion } from "framer-motion"; -import type { GetStaticProps } from "next"; -import { type NextPage } from "next"; -import Image from "next/image"; -import { useRouter } from "next/router"; -import { serverSideTranslations } from "next-i18next/serverSideTranslations"; -import React, { useState } from "react"; -import { RiBuildingLine, RiStackFill } from "react-icons/ri"; -import { RxHome, RxPlus, RxTrash } from "react-icons/rx"; -import type { Connection, OnConnectStartParams } from "reactflow"; -import { addEdge } from "reactflow"; - -import nextI18NextConfig from "../../../next-i18next.config"; -import WorkflowSidebar from "../../components/drawer/WorkflowSidebar"; -import Loader from "../../components/loader"; -import FadeIn from "../../components/motions/FadeIn"; -import BlockDialog from "../../components/workflow/BlockDialog"; -import FlowChart from "../../components/workflow/Flowchart"; -import { useAuth } from "../../hooks/useAuth"; -import type { Position } from "../../hooks/useWorkflow"; -import { useWorkflow } from "../../hooks/useWorkflow"; -import useWorkflows from "../../hooks/useWorkflows"; -import { getNodeBlockDefinitions } from "../../services/workflow/node-block-definitions"; -import { useConfigStore } from "../../stores/configStore"; -import Select from "../../ui/select"; -import { languages } from "../../utils/languages"; -import { get_avatar } from "../../utils/user"; -import LogSidebar from "../../components/drawer/LogSidebar"; - -const isTypeError = (error: unknown): error is TypeError => - error instanceof Error && error.name === "TypeError"; - -const isError = (error: unknown): error is Error => - error instanceof Error && error.name === "Error"; - -const WorkflowPage: NextPage = () => { - const { organization, setOrganization, layout, setLayout } = useConfigStore(); - const { session } = useAuth({ - protectedRoute: true, - }); - const router = useRouter(); - const [newNodePosition, setNewNodePosition] = useState({ x: 0, y: 0 }); - const [onConnectStartParams, setOnConnectStartParams] = useState< - OnConnectStartParams | undefined - >(undefined); - const handleSaveWorkflow = async () => { - try { - await saveWorkflow(); - window.alert("Workflow saved successfully!"); - } catch (error: unknown) { - if (isTypeError(error) && error.message === "Failed to fetch") - window.alert( - "An error occurred while saving the workflow. Please refresh and re-attempt to save." - ); - else if (isError(error) && error.message === "Unprocessable Entity") - window.alert("Invalid workflow. Make sure to clear unconnected nodes and remove cycles."); - else window.alert("An error occurred while saving the workflow."); - } - }; - - async function reset() { - await changeQueryParams({ w: undefined }); - nodesModel.set([]); - edgesModel.set([]); - } - - const handlePlusClick = async () => { - try { - await reset(); - await saveWorkflow(); - } catch (error: unknown) { - window.alert("An error occurred while creating a new workflow."); - } - }; - - const workflowId = router.query.w as string | undefined; - const { - nodesModel, - edgesModel, - selectedNode, - saveWorkflow, - createNode, - updateNode, - members, - isLoading, - } = useWorkflow(workflowId, session, organization?.id); - - const [open, setOpen] = useState(false); - - const handlePaneDoubleClick = (position: { x: number; y: number }) => { - if (!showCreateForm) { - setNewNodePosition(position); - setOpen(true); - } - }; - - const changeQueryParams = async (newParams: Record) => { - let updatedParams = { - ...router.query, - ...newParams, - }; - - updatedParams = Object.entries(updatedParams).reduce((acc, [key, value]) => { - if (!!value) acc[key] = value; - return acc; - }, {}); - - const newURL = { - pathname: router.pathname, - query: updatedParams, - }; - - await router.replace(newURL, undefined, { shallow: true }); - }; - - const showLoader = !router.isReady || (isLoading && !!workflowId); - const showCreateForm = !workflowId && router.isReady; - - const { workflows, createWorkflow, deleteWorkflow, refetchWorkflows } = useWorkflows( - session?.accessToken, - organization?.id - ); - - const onCreate = async (name: string) => { - const data = await createWorkflow({ name, description: "" }); - await changeQueryParams({ w: data.id }); - }; - - const changeOrg = async (org: { id: string; name: string; role: string } | undefined) => { - if (org === organization) return; - setOrganization(org); - await reset(); - await refetchWorkflows(); - }; - - const handleDeleteClick = async () => { - try { - if (!workflowId) return; - await reset(); - await deleteWorkflow(workflowId); - } catch (error: unknown) { - window.alert("An error occurred while deleting the workflow."); - } - }; - - return ( - <> - { - const node = createNode( - { - type: t.type, - input: {}, - }, - newNodePosition - ); - - if (onConnectStartParams) { - const { nodeId, handleId } = onConnectStartParams; - if (!nodeId) return; - const edge: Connection = { - source: nodeId, - target: node.id, - sourceHandle: handleId, - targetHandle: null, - }; - - const x = addEdge(edge, edgesModel.get() ?? []); - edgesModel.set(x); - } - }} - /> - -
-
- void router.push("/home")} - > - Reworkd AI - - void router.push("/")} - > - - - {router.isReady && ( - <> - - value={organization} - items={session?.user.organizations} - valueMapper={(org) => org?.name} - icon={RiBuildingLine} - defaultValue={ - session?.user.organizations?.find((o) => { - return o.id === organization?.id; - }) || { - id: "default", - name: "Select an org", - role: "member", - } - } - onChange={changeOrg} - /> - - value={workflows?.find((w) => w.id === router.query.w)} - onChange={async (e) => { - if (e) await changeQueryParams({ w: e.id }); - }} - items={workflows} - valueMapper={(item) => item?.name} - icon={RiStackFill} - defaultValue={ - workflows?.find((w) => w.id === workflowId) || { - id: "default", - name: "Select a workflow", - } - } - /> - - )} - {showCreateForm || ( - - - - )} -
-
-
- {showCreateForm || ( - { - setLayout({ - showLogSidebar: !layout.showLogSidebar, - }); - }} - /> - )} -
-
- -
- -
-
- -
- - {showCreateForm && ( -
- -
- )} - - {showLoader && ( -
- -
- )} - - - {!showLoader && !showCreateForm && !nodesModel.get() && ( - - Double Click on the canvas to add a node - - )} - - -
- -
- - ); -}; - -export default WorkflowPage; - -export const getStaticProps: GetStaticProps = async ({ locale = "en" }) => { - const supportedLocales = languages.map((language) => language.code); - const chosenLocale = supportedLocales.includes(locale) ? locale : "en"; - - return { - props: { - ...(await serverSideTranslations(chosenLocale, nextI18NextConfig.ns)), - }, - }; -}; - -const CreateWorkflow = ({ onSubmit }: { onSubmit: (name: string) => Promise }) => { - const [workflowName, setWorkflowName] = useState(""); - - function submit() { - if (!workflowName) { - window.alert("Please enter a workflow name."); - return; - } - - onSubmit(workflowName).catch(console.error); - } - - return ( -
-
-

Create a new workflow

-
-

Enter a short yet descriptive name for your workflow.

-
-
-
- - setWorkflowName(e.target.value)} - /> -
- -
-
-
- ); -}; - -interface AccountBarProps { - editors: Record< - string, - { - name?: string | null; - email?: string | null; - image?: string | null; - } - >; - onSave: () => Promise; - onShowLogs: () => void; -} - -function AccountBar(props: AccountBarProps) { - const editors = Object.entries(props.editors); - - return ( -
- <> -
- {editors.length === 0 && ( -
- )} - {editors.map(([id, user]) => ( - 1 && "-mr-2 first:ml-0" - )} - key={id} - src={get_avatar(user)} - alt="user avatar" - /> - ))} -
-
- - - -
- ); -} diff --git a/next/src/services/workflow/node-block-definitions.ts b/next/src/services/workflow/node-block-definitions.ts deleted file mode 100644 index d672f6cfdb..0000000000 --- a/next/src/services/workflow/node-block-definitions.ts +++ /dev/null @@ -1,411 +0,0 @@ -import type { IconType } from "react-icons"; -import { - FaBolt, - FaBook, - FaCodeBranch, - FaCopy, - FaFileUpload, - FaGlobeAmericas, - FaPlay, - FaRobot, - FaSlack, - FaTerminal, -} from "react-icons/fa"; -import type { Node } from "reactflow"; -import { z } from "zod"; - -import type { WorkflowNode } from "../../types/workflow"; - -const IOFieldSchema = z.object({ - name: z.string(), - description: z.string().optional(), - type: z.enum(["string", "array", "enum", "file", "oauth", "button"]), - items: z.object({ type: z.string() }).optional(), - enum: z.array(z.string()).optional(), -}); - -export type IOField = z.infer; - -export const NodeBlockDefinitionSchema = z.object({ - name: z.string(), - type: z.string(), - description: z.string(), - image_url: z.string(), - input_fields: z.array(IOFieldSchema), - output_fields: z.array(IOFieldSchema), - icon: z.custom(), - color: z.string().optional(), -}); - -export type NodeBlockDefinition = z.infer; - -const colorTypes = { - trigger: "bg-purple-500", - output: "bg-green-500", - agent: "bg-blue-500", -}; - -const UrlStatusCheckBlockDefinition: NodeBlockDefinition = { - name: "URL Status Check", - type: "UrlStatusCheck", - description: "Check if a website exists", - image_url: "/tools/web.png", - icon: FaGlobeAmericas, - input_fields: [ - { - name: "url", - description: "The URL to check", - type: "string", - }, - ], - output_fields: [ - { - name: "url", - description: "The URL to check", - type: "string", - }, - { - name: "code", - description: "The HTTP status code", - type: "string", - }, - ], -}; - -const SlackWebhookBlockDefinition: NodeBlockDefinition = { - name: "Slack Message", - type: "SlackWebhook", - description: "Sends a message to a slack webhook", - image_url: "/tools/web.png", - icon: FaSlack, - color: colorTypes.output, - input_fields: [ - { - name: "url", - description: "The Slack WebHook URL", - type: "oauth", - }, - { - name: "message", - description: "The message to send", - type: "string", - }, - ], - output_fields: [ - { - name: "message", - description: "The message that was sent", - type: "string", - }, - ], -}; - -const CompanyContextAgentBlockDefinition: NodeBlockDefinition = { - name: "Company Context Agent", - type: "CompanyContextAgent", - description: "Retrieve market, industry, and product summary of a specific company", - image_url: "/tools/web.png", - icon: FaCopy, - color: colorTypes.agent, - input_fields: [ - { - name: "company_name", - description: "enter name of company", - type: "string", - }, - ], - output_fields: [ - { - name: "result", - description: "The result was built.", - type: "string", - }, - ], -}; - -const GenericLLMAgentBlockDefinition: NodeBlockDefinition = { - name: "Generic LLM Agent", - type: "GenericLLMAgent", - description: "OpenAI agent", - image_url: "/tools/web.png", - icon: FaCopy, - color: colorTypes.agent, - input_fields: [ - { - name: "prompt", - description: "Enter a prompt", - type: "string", - }, - ], - output_fields: [ - { - name: "result", - description: "The result was built.", - type: "string", - }, - ], -}; - -const SummaryAgentBlockDefinition: NodeBlockDefinition = { - name: "Summary Agent", - type: "SummaryAgent", - description: - "Summarize and extract key market insights for specific companies and industries from documents", - image_url: "/tools/web.png", - icon: FaCopy, - color: colorTypes.agent, - input_fields: [ - { - name: "chat", - description: "chat with your PDF", - type: "button", - }, - { - name: "company_context", - description: "short description on company, market, and their core products", - type: "string", - }, - ], - output_fields: [ - { - name: "result", - description: "The result was built.", - type: "string", - }, - ], -}; - -const TextInputWebhookBlockDefinition: NodeBlockDefinition = { - name: "Text Input", - type: "TextInputWebhook", - description: "", - image_url: "/tools/web.png", - icon: FaTerminal, - input_fields: [ - { - name: "text", - description: "Enter text", - type: "string", - }, - ], - output_fields: [ - { - name: "result", - description: "The result was built.", - type: "string", - }, - ], -}; - -const UploadDocBlockDefinition: NodeBlockDefinition = { - name: "Upload Doc", - type: "UploadDoc", - description: "Securely upload a .docx to Amazon S3", - image_url: "/tools/web.png", - icon: FaBook, - input_fields: [ - { - name: "text", - description: "The text to upload", - type: "string", - }, - ], - output_fields: [ - { - name: "file_url", - description: "The URL to access the doc", - type: "string", - }, - ], -}; - -const DiffDocBlockDefinition: NodeBlockDefinition = { - name: "Diff Doc", - type: "DiffDoc", - description: - "Create a document that will display the diff between an original and updated string", - image_url: "/tools/web.png", - icon: FaBook, - input_fields: [ - { - name: "original", - description: "The original version of the text", - type: "string", - }, - { - name: "updated", - description: "The updated version of the text", - type: "string", - }, - ], - output_fields: [ - { - name: "file_url", - description: "The URL to access the diff PDF.", - type: "string", - }, - ], -}; - -const IfBlockDefinition: NodeBlockDefinition = { - name: "If Condition", - type: "IfCondition", - description: "Conditionally take a path", - image_url: "/tools/web.png", - icon: FaCodeBranch, - input_fields: [ - { - name: "value_one", - type: "string", - }, - { - name: "operator", - description: "The type of equality to check for", - type: "string", - enum: ["==", "!="], - }, - { - name: "value_two", - type: "string", - }, - ], - output_fields: [ - { - name: "result", - description: "The result of the condition", - type: "string", - }, - ], -}; - -const APITriggerBlockDefinition: NodeBlockDefinition = { - name: "APITrigger", - type: "APITriggerBlock", - description: "Trigger a workflow through an API call.", - icon: FaBolt, - color: colorTypes.trigger, - image_url: "/tools/web.png", - input_fields: [], - output_fields: [ - { - name: "message", - description: "Input string to the API call", - type: "string", - }, - ], -}; - -const ManualTriggerBlockDefinition: NodeBlockDefinition = { - name: "Manual Trigger", - type: "ManualTriggerBlock", - description: "Trigger a block manually", - icon: FaPlay, - color: colorTypes.trigger, - image_url: "/tools/web.png", - input_fields: [], - output_fields: [], -}; - -const WebInteractionAgentBlockDefinition: NodeBlockDefinition = { - name: "Web Interaction Agent", - type: "WebInteractionAgent", - description: "Dynamically interact with a website", - image_url: "/tools/web.png", - icon: FaRobot, - color: colorTypes.agent, - input_fields: [ - { - name: "url", - description: "The website the agent will interact with", - type: "string", - }, - { - name: "goals", - description: "The actions the agent should take on the site", - type: "string", - }, - ], - output_fields: [], -}; - -const FileUploadBlockDefinition: NodeBlockDefinition = { - name: "File Upload", - type: "FileUploadBlock", - description: "Upload a file", - icon: FaFileUpload, - image_url: "/tools/web.png", - input_fields: [ - { - name: "file", - description: "The file to upload", - type: "file", - }, - ], - output_fields: [], -}; - -const ContentRefresherAgent: NodeBlockDefinition = { - name: "Content Refresher Agent", - type: "ContentRefresherAgent", - description: "Refresh the content on an existing page", - image_url: "/tools/web.png", - icon: FaRobot, - color: colorTypes.agent, - input_fields: [ - { - name: "url", - description: "The page whose content the agent will refresh", - type: "string", - }, - { - name: "competitors", - description: "List of comma-separated competitors you don't want to pull content from", - type: "string", - }, - { - name: "keywords", - description: "List of comma-separated keywords you'd like to pull content from. If you enter less than 3, we'll generate keywords for you.", - type: "string", - }, - ], - output_fields: [ - { - name: "original_report", - description: "The original report to be refreshed", - type: "string", - }, - { - name: "refreshed_report", - description: "The refreshed report with new content added", - type: "string", - }, - { - name: "refreshed_bullet_points", - description: "Relevant new information not present in source report", - type: "string", - }, - ], -}; - -export const getNodeBlockDefinitions = (): NodeBlockDefinition[] => { - return [ - ManualTriggerBlockDefinition, - APITriggerBlockDefinition, - SlackWebhookBlockDefinition, - DiffDocBlockDefinition, - UploadDocBlockDefinition, - TextInputWebhookBlockDefinition, - FileUploadBlockDefinition, - IfBlockDefinition, - UrlStatusCheckBlockDefinition, - WebInteractionAgentBlockDefinition, - ContentRefresherAgent, - GenericLLMAgentBlockDefinition, - SummaryAgentBlockDefinition, - CompanyContextAgentBlockDefinition, - ]; -}; - -export const getNodeBlockDefinitionFromNode = (node: Node) => { - return getNodeBlockDefinitions().find((d) => d.type === node.data.block.type); -}; diff --git a/next/src/services/workflow/oauthApi.ts b/next/src/services/workflow/oauthApi.ts deleted file mode 100644 index e2cbc88d0b..0000000000 --- a/next/src/services/workflow/oauthApi.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { Session } from "next-auth"; -import { z } from "zod"; - -import { env } from "../../env/client.mjs"; -import { get, post } from "../fetch-utils"; - -export default class OauthApi { - readonly accessToken?: string; - readonly organizationId?: string; - - constructor(accessToken?: string, organizationId?: string) { - this.accessToken = accessToken; - this.organizationId = organizationId; - } - - static fromSession(session: Session | null) { - return new OauthApi(session?.accessToken, session?.user?.organizations[0]?.id); - } - - async install(provider: string, redirectUri?: string) { - const url = `${env.NEXT_PUBLIC_VERCEL_URL}${redirectUri || ""}`; - - return await get( - `/api/auth/${provider}?redirect=${encodeURIComponent(url)}`, - z.string().url(), - this.accessToken, - this.organizationId - ); - } - - async uninstall(provider: string) { - return await get( - `/api/auth/${provider}/uninstall`, - z.object({ - success: z.boolean(), - }), - this.accessToken, - this.organizationId - ); - } - // TODO: decouple this - async get_info(provider: string) { - return await get( - `/api/auth/${provider}/info`, - z - .object({ - name: z.string(), - id: z.string(), - }) - .array(), - this.accessToken, - this.organizationId - ); - } - - async get_info_sid() { - return await get( - `/api/auth/sid/info`, - z.object({ - connected: z.boolean(), - }), - this.accessToken, - this.organizationId - ); - } -} diff --git a/next/src/services/workflow/workflowApi.ts b/next/src/services/workflow/workflowApi.ts deleted file mode 100644 index 7d480f74f3..0000000000 --- a/next/src/services/workflow/workflowApi.ts +++ /dev/null @@ -1,126 +0,0 @@ -import axios from "axios"; -import { z } from "zod"; - -import type { Workflow } from "../../types/workflow"; -import { WorkflowSchema } from "../../types/workflow"; -import { delete_, get, post, put } from "../fetch-utils"; - -const WorkflowMetaSchema = z.object({ - id: z.string(), - name: z.string(), - description: z.string(), - user_id: z.string(), - organization_id: z.string().nullable(), -}); - -const PresignedPostSchema = z.object({ - url: z.string(), - fields: z.record(z.string()), -}); - -export type WorkflowMeta = z.infer; -export type PresignedPost = z.infer; - -export default class WorkflowApi { - readonly accessToken?: string; - readonly organizationId?: string; - - constructor(accessToken?: string, organizationId?: string) { - this.accessToken = accessToken; - this.organizationId = organizationId; - } - - async getAll() { - return await get( - "/api/workflow", - z.array(WorkflowMetaSchema), - this.accessToken, - this.organizationId - ); - } - - async get(id: string) { - return await get(`/api/workflow/${id}`, WorkflowSchema, this.accessToken, this.organizationId); - } - - async update(id: string, data: Workflow) { - await put(`/api/workflow/${id}`, z.any(), data, this.accessToken, this.organizationId); - } - - async delete(id: string) { - await delete_(`/api/workflow/${id}`, z.any(), {}, this.accessToken, this.organizationId); - } - - async create(workflow: Omit) { - return await post( - "/api/workflow", - WorkflowMetaSchema, - workflow, - this.accessToken, - this.organizationId - ); - } - - async execute(id: string) { - return await post( - `/api/workflow/${id}/execute`, - z.string(), - {}, - this.accessToken, - this.organizationId - ); - } - - async upload(workflow_id: string, block_ref: string, files: File[]) { - const posts = await put( - `/api/workflow/${workflow_id}/block/${block_ref}/upload`, - z.record(PresignedPostSchema), - { files: files.map((file) => file.name) }, - this.accessToken, - this.organizationId - ); - - await Promise.all( - // @ts-ignore - Object.entries(posts).map(([filename, post], i) => this.uploadFile(post, files[i])) - ); - } - - async uploadFile(req: PresignedPost, file: File) { - const { url, fields } = req; - - const formData = new FormData(); - Object.entries(fields).forEach(([key, value]) => { - formData.append(key, value); - }); - formData.append("file", file); - - const uploadResponse = await axios.post(url, formData, { - headers: { - "Content-Type": file.type, - }, - }); - - return uploadResponse.status; - } - - async blockInfo(workflow_id: string, block_ref: string) { - return await get( - `/api/workflow/${workflow_id}/block/${block_ref}`, - z.object({ - files: z.array(z.string()), - }), - this.accessToken, - this.organizationId - ); - } - - async blockInfoDelete(workflow_id: string, block_ref: string) { - await delete_( - `/api/workflow/${workflow_id}/block/${block_ref}`, - z.any(), - this.accessToken, - this.organizationId - ); - } -} diff --git a/next/src/stores/workflowStore.ts b/next/src/stores/workflowStore.ts deleted file mode 100644 index 8a752f12f7..0000000000 --- a/next/src/stores/workflowStore.ts +++ /dev/null @@ -1,125 +0,0 @@ -import type { Edge as ReactFlowEdge, Node as ReactFlowNode } from "reactflow"; -import { create } from "zustand"; - -import { createSelectors } from "./helpers"; -import type { WorkflowEdge, WorkflowNode } from "../types/workflow"; - -interface Workflow { - id: string; - nodes: ReactFlowNode[]; - edges: ReactFlowEdge[]; -} - -interface Input { - field: string; - value: string; -} - -const initialState = { - workflow: null, -}; - -type Store = { - workflow: Workflow | null; - setWorkflow: (workflow: Workflow) => void; - updateWorkflow: (workflow: Partial) => void; - setInputs: ( - workflow: Workflow, - nodeToUpdate: ReactFlowNode, - updatedInput: Input - ) => void; - setNodes: (nodes: ReactFlowNode[]) => void; - setEdges: (edges: ReactFlowEdge[]) => void; - getNodes: () => ReactFlowNode[] | null; - getEdges: () => ReactFlowEdge[] | null; -}; - -export const useWorkflowStore = createSelectors( - create((set, get) => ({ - ...initialState, - - setWorkflow: (workflow: Workflow) => { - set({ workflow }); - }, - - setInputs: ( - workflow: Workflow, - nodeToUpdate: ReactFlowNode, - updatedInput: Input - ) => { - set({ - workflow: { - ...workflow, - nodes: workflow.nodes.map((node) => { - if (node.data.id === nodeToUpdate.data.id) { - if (node.data.block.input) { - const updatedInputs = Object.keys(node.data.block.input).reduce( - (acc, field) => { - if (field === updatedInput.field) { - acc[field] = updatedInput.value; - } - return acc; - }, - { ...node.data.block.input } - ); - return { - ...node, - block: { - ...node.data.block, - input: updatedInputs, - }, - }; - } - } - return node; - }), - }, - }); - }, - - updateWorkflow: (workflow: Partial) => { - const currentWorkflow = get().workflow; - if (currentWorkflow) { - set({ - workflow: { - ...currentWorkflow, - ...workflow, - }, - }); - } - }, - - setNodes: (nodes: ReactFlowNode[]) => { - const currentWorkflow = get().workflow; - if (currentWorkflow) { - set({ - workflow: { - ...currentWorkflow, - id: currentWorkflow?.id, - nodes, - }, - }); - } - }, - setEdges: (edges: ReactFlowEdge[]) => { - const currentWorkflow = get().workflow; - if (currentWorkflow) { - set({ - workflow: { - ...currentWorkflow, - id: currentWorkflow.id, - edges, - }, - }); - } - }, - getNodes: () => { - const currentWorkflow = get().workflow; - return currentWorkflow ? currentWorkflow.nodes : null; - }, - getEdges: () => { - const currentWorkflow = get().workflow; - return currentWorkflow ? currentWorkflow.edges : null; - }, - })) -); diff --git a/next/src/types/workflow.ts b/next/src/types/workflow.ts deleted file mode 100644 index bd7d6d7171..0000000000 --- a/next/src/types/workflow.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { Dispatch, SetStateAction } from "react"; -import type { Edge, Node } from "reactflow"; -import { z } from "zod"; - -const NodeBlockSchema = z.object({ - type: z.string(), - input: z.record(z.string()), -}); - -export type NodeBlock = z.infer; - -type Model = [T, Dispatch>]; - -const WorkflowNodeSchema = z.object({ - id: z.string().optional(), - ref: z.string(), - pos_x: z.number(), - pos_y: z.number(), - status: z.enum(["running", "success", "error"]).optional(), - block: NodeBlockSchema, -}); - -const WorkflowEdgeSchema = z.object({ - id: z.string(), - source: z.string(), - source_handle: z.string().optional().nullable(), - target: z.string(), - status: z.enum(["running", "success", "error"]).optional(), -}); -export const WorkflowSchema = z.object({ - id: z.string(), - nodes: z.array(WorkflowNodeSchema), - edges: z.array(WorkflowEdgeSchema), -}); - -export type WorkflowNode = z.infer; -export type WorkflowEdge = z.infer; -export type Workflow = z.infer; - -export type NodesModel = { - get: () => Node[] | null; - set: (nodes: Node[]) => void; -}; - -export type EdgesModel = { - get: () => Edge[] | null; - set: (edges: Edge[]) => void; -}; - -export const toReactFlowNode = (node: WorkflowNode) => - ({ - id: node.id ?? node.ref, - data: node, - position: { x: node.pos_x, y: node.pos_y }, - type: getNodeType(node.block), - } as Node); - -export const toReactFlowEdge = (edge: WorkflowEdge) => - ({ - ...edge, - sourceHandle: edge.source_handle, - type: "custom", - data: { - ...edge, - }, - } as Edge); - -export const getNodeType = (block: NodeBlock) => { - switch (block.type) { - case "ManualTriggerBlock": - return "trigger"; - case "IfCondition": - return "if"; - default: - return "custom"; - } -}; diff --git a/next/src/ui/InputWithSuggestions.tsx b/next/src/ui/InputWithSuggestions.tsx deleted file mode 100644 index 5b89ade12d..0000000000 --- a/next/src/ui/InputWithSuggestions.tsx +++ /dev/null @@ -1,82 +0,0 @@ -import { Menu as MenuPrimitive } from "@headlessui/react"; -import React from "react"; -import type { Node } from "reactflow"; - -import Input from "./input"; -import { MenuItems } from "../components/Menu"; -import WindowButton from "../components/WindowButton"; -import { useWorkflowStore } from "../stores/workflowStore"; -import type { WorkflowNode } from "../types/workflow"; - -interface Props extends React.InputHTMLAttributes { - label: string; - name: string; - attributes?: { [key: string]: string | number | string[] }; - helpText?: string | React.ReactNode; - icon?: React.ReactNode; - disabled?: boolean; - right?: React.ReactNode; - suggestions: { key: string; value: string }[]; - value: string; - currentNode: Node | undefined; -} - -interface Field { - value?: string; - key?: string; -} - -const InputWithSuggestions = (props: Props) => { - const [focused, setFocused] = React.useState(false); - const { workflow, setInputs } = useWorkflowStore(); - const handleClick = (field: Field, label: string) => () => { - const eventMock = { - target: { - value: `${field.key as string}`, - }, - }; - - if (workflow && props.currentNode) { - setInputs(workflow, props.currentNode, { - field: label, - value: `${field.key as string}`, - }); - } - props.onChange && props.onChange(eventMock as React.ChangeEvent); - }; - - return ( - <> - { - if (focus) setFocused(true); - }} - /> - {props.suggestions.length > 0 && focused && ( - -
- ( - } - text={field.value} - onClick={handleClick(field, props.label)} - /> - ))} - /> -
- - )} - - ); -}; - -export default InputWithSuggestions; diff --git a/next/src/ui/WorkflowSidebarInput.tsx b/next/src/ui/WorkflowSidebarInput.tsx deleted file mode 100644 index 09f6b6e4cc..0000000000 --- a/next/src/ui/WorkflowSidebarInput.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import React, {useState } from "react"; -import type { Node } from "reactflow"; - -import Combo from "./combox"; -import Dropzone from "./dropzone"; -import InputWithSuggestions from "./InputWithSuggestions"; -import OauthIntegration from "./OauthIntegration"; -import type { IOField } from "../services/workflow/node-block-definitions"; -import type { WorkflowNode } from "../types/workflow"; -import Button from "./button"; -import WorkflowChatDialog from "../components/workflow/WorkflowChatDialog"; - -interface SidebarInputProps { - inputField: IOField; - onChange: (value: string) => void; - suggestions: { key: string; value: string }[]; - node: Node | undefined; -} - - -const WorkflowSidebarInput = ({ inputField, onChange, suggestions, node }: SidebarInputProps) => { - const [showChatDialog, setShowChatDialog] = useState(false); - - if (inputField.type === "string" && inputField.enum) { - return ( - e} - onChange={(e) => onChange(e)} - /> - ); - } - if (inputField.type === "string") { - return ( - <> - onChange(e.target.value)} - suggestions={suggestions} - currentNode={node} - /> - - ); - } - if (inputField.type === "file") { - return ( - { - onChange(e.target.value); - }} - node_ref={node?.data.ref} - /> - ); - } - if (inputField.type === "oauth") { - return ( - - ); - } - if (inputField.type === "button") { - return ( - <> - - - {showChatDialog && ( - - )} - - ); - } - return <>; -}; - -export default WorkflowSidebarInput; diff --git a/next/src/ui/dropzone.tsx b/next/src/ui/dropzone.tsx deleted file mode 100644 index ba563b5391..0000000000 --- a/next/src/ui/dropzone.tsx +++ /dev/null @@ -1,110 +0,0 @@ -import { useMutation, useQuery } from "@tanstack/react-query"; -import { useSession } from "next-auth/react"; -import type { InputHTMLAttributes, ReactNode } from "react"; -import React, { useState } from "react"; -import { FaCloudUploadAlt } from "react-icons/fa"; - -import Button from "./button"; -import WorkflowApi from "../services/workflow/workflowApi"; -import { useConfigStore } from "../stores/configStore"; -import { useWorkflowStore } from "../stores/workflowStore"; - -interface Props extends InputHTMLAttributes { - label: string; - name?: string; - helpText?: string | React.ReactNode; - icon?: ReactNode; - node_ref?: string | undefined; -} - -const Dropzone = (props: Props) => { - const { data: session } = useSession(); - const [files, setFiles] = useState([]); - const workflow = useWorkflowStore.getState(); - const orgId = useConfigStore().organization?.id; - - const { mutateAsync: uploadFiles } = useMutation(async (files: File[]) => { - if (!files.length || !workflow?.workflow?.id || !props.node_ref) return; - await new WorkflowApi(session?.accessToken, orgId).upload( - workflow.workflow.id, - props.node_ref, - files - ); - }); - - const { data: s3_files, refetch } = useQuery([undefined], () => { - if (!workflow?.workflow?.id || !props.node_ref) return; - return new WorkflowApi(session?.accessToken, orgId).blockInfo( - workflow.workflow.id, - props.node_ref - ); - }); - - const { mutateAsync: deleteFiles } = useMutation(async () => { - if (!workflow?.workflow?.id || !props.node_ref) return; - await new WorkflowApi(session?.accessToken, orgId).blockInfoDelete( - workflow.workflow.id, - props.node_ref - ); - setFiles([]); - await refetch(); - }); - - const filenames = [...(s3_files?.files || []), ...files.map((file) => file.name)]; - - return ( -
- {props.label && ( - - )} - {props.helpText && ( -

{props.helpText}

- )} - - {filenames.map((file, i) => ( -
{file}
- ))} - {filenames.length ? ( - - ) : ( - No files uploaded - )} -
- ); -}; - -export default Dropzone; diff --git a/platform/reworkd_platform/db/crud/workflow.py b/platform/reworkd_platform/db/crud/workflow.py deleted file mode 100644 index c625d89a91..0000000000 --- a/platform/reworkd_platform/db/crud/workflow.py +++ /dev/null @@ -1,159 +0,0 @@ -import asyncio -from typing import Dict, List - -from fastapi import Depends -from sqlalchemy import and_, select, or_ -from sqlalchemy.ext.asyncio import AsyncSession - -from reworkd_platform.db.crud.base import BaseCrud -from reworkd_platform.db.crud.edge import EdgeCRUD -from reworkd_platform.db.crud.node import NodeCRUD -from reworkd_platform.db.dependencies import get_db_session -from reworkd_platform.db.models.workflow import WorkflowModel -from reworkd_platform.schemas.user import UserBase -from reworkd_platform.schemas.workflow.base import ( - Workflow, - WorkflowFull, - WorkflowUpdate, - WorkflowCreate, -) -from reworkd_platform.web.api.dependencies import get_current_user -from reworkd_platform.web.api.http_responses import forbidden - - -class WorkflowCRUD(BaseCrud): - def __init__(self, session: AsyncSession, user: UserBase): - super().__init__(session) - self.user = user - self.node_service = NodeCRUD(session) - self.edge_service = EdgeCRUD(session) - - @staticmethod - def inject( - session: AsyncSession = Depends(get_db_session), - user: UserBase = Depends(get_current_user), - ) -> "WorkflowCRUD": - return WorkflowCRUD(session, user) - - async def get_all(self) -> List[Workflow]: - query = ( - select(WorkflowModel) - .where( - or_( - and_( - WorkflowModel.user_id == self.user.id, - WorkflowModel.organization_id.is_(None), - WorkflowModel.delete_date.is_(None), - ), - and_( - WorkflowModel.organization_id == self.user.organization_id, - WorkflowModel.organization_id.is_not(None), - WorkflowModel.delete_date.is_(None), - ), - ) - ) - .order_by(WorkflowModel.create_date.desc()) - ) - - return [ - workflow.to_schema() - for workflow in (await self.session.execute(query)).scalars().all() - ] - - async def get(self, workflow_id: str) -> WorkflowFull: - # fetch workflow, nodes, and edges concurrently - workflow, nodes, edges = await asyncio.gather( - WorkflowModel.get_or_404(self.session, workflow_id), - self.node_service.get_nodes(workflow_id), - self.edge_service.get_edges(workflow_id), - ) - - self.validate_permissions(workflow) - - # get node blocks - blocks = await self.node_service.get_node_blocks( - [node.id for node in nodes.values()] - ) - - node_block_pairs = [(node, blocks.get(node.id)) for node in nodes.values()] - - return WorkflowFull( - **workflow.to_schema().dict(), - nodes=[node.to_schema(block) for node, block in node_block_pairs if block], - edges=[edge.to_schema() for edge in edges.values()], - ) - - async def create(self, workflow_create: WorkflowCreate) -> Workflow: - return ( - await WorkflowModel( - user_id=self.user.id, - organization_id=self.user.organization_id, - name=workflow_create.name, - description=workflow_create.description, - ).save(self.session) - ).to_schema() - - async def delete(self, workflow_id: str) -> None: - """Soft a delete workflow""" - # TODO: Make sure workflow logic doesnt run on deleted workflows - workflow = await WorkflowModel.get_or_404(self.session, workflow_id) - - self.validate_permissions(workflow) - await workflow.delete(self.session) - - async def update(self, workflow_id: str, workflow_update: WorkflowUpdate) -> str: - workflow, all_nodes, all_edges, all_blocks = await asyncio.gather( - WorkflowModel.get_or_404(self.session, workflow_id), - self.node_service.get_nodes(workflow_id), - self.edge_service.get_edges(workflow_id), - self.node_service.get_node_blocks( - [node.id for node in workflow_update.nodes if node.id] - ), - ) - - self.validate_permissions(workflow) - - # TODO: use co-routines to make this faster - # Map from ref to id so edges can use id's instead of refs - ref_to_id: Dict[str, str] = {} - for n in workflow_update.nodes: - if not n.id: - node = await self.node_service.create_node_with_block(n, workflow_id) - elif n.id not in all_nodes: - raise Exception("Node not found") - else: - node = await self.node_service.update_node_with_block( - n, all_nodes[n.id], all_blocks[n.id] - ) - ref_to_id[node.ref] = node.id - - # Delete nodes - await self.node_service.mark_old_nodes_deleted(workflow_update.nodes, all_nodes) - - # Mark edges as added - for edge_model in all_edges.values(): - self.edge_service.add_edge(edge_model) - - # Modify the edges' source and target to their corresponding IDs - for e in workflow_update.edges: - e.source = ref_to_id.get(e.source, e.source) - e.target = ref_to_id.get(e.target, e.target) - - # update edges - for e in workflow_update.edges: - if self.edge_service.add_edge(e): - await self.edge_service.create_edge(e, workflow_id) - - # Delete edges - await self.edge_service.delete_old_edges(workflow_update.edges, all_edges) - - return "OK" - - def validate_permissions(self, workflow: WorkflowModel) -> None: - if ( - workflow.user_id == self.user.id - or workflow.organization_id == self.user.organization_id - ): - return - - raise forbidden() diff --git a/platform/reworkd_platform/db/models/workflow.py b/platform/reworkd_platform/db/models/workflow.py deleted file mode 100644 index 033044274a..0000000000 --- a/platform/reworkd_platform/db/models/workflow.py +++ /dev/null @@ -1,74 +0,0 @@ -from sqlalchemy import JSON, Float, ForeignKey, String -from sqlalchemy.orm import mapped_column, relationship - -from reworkd_platform.db.base import Base, TrackedModel, UserMixin -from reworkd_platform.schemas.workflow.base import Block, Edge, Node, Workflow - - -class WorkflowModel(TrackedModel, UserMixin): - __tablename__ = "workflow" - - name = mapped_column(String, nullable=False) - description = mapped_column(String, nullable=False) - - nodes = relationship("WorkflowNodeModel", back_populates="workflow", uselist=True) - edges = relationship("WorkflowEdgeModel", back_populates="workflow", uselist=True) - - def to_schema(self) -> Workflow: - return Workflow( - id=str(self.id), - user_id=self.user_id, - organization_id=self.organization_id, - name=self.name, - description=self.description, - ) - - -class WorkflowNodeModel(TrackedModel): - __tablename__ = "workflow_node" - ref = mapped_column(String, nullable=False) - workflow_id = mapped_column(String, ForeignKey("workflow.id")) - - pos_x = mapped_column(Float) - pos_y = mapped_column(Float) - - workflow = relationship("WorkflowModel", back_populates="nodes") - - def to_schema(self, block: "NodeBlockModel") -> Node: - return Node( - id=self.id, - ref=self.ref, - pos_x=self.pos_x, - pos_y=self.pos_y, - block=block.to_schema(), - ) - - -class WorkflowEdgeModel(Base): - __tablename__ = "workflow_edge" - workflow_id = mapped_column(String, ForeignKey("workflow.id")) - - source = mapped_column(String) - source_handle = mapped_column(String, nullable=True) - target = mapped_column(String) - - workflow = relationship("WorkflowModel", back_populates="edges") - - def to_schema(self) -> Edge: - return Edge( - id=self.id, - source=self.source, - source_handle=self.source_handle, - target=self.target, - ) - - -class NodeBlockModel(Base): - __tablename__ = "node_block" - node_id = mapped_column(String, ForeignKey("workflow_node.id")) - - type = mapped_column(String) - input = mapped_column(JSON) - - def to_schema(self) -> Block: - return Block(id=self.id, type=self.type, input=self.input) diff --git a/platform/reworkd_platform/schemas/workflow/__init__.py b/platform/reworkd_platform/schemas/workflow/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/base.py b/platform/reworkd_platform/schemas/workflow/base.py deleted file mode 100644 index 6a7101eee1..0000000000 --- a/platform/reworkd_platform/schemas/workflow/base.py +++ /dev/null @@ -1,97 +0,0 @@ -from typing import List, Optional, Any - -from networkx import DiGraph -from pydantic import BaseModel, Field - - -class BlockIOBase(BaseModel): - """ - Base input/output type inherited by all blocks. Allows for overrides - """ - - class Config: - extra = "allow" - - -class BlockUpsert(BaseModel): - id: Optional[str] - type: str - input: BlockIOBase - - -class Block(BaseModel): - id: str - type: str - input: BlockIOBase - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - raise NotImplementedError("Base workflow Node class must be inherited") - - -class EdgeUpsert(BaseModel): - id: Optional[str] - source: str - source_handle: Optional[str] - target: str - - -class NodeUpsert(BaseModel): - id: Optional[str] - ref: str = Field(description="Reference ID generated by the frontend") - pos_x: float - pos_y: float - block: BlockUpsert - - -class Node(BaseModel): - id: str - ref: str - pos_x: float - pos_y: float - block: Block - - -class Edge(BaseModel): - id: str - source: str - source_handle: Optional[str] - target: str - - -class WorkflowCreate(BaseModel): - name: str - description: str - - -class Workflow(BaseModel): - id: str - user_id: str - organization_id: Optional[str] = Field(default=None) - name: str - description: str - - -# noinspection DuplicatedCode -class WorkflowUpdate(BaseModel): - nodes: List[NodeUpsert] - edges: List[EdgeUpsert] - - def to_graph(self) -> DiGraph: - graph = DiGraph() - graph.add_nodes_from([v.id or v.ref for v in self.nodes]) - graph.add_edges_from([(e.source, e.target) for e in self.edges]) - - return graph - - -# noinspection DuplicatedCode -class WorkflowFull(Workflow): - nodes: List[Node] - edges: List[Edge] - - def to_graph(self) -> DiGraph: - graph = DiGraph() - graph.add_nodes_from([v.id for v in self.nodes]) - graph.add_edges_from([(e.source, e.target) for e in self.edges]) - - return graph diff --git a/platform/reworkd_platform/schemas/workflow/blocks/__init__.py b/platform/reworkd_platform/schemas/workflow/blocks/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/blocks/agents/__init__.py b/platform/reworkd_platform/schemas/workflow/blocks/agents/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/blocks/agents/company_context_agent.py b/platform/reworkd_platform/schemas/workflow/blocks/agents/company_context_agent.py deleted file mode 100644 index b0e77d6d69..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/agents/company_context_agent.py +++ /dev/null @@ -1,58 +0,0 @@ -from typing import Any - -import openai -from loguru import logger - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.settings import settings - - -class CompanyContextAgentInput(BlockIOBase): - company_name: str - - -class CompanyContextAgentOutput(CompanyContextAgentInput): - result: str - - -class CompanyContextAgent(Block): - type = "OpenAIAgent" - description = "Extract key details from text using OpenAI" - input: CompanyContextAgentInput - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - try: - response = await execute_prompt(company=self.input.company_name) - - except Exception as err: - logger.error(f"Failed to extract text with OpenAI: {err}") - raise - - return CompanyContextAgentOutput(**self.input.dict(), result=response) - - -async def execute_prompt(company: str) -> str: - openai.api_key = settings.openai_api_key - - prompt = f""" - Write a one-sentence description of "{company}". - Define their market, sector, and primary products. - - Be as clear, informative, and descriptive as necessary. - You will not make up information or add any information outside of the above text. - Only use the given information and nothing more. - """ - - response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": prompt}], - temperature=1, - max_tokens=500, - top_p=1, - frequency_penalty=0, - presence_penalty=0, - ) - - response_message_content = response["choices"][0]["message"]["content"] - - return response_message_content diff --git a/platform/reworkd_platform/schemas/workflow/blocks/agents/content_refresher_agent.py b/platform/reworkd_platform/schemas/workflow/blocks/agents/content_refresher_agent.py deleted file mode 100644 index a1c6bd4ef6..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/agents/content_refresher_agent.py +++ /dev/null @@ -1,307 +0,0 @@ -import re -from typing import Any, Callable, Dict, List, Optional, Tuple - -import requests -from bs4 import BeautifulSoup -from loguru import logger -from scrapingbee import ScrapingBeeClient - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.services.anthropic import HumanAssistantPrompt, ClaudeService -from reworkd_platform.services.serp import SerpService -from reworkd_platform.services.sockets import websockets -from reworkd_platform.settings import settings, Settings - - -class ContentRefresherInput(BlockIOBase): - url: str - competitors: Optional[str] = None - keywords: Optional[str] = None - - -class ContentRefresherOutput(ContentRefresherInput): - original_report: str - refreshed_report: str - refreshed_bullet_points: str - - -class ContentRefresherAgent(Block): - type = "ContentRefresherAgent" - description = "Refresh the content on an existing page" - input: ContentRefresherInput - - async def run(self, workflow_id: str, **kwargs: Any) -> ContentRefresherOutput: - def log(msg: Any) -> None: - websockets.log(workflow_id, msg) - - logger.info(f"Starting {self.type}") - log(f"Starting {self.type} for {self.input.url}") - return await ContentRefresherService(settings, log).refresh(self.input) - - -class ContentRefresherService: - def __init__(self, settings: Settings, log: Callable[[str], None]): - self.log = log - self.claude = ClaudeService(api_key=settings.anthropic_api_key) - self.scraper = ScrapingBeeClient(api_key=settings.scrapingbee_api_key) - self.serp = SerpService(api_key=settings.serp_api_key) - - async def refresh(self, input_: ContentRefresherInput) -> ContentRefresherOutput: - target_url = input_.url - - target_content = await self.get_page_content(target_url) - self.log("Extracting content from provided URL") - - keywords = self.parse_input_keywords(input_.keywords) - self.log("Parsing keywords from input") - - if len(keywords) < 3: - additional_keywords = await self.find_content_kws(target_content, keywords) - keywords.extend(additional_keywords) - self.log("Finding more keywords from source content") - self.log("Keywords: " + ", ".join(keywords)) - - sources = self.search_results(", ".join(keywords)) - sources = [ - source for source in sources if source["url"] != target_url - ] # TODO: check based on content overlap - - self.log("Finding sources to refresh content") - self.log( - "\n".join([f"- {source['title']}: {source['url']}" for source in sources]) - ) - - if input_.competitors: - self.log("Removing competitors from sources") - competitors = input_.competitors.split(",") - sources = self.remove_competitors(sources, competitors, self.log) - - domain = self.extract_domain(target_url) - if domain: - sources = [source for source in sources if domain not in source["url"]] - - self.log(f"Omitting sources from target's domain: {domain}") - - for source in sources[:3]: # TODO: remove limit of 3 sources - source["content"] = await self.get_page_content(source["url"]) - - source_contents = [ - source for source in sources if source.get("content", None) is not None - ] - - new_info = [ - await self.find_new_info(target_content, source_content, self.log) - for source_content in source_contents - ] - - new_infos = "\n\n".join(new_info) - self.log("Extracting new, relevant information not present in your content") - for info in new_info: - self.log(info) - - self.log("Updating provided content with new information") - updated_target_content = await self.add_info(target_content, new_infos) - self.log("Content refresh concluded") - - return ContentRefresherOutput( - **input_.dict(), - original_report=target_content, - refreshed_report=updated_target_content, - refreshed_bullet_points=new_infos, - ) - - async def get_page_content(self, url: str) -> str: - page = requests.get(url) - if page.status_code != 200: - page = self.scraper.get(url) - - html = BeautifulSoup(page.content, "html.parser") - pgraphs = self.get_article_from_html(html) - - prompt = HumanAssistantPrompt( - human_prompt=f"Below is a numbered list of the text in all the

and

  • tags on a web page: {pgraphs} Within this list, some lines may not be relevant to the primary content of the page (e.g. footer text, advertisements, etc.). Please identify the range of line numbers that correspond to the main article's content (i.e. article's paragraphs). Your response should only mention the range of line numbers, for example: 'lines 5-25'.", - assistant_prompt="Given the extracted text, the main content's line numbers are:", - ) - - line_nums = await self.claude.completion( - prompt=prompt, - max_tokens_to_sample=500, - temperature=0, - ) - - if len(line_nums) == 0: - return "" - - content = self.extract_content_from_line_nums(pgraphs, line_nums) - return "\n".join(content) - - async def find_content_kws( - self, content: str, input_keywords: List[str] - ) -> List[str]: - # Claude: find search keywords that content focuses on - num_keywords_to_find = 8 - len(input_keywords) - prompt = HumanAssistantPrompt( - human_prompt=f"Below is content from a web article:\n{content}\nPlease list {num_keywords_to_find} keywords that best describe the content of the article. Separate each keyword (or phrase) with commas so we can use them to query a search engine effectively. e.g. 'gardasil lawsuits, gardasil side effects, autoimmune, ovarian.' Here are the existing keywords, choose different ones than the following: {', '.join(input_keywords)}", - assistant_prompt="Here is a short search query that best matches the content of the article:", - ) - - response = await self.claude.completion( - prompt=prompt, - max_tokens_to_sample=20, - ) - keywords_list = [keyword.strip() for keyword in response.split(",")] - - return keywords_list - - def search_results(self, search_query: str) -> List[Dict[str, str]]: - source_information = [ - { - "url": result.get("link", None), - "title": result.get("title", None), - "date": result.get("date", None), - } - for result in self.serp.search(search_query).get("organic", []) - ] - return source_information - - async def find_new_info( - self, target: str, source: Dict[str, str], log: Callable[[str], None] - ) -> str: - source_metadata = f"{source['url']}, {source['title']}" + ( - f", {source['date']}" if source["date"] else "" - ) - source_content = source["content"] - - # Claude: info mentioned in source that is not mentioned in target - prompt = HumanAssistantPrompt( - human_prompt=f"Below is the TARGET article:\n{target}\n----------------\nBelow is the SOURCE article:\n{source_content}\n----------------\nIn a bullet point list, identify all facts, figures, or ideas that are mentioned in the SOURCE article but not in the TARGET article.", - assistant_prompt="Here is a list of claims in the SOURCE that are not in the TARGET:", - ) - log(f"Identifying new details to refresh with from '{source['title']}'") - - response = await self.claude.completion( - prompt=prompt, - max_tokens_to_sample=5000, - ) - - new_info = "\n".join(response.split("\n\n")) - new_info += "\n\nSource: " + source_metadata - return new_info - - async def add_info(self, target: str, info: str) -> str: - # Claude: rewrite target to include the info - prompt = HumanAssistantPrompt( - human_prompt=f"Below is the TARGET article:\n{target}\n----------------\nBelow are notes from SOURCE articles that are not currently present in the TARGET:\n{info}\n----------------\nPlease rewrite the TARGET article to include unique, new information from the SOURCE articles. The format of the article you write should follow the TARGET article. The goal is to add as many relevant details from SOURCE to TARGET. Don't remove any details from the TARGET article, unless you are refreshing that specific content with new information. After any new source info that is added to target, include inline citations using the following example format: 'So this is a cited sentence at the end of a paragraph[1](https://www.wisnerbaum.com/prescription-drugs/gardasil-lawsuit/, Gardasil Vaccine Lawsuit Update August 2023 - Wisner Baum).' Do not cite info that already existed in the TARGET article. Do not list citations separately at the end of the response", - assistant_prompt="Here is a rewritten version of the target article that incorporates relevant information from the source articles:", - ) - - response = await self.claude.completion( - prompt=prompt, - max_tokens_to_sample=5000, - ) - - response = "\n".join( - [ - paragraph.strip() - for paragraph in response.split("\n\n") - if paragraph.strip() - ] - ) - return response - - def extract_content_from_line_nums(self, pgraphs: str, line_nums: str) -> List[str]: - pgraph_elements = pgraphs.split("\n") - content = [] - for line_num in line_nums.split(","): - if "-" in line_num: - start, end = self.extract_initial_line_numbers(line_num) - if start and end: - for i in range(start, min(end + 1, len(pgraph_elements) + 1)): - text = ".".join(pgraph_elements[i - 1].split(".")[1:]).strip() - content.append(text) - elif line_num.isdigit(): - text = ".".join( - pgraph_elements[int(line_num) - 1].split(".")[1:] - ).strip() - content.append(text) - return content - - @staticmethod - def extract_domain(url: str) -> Optional[str]: - if "." not in url: - return None - - domain_extraction_pattern = ( - r"^(?:https?://)?(?:[^/]+\.)?([^/]+\.[A-Za-z_0-9.-]+).*" - ) - match = re.search(domain_extraction_pattern, url) - if match: - return match.group(1) - else: - return None - - @staticmethod - def remove_competitors( - sources: List[Dict[str, str]], - competitors: List[str], - log: Callable[[str], None], - ) -> List[Dict[str, str]]: - normalized_competitors = [comp.replace(" ", "").lower() for comp in competitors] - competitor_pattern = re.compile( - "|".join(re.escape(comp) for comp in normalized_competitors) - ) - filtered_sources = [] - for source in sources: - if competitor_pattern.search( - source["url"].replace(" ", "").lower() - ) or competitor_pattern.search(source["title"].replace(" ", "").lower()): - log(f"Removing competitive source: '{source['title']}'") - else: - filtered_sources.append(source) - - return filtered_sources - - @staticmethod - def extract_initial_line_numbers( - line_nums: str, - ) -> Tuple[Optional[int], Optional[int]]: - match = re.search(r"(\d+)-(\d+)", line_nums) - if match: - return int(match.group(1)), int(match.group(2)) - else: - return None, None - - @staticmethod - def get_article_from_html(html: BeautifulSoup) -> str: - elements = [] - processed_lists = set() - for p in html.find_all("p"): - elements.append(p) - next_sibling = p.find_next_sibling(["ul", "ol"]) - if next_sibling and next_sibling not in processed_lists: - elements.extend(next_sibling.find_all("li")) - processed_lists.add(next_sibling) - - if not elements: - return "No

    or

  • tags found on page" - - formatted_elements = [] - for i, element in enumerate(elements): - text = re.sub(r"\s+", " ", element.text).strip() - prefix = f"{i + 1}. " - if element.name == "li": - prefix += "• " - formatted_elements.append(f"{prefix}{text}") - - return "\n".join(formatted_elements) - - @staticmethod - def parse_input_keywords(input_keywords: Optional[str]) -> List[str]: - if not input_keywords: - return [] - - keywords = [ - keyword.strip() for keyword in input_keywords.split(",") if keyword.strip() - ] - - return keywords diff --git a/platform/reworkd_platform/schemas/workflow/blocks/agents/generic_llm_agent.py b/platform/reworkd_platform/schemas/workflow/blocks/agents/generic_llm_agent.py deleted file mode 100644 index e1bce9eeef..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/agents/generic_llm_agent.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Any - -import openai -from loguru import logger - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.settings import settings - - -class GenericLLMAgentInput(BlockIOBase): - prompt: str - - -class GenericLLMAgentOutput(GenericLLMAgentInput): - result: str - - -class GenericLLMAgent(Block): - type = "GenericLLMAgent" - description = "Extract key details from text using OpenAI" - input: GenericLLMAgentInput - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - try: - response = await execute_prompt(prompt=self.input.prompt) - - except Exception as err: - logger.error(f"Failed to extract text with OpenAI: {err}") - raise - - return GenericLLMAgentOutput(**self.input.dict(), result=response) - - -async def execute_prompt(prompt: str) -> str: - openai.api_key = settings.openai_api_key - - response = await openai.ChatCompletion.acreate( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": prompt}], - temperature=1, - max_tokens=500, - top_p=1, - frequency_penalty=0, - presence_penalty=0, - ) - - response_message_content = response["choices"][0]["message"]["content"] - logger.info(f"response = {response_message_content}") - return response_message_content diff --git a/platform/reworkd_platform/schemas/workflow/blocks/agents/summary_agent.py b/platform/reworkd_platform/schemas/workflow/blocks/agents/summary_agent.py deleted file mode 100644 index 7a98033625..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/agents/summary_agent.py +++ /dev/null @@ -1,181 +0,0 @@ -import os -import tempfile -from collections import defaultdict -from typing import Any, List, Dict, Union -from loguru import logger - -import openai -import pinecone -from langchain.chains.question_answering import load_qa_chain -from langchain.document_loaders import PyPDFLoader -from langchain.embeddings import OpenAIEmbeddings -from langchain.embeddings.base import Embeddings -from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain.vectorstores import Pinecone -from tabula.io import read_pdf - -from reworkd_platform.schemas.agent import ModelSettings -from reworkd_platform.schemas.user import UserBase -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.services.aws.s3 import SimpleStorageService -from reworkd_platform.settings import settings -from reworkd_platform.web.api.agent.model_factory import create_model - - -class SummaryAgentInput(BlockIOBase): - company_context: str - - -class SummaryAgentOutput(SummaryAgentInput): - result: str - - -class SummaryAgent(Block): - type = "SummaryAgent" - description = "Extract key details from text using OpenAI" - input: SummaryAgentInput - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - with tempfile.TemporaryDirectory() as temp_dir: - files = SimpleStorageService( - bucket=settings.s3_bucket_name - ).download_folder( - prefix=f"{workflow_id}/", - path=temp_dir, - ) - - docsearch = self.chunk_documents_to_pinecone( - files=files, - embeddings=( - OpenAIEmbeddings( - client=None, - # Meta private value but mypy will complain its missing - openai_api_key=settings.openai_api_key, - ) - ), - path=temp_dir, - workflow_id=workflow_id, - ) - - response = await self.execute_query_on_pinecone( - company_context=self.input.company_context, - docsearch=docsearch, - workflow_id=workflow_id, - path=temp_dir, - ) - - logger.info(f"SummaryAgent response: {response}") - return SummaryAgentOutput(**self.input.dict(), result=response) - - def name_table(self, table: str) -> str: - openai.api_key = settings.openai_api_key - - prompt = f""" - Write a title for the table that is less than 9 words: {table} - """ - - response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": prompt}], - temperature=1, - max_tokens=500, - top_p=1, - frequency_penalty=0, - presence_penalty=0, - ) - - response_message_content = response["choices"][0]["message"]["content"] - - return response_message_content - - def read_and_preprocess_tables( - self, relevant_table_metadata: dict[str, list[int]] - ) -> list[str]: - processed = [] - parsed_dfs_from_file: Union[List[Any], Dict[str, Any]] = [] - - for source in relevant_table_metadata.keys(): - page_numbers = relevant_table_metadata[source] - filtered_page_numbers = list(filter(lambda x: x != 0, page_numbers)) - if len(filtered_page_numbers) > 1: - filtered_page_numbers.sort() - start_page = filtered_page_numbers[0] - end_page = filtered_page_numbers[-1] - parsed_dfs_from_file = read_pdf( - source, pages=f"{start_page}-{end_page}" - ) - if isinstance(parsed_dfs_from_file, list): - for df in parsed_dfs_from_file: - if not df.empty: - df_name = self.name_table(str(df.iloc[:5])) - processed_df = "\n".join([df.to_csv(index=False)]) - processed_df_with_title = "\n".join([df_name, processed_df]) - processed.append(processed_df_with_title) - elif isinstance(parsed_dfs_from_file, dict): - for key, df in parsed_dfs_from_file.items(): - if not df.empty: - df_name = self.name_table(str(df.iloc[:5])) - processed_df = "\n".join([df.to_csv(index=False)]) - processed_df_with_title = "\n".join([df_name, processed_df]) - processed.append(processed_df_with_title) - else: - # Handle unexpected case - raise ValueError("Unexpected type encountered.") - - return processed - - def chunk_documents_to_pinecone( - self, files: list[str], embeddings: Embeddings, path: str, workflow_id: str - ) -> Pinecone: - index_name = "prod" - index = pinecone.Index(index_name) - index.delete(delete_all=True, namespace=workflow_id) - text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0) - - texts = [] - for file in files: - filepath = os.path.join(path, file) - data = PyPDFLoader(filepath).load() - pdf_data = data - texts.extend(text_splitter.split_documents(pdf_data)) - - docsearch = Pinecone.from_documents( - [t for t in texts], embeddings, index_name=index_name, namespace=workflow_id - ) - - return docsearch - - async def execute_query_on_pinecone( - self, company_context: str, docsearch: Pinecone, workflow_id: str, path: str - ) -> str: - similarity_prompt = f"""{company_context}. Include information relevant to the market, strategies, products, interesting quantitative metrics and trends that can inform decision-making. Gather info from different documents.""" - docs = docsearch.similarity_search( - similarity_prompt, k=7, namespace=workflow_id - ) - relevant_table_metadata: Dict[str, List[Any]] = defaultdict(list) - for doc in docs: - doc_source = doc.metadata["source"] - page_number = int(doc.metadata["page"]) - if page_number not in relevant_table_metadata[doc_source]: - relevant_table_metadata[doc_source].append(page_number) - - processed_tables = self.read_and_preprocess_tables(relevant_table_metadata) - - prompt = f"""Help extract information relevant to a company with the following details: {company_context} from the following documents. Start with the company background info. Then, include information relevant to the market, strategies, and products. Here are the documents: {docs}. After each point, reference the source you got the information from. - - Also list any interesting quantitative metrics or trends based on the following tables: {processed_tables}. Include which table you got information from. - - Cite sources for sentences using the page number from original source document. Do not list sources at the end of the writing. - - Example: "This is a cited sentence. (Source: Luxury Watch Market Size Report, Page 17). - - Format your response as slack markdown. - """ - - llm = create_model( - ModelSettings(model="gpt-3.5-turbo-16k", max_tokens=2000), - UserBase(id="", name=None, email="test@example.com"), - streaming=False, - ) - - return await load_qa_chain(llm).arun(input_documents=docs, question=prompt) diff --git a/platform/reworkd_platform/schemas/workflow/blocks/agents/web_interaction_agent.py b/platform/reworkd_platform/schemas/workflow/blocks/agents/web_interaction_agent.py deleted file mode 100644 index d388b52a37..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/agents/web_interaction_agent.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import Any - -from loguru import logger - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase - - -class WebInteractionInput(BlockIOBase): - url: str - goals: str - - -class WebInteractionOutput(WebInteractionInput): - successful: bool - - -class WebInteractionAgent(Block): - type = "WebInteractionAgent" - description = "Navigate a website" - input: WebInteractionInput - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - logger.info(f"Starting {self.type}") - - # Rohan 🙏 - - return WebInteractionOutput(**self.input.dict(), successful=True) diff --git a/platform/reworkd_platform/schemas/workflow/blocks/conditions/__init__.py b/platform/reworkd_platform/schemas/workflow/blocks/conditions/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/blocks/conditions/if_condition.py b/platform/reworkd_platform/schemas/workflow/blocks/conditions/if_condition.py deleted file mode 100644 index f80dcc6ac5..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/conditions/if_condition.py +++ /dev/null @@ -1,62 +0,0 @@ -from typing import Literal, TypeVar, Any - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase - -T = TypeVar("T", float, str) - - -class IfInput(BlockIOBase): - value_one: str - operator: Literal["==", "!=", "<", ">", "<=", ">="] - value_two: str - - -class IfOutput(BlockIOBase): - result: bool - - -class IfCondition(Block): - type = "IfCondition" - description = "Conditionally take a path" - input: IfInput - - async def run(self, workflow_id: str, **kwargs: Any) -> IfOutput: - value_one = self.input.value_one - value_two = self.input.value_two - operator = self.input.operator - - result = compare(value_one, operator, value_two) - - return IfOutput(result=result) - - -def compare(value_one: str, operator: str, value_two: str) -> bool: - if is_number(value_one) and is_number(value_two): - return perform_operation(float(value_one), operator, float(value_two)) - else: - return perform_operation(value_one, operator, value_two) - - -def perform_operation(value_one: T, operator: str, value_two: T) -> bool: - if operator == "==": - return value_one == value_two - elif operator == "!=": - return value_one != value_two - elif operator == "<": - return value_one < value_two - elif operator == ">": - return value_one > value_two - elif operator == "<=": - return value_one <= value_two - elif operator == ">=": - return value_one >= value_two - else: - raise ValueError(f"Invalid operator: {operator}") - - -def is_number(string: str) -> bool: - try: - float(string) - return True - except ValueError: - return False diff --git a/platform/reworkd_platform/schemas/workflow/blocks/do_nothing.py b/platform/reworkd_platform/schemas/workflow/blocks/do_nothing.py deleted file mode 100644 index 687da20613..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/do_nothing.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Any - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase - - -class DoNothingBlock(Block): - type = "DoNothing" - description = "Literally does nothing" - image_url = "" - input: BlockIOBase - - async def run(self, workflow_id: Any, **kwargs: Any) -> BlockIOBase: - return BlockIOBase() diff --git a/platform/reworkd_platform/schemas/workflow/blocks/pdf/__init__.py b/platform/reworkd_platform/schemas/workflow/blocks/pdf/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/blocks/pdf/diff_doc.py b/platform/reworkd_platform/schemas/workflow/blocks/pdf/diff_doc.py deleted file mode 100644 index e3ae0be066..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/pdf/diff_doc.py +++ /dev/null @@ -1,99 +0,0 @@ -import difflib -import io -from typing import Any, List - -from docx import Document -from docx.shared import RGBColor - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.services.aws.s3 import SimpleStorageService -from reworkd_platform.services.sockets import websockets -from reworkd_platform.settings import settings -from reworkd_platform.services.url_shortener import UrlShortenerService - - -class DiffDocInput(BlockIOBase): - original: str - updated: str - - -class DiffDocOutput(BlockIOBase): - file_url: str - - -class DiffDoc(Block): - type = "DiffDoc" - description = ( - "Create a Word document that shows the difference between two bodies of text" - ) - input: DiffDocInput - - async def run(self, workflow_id: str, **kwargs: Any) -> DiffDocOutput: - with io.BytesIO() as diff_doc_file: - websockets.log(workflow_id, "Creating diff of original and updated text") - diffs = get_diff(self.input.original, self.input.updated) - diff_doc_file = get_diff_doc(diffs, diff_doc_file) - - websockets.log(workflow_id, "Uploading diff doc to S3") - s3_service = SimpleStorageService(settings.s3_bucket_name) - s3_service.upload_to_bucket( - object_name=f"docs/{workflow_id}/{self.id}.docx", - file=diff_doc_file, - ) - - file_url = s3_service.create_presigned_download_url( - object_name=f"docs/{workflow_id}/{self.id}.docx", - ) - websockets.log(workflow_id, f"Diff Doc successfully uploaded to S3") - - shortener = UrlShortenerService() - tiny_url = await shortener.get_shortened_url(file_url) - websockets.log(workflow_id, f"Download the diff doc via: {tiny_url}") - - return DiffDocOutput(file_url=tiny_url) - - -def get_diff(original: str, updated: str) -> List[List[str]]: - original_paragraphs = original.split("\n") - updated_paragraphs = updated.split("\n") - - diffs = [] - for orig_par, updt_par in zip(original_paragraphs, updated_paragraphs): - differ = difflib.Differ() - words1 = orig_par.split() - words2 = updt_par.split() - - diffs.append(list(differ.compare(words1, words2))) - - return diffs - - -def get_diff_doc(diff_list: List[List[str]], in_memory_file: io.BytesIO) -> io.BytesIO: - """ - Create a Word document that shows the difference between two bodies of text. - Each element of diff_list is a list of strings of type " word", "- word", or "+ word". - """ - - doc = Document() - - for diff in diff_list: - paragraph = doc.add_paragraph() - paragraph.paragraph_format.space_after = 0 - - for word in diff: - if word.startswith(" "): - run = paragraph.add_run(word[2:] + " ") - run.font.color.rgb = RGBColor(0x00, 0x00, 0x00) # Black color - elif word.startswith("- "): - run = paragraph.add_run(word[2:] + " ") - run.font.color.rgb = RGBColor(0xFF, 0x00, 0x00) # Red color - elif word.startswith("+ "): - run = paragraph.add_run(word[2:] + " ") - run.font.color.rgb = RGBColor(0x00, 0x80, 0x00) # Green color - else: - continue - - doc.save(in_memory_file) - - in_memory_file.seek(0) - return in_memory_file diff --git a/platform/reworkd_platform/schemas/workflow/blocks/pdf/upload_doc.py b/platform/reworkd_platform/schemas/workflow/blocks/pdf/upload_doc.py deleted file mode 100644 index b30983b8c1..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/pdf/upload_doc.py +++ /dev/null @@ -1,48 +0,0 @@ -import io -from typing import Any - -from docx import Document - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.services.aws.s3 import SimpleStorageService -from reworkd_platform.services.sockets import websockets -from reworkd_platform.services.url_shortener import UrlShortenerService -from reworkd_platform.settings import settings - - -class UploadDocInput(BlockIOBase): - text: str - - -class UploadDocOutput(BlockIOBase): - file_url: str - - -class UploadDoc(Block): - type = "UploadDoc" - description = "Securely upload a .docx to Amazon S3" - input: UploadDocInput - - async def run(self, workflow_id: str, **kwargs: Any) -> UploadDocOutput: - with io.BytesIO() as pdf_file: - websockets.log(workflow_id, "Creating doc") - doc = Document() - doc.add_paragraph(self.input.text) - doc.save(pdf_file) - - websockets.log(workflow_id, "Uploading doc to S3") - s3_service = SimpleStorageService(settings.s3_bucket_name) - s3_service.upload_to_bucket( - object_name=f"docs/{workflow_id}/{self.id}.docx", - file=pdf_file, - ) - file_url = s3_service.create_presigned_download_url( - object_name=f"docs/{workflow_id}/{self.id}.docx", - ) - - websockets.log(workflow_id, f"Doc successfully uploaded to S3") - shortener = UrlShortenerService() - tiny_url = await shortener.get_shortened_url(file_url) - websockets.log(workflow_id, f"Download the doc via: {tiny_url}") - - return UploadDocOutput(file_url=tiny_url) diff --git a/platform/reworkd_platform/schemas/workflow/blocks/slack/__init__.py b/platform/reworkd_platform/schemas/workflow/blocks/slack/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/blocks/slack/slack_bot.py b/platform/reworkd_platform/schemas/workflow/blocks/slack/slack_bot.py deleted file mode 100644 index 5e35798024..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/slack/slack_bot.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Any, Optional, Dict - -from loguru import logger -from slack_sdk import WebClient - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.services.security import encryption_service - - -class SlackWebhookInput(BlockIOBase): - url: str - message: str - - -class SlackWebhookOutput(SlackWebhookInput): - url: str - message: str - - -class SlackMessageBlock(Block): - type = "SlackWebhook" - description = "Sends a message to a slack webhook" - input: SlackWebhookInput - - async def run( - self, - workflow_id: str, - credentials: Optional[Dict[str, str]] = None, - **kwargs: Any, - ) -> BlockIOBase: - logger.info(f"Starting {self.type} with {self.input.message}") - - if not credentials or not (token := credentials.get("slack", None)): - raise ValueError("No credentials provided") - - token = encryption_service.decrypt(token) - WebClient(token=token).chat_postMessage( - channel=self.input.url, - text=self.input.message, - ) - - return SlackWebhookOutput(**self.input.dict()) diff --git a/platform/reworkd_platform/schemas/workflow/blocks/text_input_webhook.py b/platform/reworkd_platform/schemas/workflow/blocks/text_input_webhook.py deleted file mode 100644 index 70f4bf637f..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/text_input_webhook.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Any - -from loguru import logger - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase - - -class TextInputWebhookInput(BlockIOBase): - text: str - - -class TextInputWebhookOutput(BlockIOBase): - result: str - - -class TextInputWebhook(Block): - type = "TextInputWebhook" - description = "Enter Text to extract key details from" - input: TextInputWebhookInput - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - logger.info(f"Starting {self.type}") - - return TextInputWebhookOutput(result=self.input.text) diff --git a/platform/reworkd_platform/schemas/workflow/blocks/triggers/__init__.py b/platform/reworkd_platform/schemas/workflow/blocks/triggers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platform/reworkd_platform/schemas/workflow/blocks/triggers/api_trigger.py b/platform/reworkd_platform/schemas/workflow/blocks/triggers/api_trigger.py deleted file mode 100644 index c29dfe76ff..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/triggers/api_trigger.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Any - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase - - -class APITriggerInput(BlockIOBase): - message: str - - -class APITriggerOutput(BlockIOBase): - message: str - - -class APITriggerBlock(Block): - type = "APITriggerBlock" - description = "Trigger the workflow through an API call" - image_url = "" - - async def run(self, workflow_id: str, **kwargs: Any) -> APITriggerOutput: - return APITriggerOutput(**self.input.dict()) diff --git a/platform/reworkd_platform/schemas/workflow/blocks/triggers/manual_trigger.py b/platform/reworkd_platform/schemas/workflow/blocks/triggers/manual_trigger.py deleted file mode 100644 index f962da4f78..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/triggers/manual_trigger.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Any - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase -from reworkd_platform.services.sockets import websockets - - -class ManualTriggerBlock(Block): - type = "ManualTrigger" - description = "Manually trigger the workflow" - image_url = "" - input: BlockIOBase - - async def run(self, workflow_id: Any, **kwargs: Any) -> BlockIOBase: - websockets.log(workflow_id, f"Manual workflow started") - return BlockIOBase() diff --git a/platform/reworkd_platform/schemas/workflow/blocks/url_status_check.py b/platform/reworkd_platform/schemas/workflow/blocks/url_status_check.py deleted file mode 100644 index 9455143c0f..0000000000 --- a/platform/reworkd_platform/schemas/workflow/blocks/url_status_check.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Optional, Any - -import aiohttp -from loguru import logger -from requests import RequestException - -from reworkd_platform.schemas.workflow.base import Block, BlockIOBase - - -class UrlStatusCheckBlockInput(BlockIOBase): - url: str - - -class UrlStatusCheckBlockOutput(UrlStatusCheckBlockInput): - code: Optional[int] - - -class UrlStatusCheckBlock(Block): - type = "UrlStatusCheck" - description = "Outputs the status code of a GET request to a URL" - image_url = "" - input: UrlStatusCheckBlockInput - - async def run(self, workflow_id: str, **kwargs: Any) -> BlockIOBase: - logger.info(f"Starting UrlStatusCheckBlock with url: {self.input.url}") - try: - async with aiohttp.ClientSession() as session: - async with session.get(self.input.url) as response: - code = response.status - except RequestException: - logger.info(f"UrlStatusCheckBlock errored: {RequestException}") - - logger.info(f"UrlStatusCheckBlock Code: {code}") - output = UrlStatusCheckBlockOutput(code=code, **self.input.dict()) - return output