diff --git a/AIAgent/Package.swift b/AIAgent/Package.swift index 80b9753..99f1d2d 100644 --- a/AIAgent/Package.swift +++ b/AIAgent/Package.swift @@ -18,15 +18,17 @@ let package = Package( dependencies: [ .package(url: "https://github.com/bsorrentino/LangGraph-Swift.git", exact: "1.2.2"), // .package(path: "/Users/bsorrentino/WORKSPACES/GITHUB.me/AppleOS/LangGraph-Swift"), - .package(url: "https://github.com/bsorrentino/Swift-OpenAI.git", branch: "develop"), // Add the dependency here - ], +// .package(url: "https://github.com/bsorrentino/Swift-OpenAI.git", branch: "develop"), // Add the dependency here + .package(url: "https://github.com/MacPaw/OpenAI.git", branch: "main") + ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. // Targets can depend on other targets in this package and products from dependencies. .target( name: "AIAgent", dependencies: [ - .product(name: "OpenAI", package: "Swift-OpenAI"), +// .product(name: "OpenAI", package: "Swift-OpenAI"), + .product(name: "OpenAI", package: "OpenAI"), .product(name: "LangGraph", package: "LangGraph-Swift") ], resources: [ .process("Resources")]), .testTarget( diff --git a/AIAgent/Sources/AIAgent/AgentExecutor.swift b/AIAgent/Sources/AIAgent/AgentExecutor.swift index 2acba67..25bcccb 100644 --- a/AIAgent/Sources/AIAgent/AgentExecutor.swift +++ b/AIAgent/Sources/AIAgent/AgentExecutor.swift @@ -65,6 +65,11 @@ struct DiagramDescription : Codable { var description: DiagramNLPDescription // NLP description } +public enum DiagramImageValue { + case data( Data ) + case url( String ) +} + struct AgentExecutorState : AgentState { var data: [String : Any] @@ -77,8 +82,8 @@ struct AgentExecutorState : AgentState { data = initState } - var diagramImageUrlOrData:String? { - data["diagram_image_url_or_data"] as? String + var diagramImageUrlOrData:DiagramImageValue? { + data["diagram_image_url_or_data"] as? DiagramImageValue } var diagramCode:String? { @@ -120,24 +125,42 @@ func describeDiagramImage( state: AgentExecutorState, openAI:OpenAI, delegate:T ) async throws -> PartialAgentState { - guard let imageUrl = state.diagramImageUrlOrData else { + guard let imageUrlValue = state.diagramImageUrlOrData else { throw _EX("diagramImageUrlOrData not initialized!") } await delegate.progress("starting analyze\ndiagram 👀") let prompt = try loadPromptFromBundle(fileName: "describe_diagram_prompt") + + let query = switch( imageUrlValue ) { + case .url( let url): + ChatQuery(messages: [ + .user(.init(content: .vision([ + .chatCompletionContentPartTextParam(.init(text: prompt)), + .chatCompletionContentPartImageParam(.init(imageUrl: .init(url: url, detail: .auto))) + ]))) + ], model: Model.gpt4_vision_preview, maxTokens: 2000) + case .data(let data): + ChatQuery(messages: [ + .user(.init(content: .vision([ + .chatCompletionContentPartTextParam(.init(text: prompt)), + .chatCompletionContentPartImageParam(.init(imageUrl: .init(url: data, detail: .auto))) + ]))) + ], model: Model.gpt4_o, maxTokens: 2000) + + } - let query = ChatQuery( - model: .gpt4_vision_preview, - messages: [ - Chat(role: .user, content: [ - ChatContent(text: prompt), - ChatContent(imageUrl: imageUrl ) - ]) - ], - maxTokens: 2000 - ) +// let query = ChatQuery( +// model: .gpt4_vision_preview, +// messages: [ +// Chat(role: .user, content: [ +// ChatContent(text: prompt), +// ChatContent(imageUrl: imageUrl ) +// ]) +// ], +// maxTokens: 2000 +// ) let chatResult = try await openAI.chats(query: query) @@ -179,15 +202,19 @@ func translateSequenceDiagramDescriptionToPlantUML( sta .replacingOccurrences(of: "{diagram_title}", with: diagram.title) .replacingOccurrences(of: "{diagram_description}", with: description) - let query = ChatQuery( - model: .gpt3_5Turbo, - messages: [ - Chat(role: .user, content: [ - ChatContent(text: prompt), - ]) - ], - maxTokens: 2000 - ) + let query = ChatQuery(messages: [ + .user(.init(content: .string(prompt))) + ], model: Model.gpt3_5Turbo, maxTokens: 2000) + +// let query = ChatQuery( +// model: .gpt3_5Turbo, +// messages: [ +// Chat(role: .user, content: [ +// ChatContent(text: prompt), +// ]) +// ], +// maxTokens: 2000 +// ) let chatResult = try await openAI.chats(query: query) @@ -224,15 +251,19 @@ func translateGenericDiagramDescriptionToPlantUML( stat prompt = prompt .replacingOccurrences(of: "{diagram_description}", with: content) - let query = ChatQuery( - model: .gpt3_5Turbo, - messages: [ - Chat(role: .user, content: [ - ChatContent(text: prompt), - ]) - ], - maxTokens: 2000 - ) + let query = ChatQuery(messages: [ + .user(.init(content: .string(prompt))) + ], model: Model.gpt3_5Turbo, maxTokens: 2000) + +// let query = ChatQuery( +// model: .gpt3_5Turbo, +// messages: [ +// Chat(role: .user, content: [ +// ChatContent(text: prompt), +// ]) +// ], +// maxTokens: 2000 +// ) let chatResult = try await openAI.chats(query: query) @@ -264,7 +295,7 @@ func routeDiagramTranslation( state: AgentExecutorState ) async throws -> String } public func runTranslateDrawingToPlantUML( openAI: OpenAI, - imageUrl: String, + imageValue: DiagramImageValue, delegate:T ) async throws -> String? { let workflow = GraphState { AgentExecutorState() } @@ -295,7 +326,7 @@ public func runTranslateDrawingToPlantUML( openAI: Open let app = try workflow.compile() let inputs:[String : Any] = [ - "diagram_image_url_or_data": imageUrl + "diagram_image_url_or_data": imageValue ] let response = try await app.invoke( inputs: inputs) @@ -308,20 +339,30 @@ public func updatePlantUML( openAI: OpenAI, withModel model: Model, input: String, withInstruction instruction: String ) async throws -> String? { - let query = ChatQuery( - model: model, - messages: [ - .init(role: .system, content: - """ - You are my plantUML assistant. - You must answer exclusively with diagram syntax. - """), - .init( role: .assistant, content: input ), - .init( role: .user, content: instruction ) - ], - temperature: 0.0, - topP: 1.0 - ) + + let query = ChatQuery(messages: [ + .system(.init(content: """ + You are my plantUML assistant. + You must answer exclusively with diagram syntax. + """)), + .assistant(.init( content: input)), + .user(.init(content: .string(instruction))) + ], model: model, temperature: 0.0, topP: 1.0) + +// let query = ChatQuery( +// model: model, +// messages: [ +// .init(role: .system, content: +// """ +// You are my plantUML assistant. +// You must answer exclusively with diagram syntax. +// """), +// .init( role: .assistant, content: input ), +// .init( role: .user, content: instruction ) +// ], +// temperature: 0.0, +// topP: 1.0 +// ) let chat = try await openAI.chats(query: query) diff --git a/AIAgent/Sources/AIAgent/AgentExecutorDemo.swift b/AIAgent/Sources/AIAgent/AgentExecutorDemo.swift index 7fe1ef3..16bdeaa 100644 --- a/AIAgent/Sources/AIAgent/AgentExecutorDemo.swift +++ b/AIAgent/Sources/AIAgent/AgentExecutorDemo.swift @@ -30,7 +30,7 @@ struct AgentExecutorDemoState : AgentState { } public func runTranslateDrawingToPlantUMLDemo( openAI: OpenAI, - imageUrl: String, + imageValue: DiagramImageValue, delegate:T ) async throws -> String? { let workflow = GraphState { AgentExecutorState() } diff --git a/PlantUML/OpenAIObservableService.swift b/PlantUML/OpenAIObservableService.swift index f0012f7..7249fd9 100644 --- a/PlantUML/OpenAIObservableService.swift +++ b/PlantUML/OpenAIObservableService.swift @@ -28,6 +28,7 @@ class OpenAIObservableService : ObservableObject { // @Published public var inputModel:String @AppStorage("openaiModel") private var openAIModel:String = "gpt-3.5-turbo" + @AppStorage("visionModel") private var visionModel:String = "gpt-4o" @AppSecureStorage("openaikey") private var openAIKey:String? @AppSecureStorage("openaiorg") private var openAIOrg:String? @@ -140,7 +141,7 @@ class OpenAIObservableService : ObservableObject { extension OpenAIObservableService { @MainActor - func processImageWithAgents( imageUrl: String, delegate:T ) async -> String? { + func processImageWithAgents( imageData: Data, delegate:T ) async -> String? { guard let openAI, case .Ready = status else { delegate.progress("WARNING: OpenAI API not initialized") @@ -152,8 +153,8 @@ extension OpenAIObservableService { do { async let runTranslation = DEMO_MODE ? - try runTranslateDrawingToPlantUMLDemo( openAI: openAI, imageUrl: imageUrl, delegate:delegate) : - try runTranslateDrawingToPlantUML( openAI: openAI, imageUrl: imageUrl, delegate:delegate); + try runTranslateDrawingToPlantUMLDemo( openAI: openAI, imageValue: DiagramImageValue.data(imageData), delegate:delegate) : + try runTranslateDrawingToPlantUML( openAI: openAI, imageValue: DiagramImageValue.data(imageData), delegate:delegate); if let content = try await runTranslation { diff --git a/PlantUML/PlantUMLDrawingView.swift b/PlantUML/PlantUMLDrawingView.swift index e02a9ab..311e04d 100644 --- a/PlantUML/PlantUMLDrawingView.swift +++ b/PlantUML/PlantUMLDrawingView.swift @@ -129,12 +129,11 @@ extension PlantUMLDrawingView : AgentExecutorDelegate { let image = image().withBackground(color: backgroundColor) if let imageData = image.pngData() { - + if SAVE_DRAWING_IMAGE { saveData(imageData, toFile: "image.png", inDirectory: .picturesDirectory) } - let base64Image = imageData.base64EncodedString() processing.toggle() isUseDrawingTool = false @@ -146,16 +145,24 @@ extension PlantUMLDrawingView : AgentExecutorDelegate { document.drawing = canvas.drawing.dataRepresentation() dismiss() } - - if let content = await service.processImageWithAgents( imageUrl: "data:image/png;base64,\(base64Image)", delegate: self ) { - - document.text = content + + if let content = await service.processImageWithAgents( imageData: imageData, delegate: self ) { + + document.text = content } + +// let base64Image = imageData.base64EncodedString() +// +// if let content = await service.processImageWithAgents( imageUrl: "data:image/png;base64,\(base64Image)", delegate: self ) { +// +// document.text = content +// +// } + } - + } - } } diff --git a/PlantUML/Settings.bundle/Root.plist b/PlantUML/Settings.bundle/Root.plist index e23007a..458d8e7 100644 --- a/PlantUML/Settings.bundle/Root.plist +++ b/PlantUML/Settings.bundle/Root.plist @@ -10,36 +10,6 @@ Type PSGroupSpecifier - DefaultValue chrome @@ -236,11 +206,11 @@ Key openaiModel Title - Model + Prompt Model Titles gpt-3.5-turbo - gpt-4 + gpt-4o Type PSMultiValueSpecifier @@ -250,6 +220,24 @@ gpt-4 + + DefaultValue + gpt-4o + Key + visionModel + Title + Vision Model + Titles + + gpt-4o + + Type + PSMultiValueSpecifier + Values + + gpt-4o + + diff --git a/PlantUMLApp.xcodeproj/project.pbxproj b/PlantUMLApp.xcodeproj/project.pbxproj index 23e206d..d521fdd 100644 --- a/PlantUMLApp.xcodeproj/project.pbxproj +++ b/PlantUMLApp.xcodeproj/project.pbxproj @@ -3,15 +3,10 @@ archiveVersion = 1; classes = { }; - objectVersion = 60; + objectVersion = 55; objects = { /* Begin PBXBuildFile section */ - A00433272BA5FC23000954B3 /* AIAgent in Frameworks */ = {isa = PBXBuildFile; productRef = A00433262BA5FC23000954B3 /* AIAgent */; }; - A00433292BA5FC23000954B3 /* AppSecureStorage in Frameworks */ = {isa = PBXBuildFile; productRef = A00433282BA5FC23000954B3 /* AppSecureStorage */; }; - A004332B2BA5FC23000954B3 /* CodeViewer in Frameworks */ = {isa = PBXBuildFile; productRef = A004332A2BA5FC23000954B3 /* CodeViewer */; }; - A004332D2BA5FC23000954B3 /* PlantUMLFramework in Frameworks */ = {isa = PBXBuildFile; productRef = A004332C2BA5FC23000954B3 /* PlantUMLFramework */; }; - A004332F2BA5FC23000954B3 /* PlantUMLKeyboard in Frameworks */ = {isa = PBXBuildFile; productRef = A004332E2BA5FC23000954B3 /* PlantUMLKeyboard */; }; A0277C0B293D11D7005435AE /* AsyncImage+Cache.swift in Sources */ = {isa = PBXBuildFile; fileRef = A09A6DDA293D0E5E000856ED /* AsyncImage+Cache.swift */; }; A02BB3E82B41C9430073D432 /* OpenAIObservableService.swift in Sources */ = {isa = PBXBuildFile; fileRef = A02BB3E72B41C9430073D432 /* OpenAIObservableService.swift */; }; A036CC3D2A0FC67C003FDB5E /* PlantUMLKeyboard in Frameworks */ = {isa = PBXBuildFile; productRef = A084B53329EB15C50043B853 /* PlantUMLKeyboard */; }; @@ -20,6 +15,11 @@ A04512F12B3DFCB900CD1158 /* PlantUMLDiagramMenu.swift in Sources */ = {isa = PBXBuildFile; fileRef = A04512F02B3DFCB900CD1158 /* PlantUMLDiagramMenu.swift */; }; A04512F32B3E2D8E00CD1158 /* PlantUMLDrawingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A04512F22B3E2D8E00CD1158 /* PlantUMLDrawingView.swift */; }; A047206F29549ACC007E061F /* SwiftUI+Share.swift in Sources */ = {isa = PBXBuildFile; fileRef = A047206E29549ACC007E061F /* SwiftUI+Share.swift */; }; + A04F96B22C31763900044DCB /* AIAgent in Frameworks */ = {isa = PBXBuildFile; productRef = A04F96B12C31763900044DCB /* AIAgent */; }; + A04F96B42C31763900044DCB /* AppSecureStorage in Frameworks */ = {isa = PBXBuildFile; productRef = A04F96B32C31763900044DCB /* AppSecureStorage */; }; + A04F96B62C31763900044DCB /* CodeViewer in Frameworks */ = {isa = PBXBuildFile; productRef = A04F96B52C31763900044DCB /* CodeViewer */; }; + A04F96B82C31763900044DCB /* PlantUMLFramework in Frameworks */ = {isa = PBXBuildFile; productRef = A04F96B72C31763900044DCB /* PlantUMLFramework */; }; + A04F96BA2C31763900044DCB /* PlantUMLKeyboard in Frameworks */ = {isa = PBXBuildFile; productRef = A04F96B92C31763900044DCB /* PlantUMLKeyboard */; }; A068572E29D8B31100E82C2F /* View+Clipboard.swift in Sources */ = {isa = PBXBuildFile; fileRef = A068572D29D8B31100E82C2F /* View+Clipboard.swift */; }; A082CB002A9A68D80055D6D6 /* Settings.bundle in Resources */ = {isa = PBXBuildFile; fileRef = A082CAFF2A9A68D80055D6D6 /* Settings.bundle */; }; A0943A6F2944A44900342426 /* ScaleToFit+ToggleStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A0943A6E2944A44900342426 /* ScaleToFit+ToggleStyle.swift */; }; @@ -145,11 +145,11 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - A004332D2BA5FC23000954B3 /* PlantUMLFramework in Frameworks */, - A00433272BA5FC23000954B3 /* AIAgent in Frameworks */, - A00433292BA5FC23000954B3 /* AppSecureStorage in Frameworks */, - A004332F2BA5FC23000954B3 /* PlantUMLKeyboard in Frameworks */, - A004332B2BA5FC23000954B3 /* CodeViewer in Frameworks */, + A04F96B82C31763900044DCB /* PlantUMLFramework in Frameworks */, + A04F96B22C31763900044DCB /* AIAgent in Frameworks */, + A04F96B42C31763900044DCB /* AppSecureStorage in Frameworks */, + A04F96BA2C31763900044DCB /* PlantUMLKeyboard in Frameworks */, + A04F96B62C31763900044DCB /* CodeViewer in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -335,11 +335,11 @@ ); name = PlantUMLApp; packageProductDependencies = ( - A00433262BA5FC23000954B3 /* AIAgent */, - A00433282BA5FC23000954B3 /* AppSecureStorage */, - A004332A2BA5FC23000954B3 /* CodeViewer */, - A004332C2BA5FC23000954B3 /* PlantUMLFramework */, - A004332E2BA5FC23000954B3 /* PlantUMLKeyboard */, + A04F96B12C31763900044DCB /* AIAgent */, + A04F96B32C31763900044DCB /* AppSecureStorage */, + A04F96B52C31763900044DCB /* CodeViewer */, + A04F96B72C31763900044DCB /* PlantUMLFramework */, + A04F96B92C31763900044DCB /* PlantUMLKeyboard */, ); productName = PlantUML; productReference = A0D3C64628984A0E000838D7 /* PlantUMLApp.app */; @@ -400,11 +400,6 @@ ); mainGroup = A0D3C63D28984A0E000838D7; packageReferences = ( - A052514A2BA5FAE5000096F3 /* XCLocalSwiftPackageReference "AIAgent" */, - A052514B2BA5FB29000096F3 /* XCLocalSwiftPackageReference "AppSecureStorage" */, - A052514C2BA5FB4A000096F3 /* XCLocalSwiftPackageReference "PlantUMLEditor" */, - A052514D2BA5FB5D000096F3 /* XCLocalSwiftPackageReference "PlantUMLFramework" */, - A052514E2BA5FB67000096F3 /* XCLocalSwiftPackageReference "PlantUMLKeyboard" */, ); productRefGroup = A0D3C64728984A0E000838D7 /* Products */; projectDirPath = ""; @@ -861,47 +856,24 @@ }; /* End XCConfigurationList section */ -/* Begin XCLocalSwiftPackageReference section */ - A052514A2BA5FAE5000096F3 /* XCLocalSwiftPackageReference "AIAgent" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = AIAgent; - }; - A052514B2BA5FB29000096F3 /* XCLocalSwiftPackageReference "AppSecureStorage" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = AppSecureStorage; - }; - A052514C2BA5FB4A000096F3 /* XCLocalSwiftPackageReference "PlantUMLEditor" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = PlantUMLEditor; - }; - A052514D2BA5FB5D000096F3 /* XCLocalSwiftPackageReference "PlantUMLFramework" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = PlantUMLFramework; - }; - A052514E2BA5FB67000096F3 /* XCLocalSwiftPackageReference "PlantUMLKeyboard" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = PlantUMLKeyboard; - }; -/* End XCLocalSwiftPackageReference section */ - /* Begin XCSwiftPackageProductDependency section */ - A00433262BA5FC23000954B3 /* AIAgent */ = { + A04F96B12C31763900044DCB /* AIAgent */ = { isa = XCSwiftPackageProductDependency; productName = AIAgent; }; - A00433282BA5FC23000954B3 /* AppSecureStorage */ = { + A04F96B32C31763900044DCB /* AppSecureStorage */ = { isa = XCSwiftPackageProductDependency; productName = AppSecureStorage; }; - A004332A2BA5FC23000954B3 /* CodeViewer */ = { + A04F96B52C31763900044DCB /* CodeViewer */ = { isa = XCSwiftPackageProductDependency; productName = CodeViewer; }; - A004332C2BA5FC23000954B3 /* PlantUMLFramework */ = { + A04F96B72C31763900044DCB /* PlantUMLFramework */ = { isa = XCSwiftPackageProductDependency; productName = PlantUMLFramework; }; - A004332E2BA5FC23000954B3 /* PlantUMLKeyboard */ = { + A04F96B92C31763900044DCB /* PlantUMLKeyboard */ = { isa = XCSwiftPackageProductDependency; productName = PlantUMLKeyboard; }; diff --git a/PlantUMLApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/PlantUMLApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 9f32e77..3bae8e9 100644 --- a/PlantUMLApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/PlantUMLApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -12,10 +12,10 @@ }, { "package": "OpenAI", - "repositoryURL": "https://github.com/bsorrentino/Swift-OpenAI.git", + "repositoryURL": "https://github.com/MacPaw/OpenAI.git", "state": { - "branch": "develop", - "revision": "b616cb913c84d264fc33f1509ff3c7fe37e29e60", + "branch": "main", + "revision": "fd13a41e987004d14f1793c570721953a2767b03", "version": null } }