Skip to content

Commit

Permalink
refactor: Add image support to prompt functions
Browse files Browse the repository at this point in the history
  • Loading branch information
nmaguiar committed May 14, 2024
1 parent a90fe1c commit 01395c1
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 2 deletions.
1 change: 1 addition & 0 deletions src/docs/USAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,7 @@ List of options to use when _in=llm_ or _llmprompt=..._:
| llmenv | String | The environment variable containing the value of 'llmoptions' (defaults to OAFP_MODEL) |
| llmoptions | String | A JSON or SLON string with OpenAF's LLM 'type' (e.g. openai/ollama), 'model' name, 'timeout' in ms for answersm, 'url' for the ollama type or 'key' for openai type |
| llmconversation | String | File to keep the LLM conversation |
| llmimage | String | For visual models you can provide a base64 image or an image file path or an URL of an image |

---

Expand Down
15 changes: 13 additions & 2 deletions src/include/inputFns.js
Original file line number Diff line number Diff line change
Expand Up @@ -439,10 +439,21 @@ var _inputFns = new Map([
if (isDef(params.llmconversation) && io.fileExists(params.llmconversation))
res.getGPT().setConversation( io.readFileJSON(params.llmconversation) )
let __res
let img
if (isString(params.llmimage)) {
if (params.llmimage.toLowerCase().match(/^https?:\/\//))
img = af.fromBytes2String(af.toBase64Bytes(af.fromInputStream2Bytes($rest().get2Stream(params.llmimage))))
else if (io.fileExists(params.llmimage))
img = af.fromBytes2String(af.toBase64Bytes(io.readFileBytes(params.llmimage)))
}
if (params.output == "md" || params.output == "mdtable" || params.output == "raw") {
__res = res.prompt(_res)
__res = isDef(img) ? res.promptImage(_res, img) : res.prompt(_res)
} else {
__res = res.promptJSON(_res)
if (isDef(img)) {
__res = res.promptImage(_res, img, __, __, __, __, true)
} else {
__res = res.promptJSON(_res)
}
}
if (isDef(params.llmconversation)) io.writeFileJSON( params.llmconversation, res.getGPT().getConversation(), "" )

Expand Down

0 comments on commit 01395c1

Please sign in to comment.