File Upload Functionality

inc/js/agents/system/asset-assistant.mjs

@@ -4,63 +4,104 @@ import mime from 'mime-types'
 import FormData from 'form-data'
 import axios from 'axios'
 //	module constants
+//	module constants
 const { MYLIFE_EMBEDDING_SERVER_BEARER_TOKEN, MYLIFE_EMBEDDING_SERVER_FILESIZE_LIMIT, MYLIFE_EMBEDDING_SERVER_FILESIZE_LIMIT_ADMIN, MYLIFE_SERVER_MBR_ID: mylifeMbrId, } = process.env
 const bearerToken = MYLIFE_EMBEDDING_SERVER_BEARER_TOKEN
 const fileSizeLimit = parseInt(MYLIFE_EMBEDDING_SERVER_FILESIZE_LIMIT) || 1048576
 const fileSizeLimitAdmin = parseInt(MYLIFE_EMBEDDING_SERVER_FILESIZE_LIMIT_ADMIN) || 10485760
-//	module variables
-let AgentFactory
-let Globals
 //	module class definition
 class oAIAssetAssistant {
-	//	pseudo-constructor
-	#ctx // @todo: only useful if assistant only exists for duration of request
-	#file
-	#mbr_id
-	constructor(_ctx){
-		//	primary direct assignment
-		this.#ctx = _ctx
-		this.#mbr_id = this.#ctx.state.member.mbr_id
-		//	module direct assignment
-		if(!AgentFactory) AgentFactory = this.#ctx.AgentFactory
-		if(!Globals) Globals = this.#ctx.Globals
-		//	secondary direct assignment
-		this.#file = this.#extractFile(this.#ctx)
-		//	validate asset construction
-		this.#validateFile()
+	#factory
+	#uploadedFileList=[] // uploaded versions
+	#globals
+	#includeMyLife=false
+	#llm
+	#response
+	#vectorstoreId
+	#vectorstoreFileList // openai vectorstore versions
+	constructor(factory, globals, llm){
+		this.#factory = factory
+		this.#globals = globals
+		this.#llm = llm
+		this.#vectorstoreId = this.#factory.vectorstoreId
 	}
-	async init(){
-		await this.#embedFile()	//	critical step
-		await this.#enactFile()	//	critical step
+	/**
+	 * Initializes the asset assistant by uploading the files to the vectorstore and optionally embedding and enacting the files.
+	 * @param {string} vectorstoreId - The vectorstore id to upload the files into, if already exists (avatar would know).
+	 * @param {boolean} includeMyLife - Whether to embed and enact the files.
+	 * @returns {Promise<oAIAssetAssistant>} - The initialized asset assistant instance.
+	 */
+	async init(includeMyLife=false){
+		await this.updateVectorstoreFileList() // sets `this.#vectorstoreFileList`
+		this.#includeMyLife = includeMyLife
 		return this
 	}
-	//	getters
-	get ctx(){
-		return this.#ctx
+	async updateVectorstoreFileList(){
+		if(this.#vectorstoreId?.length){
+			const updateList = (await this.#llm.files(this.#vectorstoreId)).data
+				.filter(file=>!(this.#vectorstoreFileList ?? []).find(vsFile=>vsFile.id===file.id))
+			if(updateList?.length){
+				this.#vectorstoreFileList = await Promise.all(
+					updateList.map(async file =>await this.#llm.file(file.id))
+				)
+			}
+		}
 	}
-	get file(){
-		return this.#file
+	async upload(files){
+		if(!files || !files.length)
+			throw new Error('No files found in request.')
+		const newFiles = []
+		files.forEach(file => {
+			const hasFile = this.#uploadedFileList.some(_file=>_file.originalName===file.originalName)
+			if(!hasFile)
+				newFiles.push(this.#extractFile(file))
+		})
+		if(newFiles.length){ // only upload new files
+			const vectorstoreId = this.#vectorstoreId
+			this.#uploadedFileList.push(...newFiles)
+			const fileStreams = newFiles.map(file=>fs.createReadStream(file.filepath))
+			const dataRecord = await this.#llm.upload(vectorstoreId, fileStreams, this.mbr_id)
+			const { response, vectorstoreId: newVectorstoreId, success } = dataRecord
+			this.#response = response
+			this.#vectorstoreId = newVectorstoreId
+			if(!vectorstoreId && newVectorstoreId)
+				this.#factory.vectorstoreId = newVectorstoreId // saves to datacore
+			if(success && this.#vectorstoreId?.length)
+				await this.updateVectorstoreFileList()
+		}
+		files.forEach(file=>fs.unlinkSync(file.filepath)) /* delete .tmp files */
+	}
+	//	getters
+	get files(){
+		return this.vectorstoreFileList
 	}
 	get mbr_id(){
-		return this.#mbr_id
+		return this.#factory.mbr_id
 	}
-	get session(){
-		return this.#ctx.session?.MemberSession??null
+	get response(){
+		return this.#response
+	}
+	get vectorstoreFileList(){
+		return this.#vectorstoreFileList
+	}
+	get vectorstoreId(){
+		return this.#vectorstoreId
 	}
 	//	setters
 	//	private functions
 	async #embedFile(){
-		console.log(this.file)
+		const file = this.#uploadedFileList[0]
+		console.log(file)
 		console.log('#embedFile() begin')
 		const _metadata = {
-			source: 'corporate',	//	logickify this
-			source_id: this.file.originalFilename,
+			source: 'corporate',	//	logickify
+			source_id: file.originalFilename,
 			url: 'testing-0001',	//	may or may not use url
 			author: 'MAHT',	//	convert to session member (or agent)
 		}
 		const _token = bearerToken
 		const _data = new FormData()
-		_data.append('file', fs.createReadStream(this.file.filepath), { contentType: this.file.mimetype })
+		_data.append('file', fs.createReadStream(file.filepath), { contentType: file.mimetype })
 		_data.append('metadata', JSON.stringify(_metadata))
 		const _request = {
 			method: 'post',
@@ -77,30 +118,56 @@ class oAIAssetAssistant {
 				console.log(`#embedFile() finished: ${response.data.ids}`)
 				return response.data
 			})
+			.then(response=>{
+				if(this.#includeMyLife)
+					return this.#enactFile(response.data)
+				return response.data
+			})
 			.catch((error) => {
 				console.error(error.message)
 				return {'#embedFile() finished error': error.message }
 			})
 	}
-	async #enactFile(){	//	vitalizes by saving to MyLife database
+	async #enactFile(file){	//	vitalizes by saving to MyLife database
 		console.log('#enactFile() begin')
 		const _fileContent = {
-			...this.file,
+			...file,
 			...{ mbr_id: this.mbr_id }
 		}
 		const oFile = new (AgentFactory.file)(_fileContent)
 		console.log('testing factory',oFile.inspect(true))
 		return oFile
 	}
-	#extractFile(){
-		if(!this.#ctx.request?.files?.file??false) throw new Error('No file found in request.')
-		const { lastModifiedDate, filepath, newFilename, originalFilename, mimetype, size } = this.#ctx.request.files.file
+	/**
+	 * Takes an uploaded file object and extracts relevant file properties.
+	 * @param {File} file - File object
+	 * @returns {object} - Extracted file object.
+	 */
+	#extractFile(file){
+		if(!file)
+			throw new Error('No file found in request.')
+		const { lastModifiedDate, filepath, newFilename, originalFilename, mimetype, size } = file
+		this.#validateFile(file)
 		return {
-			...{ lastModifiedDate, filepath, newFilename, originalFilename, mimetype, size, localFilename: `${Globals.newGuid}.${mime.extension(mimetype)}` },
-			...this.#ctx.request.body
+			...{ lastModifiedDate, filepath, newFilename, originalFilename, mimetype, size, localFilename: `${this.#globals.newGuid}.${mime.extension(mimetype)}` },
 		}
 	}
-	#validateFile(){
+	/**
+	 * Takes an array of uploaded file objects and extracts relevant file properties.
+	 * @param {File[]} files - Array of file objects.
+	 * @returns {File[]} - Array of extracted file objects.
+	 */
+	#extractFiles(files){
+		if(!Array.isArray(files) || !files.length)
+			throw new Error('No files found in request.')
+		return files.map(file=>this.#extractFile(file))
+	}
+	/**
+	 * Validates a file object, _throws error_ if file is invalid.
+	 * @param {File} file - File object
+	 * @returns {void}
+	 */
+	#validateFile(file){
 		const allowedMimeTypes = [
 			'application/json',
 			'application/msword',
@@ -114,14 +181,14 @@ class oAIAssetAssistant {
 			'text/markdown',
 			'text/plain',
 		]
-		// reject size
-		let _mbr_id = this.#ctx.state.member.mbr_id
-		let _maxFileSize = _mbr_id === mylifeMbrId
+		const { size, mimetype } = file
+		const maxFileSize = this.mbr_id === mylifeMbrId
 			?	fileSizeLimitAdmin
 			:	fileSizeLimit
-		if (this.#file.size > _maxFileSize) throw new Error(`File size too large: ${this.#file.size}. Maximum file size is 1MB.`)
-		//	reject mime-type
-		if (!allowedMimeTypes.includes(this.#file.mimetype)) throw new Error(`Unsupported media type: ${this.#file.mimetype}. File type not allowed.`)
+		if((size ?? 0) > maxFileSize)
+			throw new Error(`File size too large: ${ size }. Maximum file size is 1MB.`)
+		if(!allowedMimeTypes.includes(mimetype))
+			throw new Error(`Unsupported media type: ${ mimetype }. File type not allowed.`)
 	}
 }
 // exports

inc/js/api-functions.mjs

@@ -287,6 +287,21 @@ async function tokenValidation(ctx, next) {
         return
     }
 }
+async function upload(ctx){
+    const { body, files: filesWrapper, } = ctx.request
+    const { type } = body
+    let files = filesWrapper['files[]'] // protocol for multiple files in `ctx.request`
+    if(!files)
+        ctx.throw(400, 'No files uploaded.')
+    if(!Array.isArray(files))
+        files = [files]
+    await mAPIKeyValidation(ctx)
+    const { avatar, } = ctx.state
+    const upload = await avatar.upload(files)
+    upload.type = type
+    upload.message = `File(s) [type=${ type }] attempted upload, see "success".`,
+    ctx.body = upload
+}
 /* "private" module functions */
 /**
  * Validates key and sets `ctx.state` and `ctx.session` properties. `ctx.state`: [ assistantType, isValidated, mbr_id, ]. `ctx.session`: [ isAPIValidated, APIMemberKey, ].
@@ -353,4 +368,5 @@ export {
     story,
     storyLibrary,
     tokenValidation,
+    upload,
 }

inc/js/functions.mjs

@@ -1,5 +1,8 @@
 /* imports */
 import oAIAssetAssistant from './agents/system/asset-assistant.mjs'
+import {
+	upload as apiUpload,
+} from './api-functions.mjs'
 /* module export functions */
 async function about(ctx){
 	ctx.state.title = `About MyLife`
@@ -79,7 +82,9 @@ async function chat(ctx){
 	ctx.body = response
 }
 async function collections(ctx){
-	ctx.body = await ctx.state.avatar.collections(ctx.params.type)
+	const { avatar, } = ctx.state
+	const { type, } = ctx.params
+	ctx.body = await avatar.collections(type)
 }
 /**
  * Manage delivery and receipt of contributions(s).
@@ -247,16 +252,18 @@ async function signup(ctx) {
         message: 'Signup successful',
     }
 }
-async function _upload(ctx){	//	post file via post
-	//	revive or create nascent AI-Asset Assistant, that will be used to process the file from validation => storage
-	//	ultimately, this may want to move up in the chain, but perhaps not, considering the need to process binary file content
-	const _oAIAssetAssistant = await new oAIAssetAssistant(ctx).init()
-	ctx.body = _oAIAssetAssistant
-}
-async function upload(ctx){	//	upload display widget/for list and/or action(s)
-	ctx.state.title = `Upload`
-	ctx.state.subtitle = `Upload your files to <em>MyLife</em>`
-	await ctx.render('upload')	//	upload
+/**
+ * Proxy for uploading files to the API.
+ * @param {Koa} ctx - Koa Context object
+ * @returns {object} - The result of the upload as `ctx.body`.
+ */
+async function upload(ctx){
+	const { avatar, } = ctx.state
+	if(avatar.isMyLife)
+		throw new Error('Only logged in members may upload files')
+	ctx.session.APIMemberKey = avatar.mbr_id
+	ctx.session.isAPIValidated = true
+	await apiUpload(ctx)
 }
 /* module private functions */
 function mGetContributions(ctx){
@@ -315,5 +322,4 @@ export {
 	privacyPolicy,
 	signup,
 	upload,
-	_upload,
 }

inc/js/mylife-agent-factory.mjs

@@ -666,6 +666,16 @@ class AgentFactory extends BotFactory{
 	get urlEmbeddingServer(){
 		return process.env.MYLIFE_EMBEDDING_SERVER_URL+':'+process.env.MYLIFE_EMBEDDING_SERVER_PORT
 	}
+	get vectorstoreId(){
+		return this.core.vectorstoreId
+	}
+	set vectorstoreId(vectorstoreId){
+		/* validate vectorstoreId */
+		if(!vectorstoreId?.length)
+			throw new Error('vectorstoreId required')
+		this.dataservices.patch(this.core.id, { vectorstoreId, }) /* no await */
+		this.core.vectorstoreId = vectorstoreId /* update local */
+	}
 }
 // private module functions
 /**