Merge pull request #351 from AutomatingSciencePipeline/development

Convert Firebase to Mongo

apps/backend/.pylintrc

@@ -4,6 +4,7 @@
 extension-pkg-allow-list=pydantic  # Updated from 'extension-pkg-whitelist'
 disable=
     C0301,  # line too long
+    C0303,  # trailing whitespace
     C0114,  # missing-module-docstring
     C0116,  # missing-function-docstring
     C0115,  # missing-class-docstring

apps/backend/app.py

@@ -8,7 +8,7 @@
 from flask import Flask, Response, request, jsonify, send_file
 from kubernetes import client, config
 import pymongo
-from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments, download_experiment_file
+from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments, download_experiment_file, get_experiment, update_exp_value
 
 from spawn_runner import create_job, create_job_object
 flaskApp = Flask(__name__)
@@ -27,7 +27,8 @@
     username=str(os.getenv("MONGODB_USERNAME")),
     password=str(os.getenv("MONGODB_PASSWORD")),
     authMechanism='SCRAM-SHA-256',
-    serverSelectionTimeoutMS=1000
+    serverSelectionTimeoutMS=1000,
+    replicaSet='rs0'
 )
 # connect to the glados database
 gladosDB = mongoClient["gladosdb"]
@@ -107,6 +108,26 @@ def download_exp_file():
         return send_file(file_stream, as_attachment=True, download_name="experiment_file", mimetype="application/octet-stream")
     except Exception:
         return Response(status=500)
+
+@flaskApp.post("/getExperiment")
+def get_experiment_post():
+    try:
+        experiment_id = request.get_json()['experimentId']
+        return {'contents': get_experiment(experiment_id, mongoClient)}
+    except Exception:
+        return Response(status=500)
+
+@flaskApp.post("/updateExperiment")
+def update_experiment():
+    try:
+        json = request.get_json()
+        experiment_id = json['experimentId']
+        field = json['field']
+        newVal = json['newValue']
+        update_exp_value(experiment_id, field, newVal, mongoClient)
+        return Response(status=200)
+    except Exception:
+        return Response(status=500)
 
 if __name__ == '__main__':
     flaskApp.run()

apps/backend/modules/mongo.py

@@ -1,7 +1,7 @@
 import json
 import pymongo
 from pymongo.errors import ConnectionFailure
-from bson import Binary
+from bson import Binary, ObjectId
 from gridfs import GridFSBucket
 
 def verify_mongo_connection(mongoClient: pymongo.MongoClient):
@@ -12,7 +12,7 @@ def verify_mongo_connection(mongoClient: pymongo.MongoClient):
         raise Exception("MongoDB server not available/unreachable") from err
 
 def upload_experiment_aggregated_results(experimentId: str, results: str, mongoClient: pymongo.MongoClient):
-    experimentResultEntry = {"_id": experimentId, "resultContent": results}
+    experimentResultEntry = {"experimentId": experimentId, "resultContent": results}
     # Get the results connection
     resultsCollection = mongoClient["gladosdb"].results
     try:
@@ -25,17 +25,16 @@ def upload_experiment_aggregated_results(experimentId: str, results: str, mongoC
         raise Exception("Encountered error while storing aggregated results in MongoDB") from err
 
 def upload_experiment_zip(experimentId: str, encoded: Binary, mongoClient: pymongo.MongoClient):
-    experimentZipEntry = {"_id": experimentId, "fileContent": encoded}
+    experimentZipEntry = {"experimentId": experimentId, "fileContent": encoded}
     zipCollection = mongoClient["gladosdb"].zips
     try:
-        # TODO: Refactor to call the backend
         resultZipId = zipCollection.insert_one(experimentZipEntry).inserted_id
         return resultZipId
     except Exception as err:
         raise Exception("Encountered error while storing results zip in MongoDB") from err
 
 def upload_log_file(experimentId: str, contents: str, mongoClient: pymongo.MongoClient):
-    logFileEntry = {"_id": experimentId, "fileContent": contents}
+    logFileEntry = {"experimentId": experimentId, "fileContent": contents}
     logCollection = mongoClient["gladosdb"].logs
     try:
         resultId = logCollection.insert_one(logFileEntry).inserted_id
@@ -89,4 +88,17 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient):
     file = bucket.open_download_stream_by_name(file_name)
     contents = file.read()
     return contents
-
+
+def get_experiment(expId: str, mongoClient: pymongo.MongoClient):
+    experimentsCollection = mongoClient["gladosdb"].experiments
+    experiment = experimentsCollection.find_one({"_id": ObjectId(expId)}, {"_id": 0})
+    if experiment is None:
+        raise Exception("Could not find experiment!")
+    experiment["id"] = expId
+    experiment["expId"] = expId
+    return experiment
+
+def update_exp_value(expId: str, field: str, newValue: str, mongoClient: pymongo.MongoClient):
+    experimentsCollection = mongoClient["gladosdb"].experiments
+    experimentsCollection.update_one({"_id": ObjectId(expId)}, {"$set": {field: newValue}})
+    return

apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx

@@ -5,9 +5,6 @@ import Parameter from '../../Parameter';
 import { useForm, formList, joiResolver } from '@mantine/form';
 import { experimentSchema } from '../../../../utils/validators';
 
-import { firebaseApp } from '../../../../firebase/firebaseClient';
-import { getDoc, getFirestore, doc } from 'firebase/firestore';
-
 import { DispatchStep } from './stepComponents/DispatchStep';
 import { InformationStep } from './stepComponents/InformationStep';
 import { ParamStep } from './stepComponents/ParamStep';
@@ -16,7 +13,9 @@ import { ConfirmationStep } from './stepComponents/ConfirmationStep';
 import { DumbTextArea } from './stepComponents/DumbTextAreaStep';
 import { DB_COLLECTION_EXPERIMENTS } from '../../../../firebase/db';
 
-const DEFAULT_TRIAL_TIMEOUT_SECONDS = 5*60*60; // 5 hours in seconds
+import { getDocumentFromId } from '../../../../lib/mongodb_funcs';
+
+const DEFAULT_TRIAL_TIMEOUT_SECONDS = 5 * 60 * 60; // 5 hours in seconds
 
 export const FormStates = {
 	Closed: -1,
@@ -85,11 +84,9 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest })
 
 	useEffect(() => {
 		if (copyID != null) {
-			const db = getFirestore(firebaseApp);
-			getDoc(doc(db, DB_COLLECTION_EXPERIMENTS, copyID)).then((docSnap) => {
-				if (docSnap.exists()) {
-					const expInfo = docSnap.data();
-					const hyperparameters = JSON.parse(expInfo['hyperparameters'])['hyperparameters'];
+			getDocumentFromId(copyID).then((expInfo) => {
+				if (expInfo) {
+					const hyperparameters = expInfo['hyperparameters'];
 					form.setValues({
 						hyperparameters: formList(hyperparameters),
 						name: expInfo['name'],
@@ -107,17 +104,17 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest })
 					});
 					setCopyId(null);
 					setStatus(FormStates.Info);
-					console.log('Copied!');
-				} else {
-					console.log('No such document!');
 				}
-			});
+				else {
+					console.log("Could not get expInfo!!!");
+				}
+			})
 		}
 	}, [copyID]); // TODO adding form or setCopyId causes render loop?
 
 
 	const fields = form.values.hyperparameters.map(({ type, ...rest }, index) => {
-		return <Parameter key = {index} form={form} type={type} index={index} {...rest} />;
+		return <Parameter key={index} form={form} type={type} index={index} {...rest} />;
 	});
 
 	const [open, setOpen] = useState(true);
@@ -191,7 +188,7 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest })
 									) : status === FormStates.Confirmation ? (
 										<ConfirmationStep form={form} />
 									) : (
-										<DispatchStep form = {form} id={id} />
+										<DispatchStep form={form} id={id} />
 									)}
 
 									<div className='flex-shrink-0 border-t border-gray-200 px-4 py-5 sm:px-6'>
@@ -230,11 +227,13 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, ...rest })
 											<button
 												className='rounded-md w-1/6 border border-transparent bg-blue-600 py-2 px-4 text-sm font-medium text-white shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2'
 												{...(status === FormStates.Dispatch ?
-													{ type: 'submit', onClick: () => {
-														setFormState(-1);
-														localStorage.removeItem('ID');
-														setStatus(FormStates.Info);
-													} } :
+													{
+														type: 'submit', onClick: () => {
+															setFormState(-1);
+															localStorage.removeItem('ID');
+															setStatus(FormStates.Info);
+														}
+													} :
 													{
 														type: 'button',
 														onClick: () => setStatus(status + 1),

apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx

@@ -1,7 +1,7 @@
 'use client'
 
 import { Dropzone, DropzoneProps } from '@mantine/dropzone';
-import { submitExperiment, uploadExec } from '../../../../../firebase/db';
+import { submitExperiment } from '../../../../../firebase/db';
 import { Group, Text } from '@mantine/core';
 
 import { useAuth } from '../../../../../firebase/fbAuth';
@@ -23,9 +23,8 @@ export const DispatchStep = ({ id, form, ...props }) => {
 
 	const onDropFile = (files: Parameters<DropzoneProps['onDrop']>[0]) => {
 		setLoading(true);
-		console.log('Submitting Experiment');
-		submitExperiment(form.values, userId as string).then(async (expId) => {
-			console.log(`Uploading file for ${expId}:`, files);
+		submitExperiment(form.values, userId as string).then(async (json) => {
+			const expId = json['id'];
 			const formData = new FormData();
 			formData.set("file", files[0]);
 			formData.set("expId", expId);
@@ -36,7 +35,7 @@ export const DispatchStep = ({ id, form, ...props }) => {
 			});
 			if (uploadResponse.ok) {
 				console.log(`Handing experiment ${expId} to the backend`);
-				const response = await fetch(`/api/experiments/${expId}`, {
+				const response = await fetch(`/api/experiments/start/${expId}`, {
 					method: 'POST',
 					headers: new Headers({ 'Content-Type': 'application/json' }),
 					credentials: 'same-origin',
@@ -107,13 +106,3 @@ export const DispatchStep = ({ id, form, ...props }) => {
 		</Dropzone>
 	);
 };
-
-function arrayBufferToBase64(buffer) {
-    let binary = '';
-    const bytes = new Uint8Array(buffer);
-    for (let i = 0; i < bytes.byteLength; i++) {
-        binary += String.fromCharCode(bytes[i]);
-    }
-    return Buffer.from(binary).toString("base64");
-}
-

apps/frontend/app/components/flows/ViewExperiment/ExperimentListing.tsx

@@ -1,17 +1,17 @@
 /* eslint-disable no-mixed-spaces-and-tabs */
 import { ChevronRightIcon } from '@heroicons/react/24/solid';
 import { useEffect, useState } from 'react';
-import { ExperimentDocumentId, subscribeToExp, updateProjectNameInFirebase, getCurrentProjectName } from '../../../../firebase/db';
 import { ExperimentData } from '../../../../firebase/db_types';
 import { MdEdit, MdPadding } from 'react-icons/md';
 import { Timestamp } from 'mongodb';
+import { updateExperimentNameById } from '../../../../lib/mongodb_funcs';
 
 export interface ExperimentListingProps {
 	projectinit: ExperimentData;
-	onCopyExperiment: (experimentId: ExperimentDocumentId) => void;
-	onDownloadResults: (experimentId: ExperimentDocumentId) => Promise<void>;
-	onDownloadProjectZip: (experimentId: ExperimentDocumentId) => Promise<void>;
-	onDeleteExperiment: (experimentId: ExperimentDocumentId) => void;
+	onCopyExperiment: (experimentId: string) => void;
+	onDownloadResults: (experimentId: string) => Promise<void>;
+	onDownloadProjectZip: (experimentId: string) => Promise<void>;
+	onDeleteExperiment: (experimentId: string) => void;
 }
 
 
@@ -46,8 +46,9 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 
 	const handleSave = (newProjectName) => {
 		// Update the project name in Firebase with the edited name
-		updateProjectNameInFirebase(project.expId, projectName);
-
+		updateExperimentNameById(project.expId, newProjectName).catch((reason) =>{
+			console.log(`Failed to update experiment name, reason: ${reason}`);
+		});
 		// Exit the editing mode
 		setIsEditing(false);
 	};
@@ -60,13 +61,18 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 	};
 
 	useEffect(() => {
+		console.log(project.creator);
 		if (editingCanceled) {
 			setProjectName(originalProjectName); // Revert to the original name
 			setEditingCanceled(true);
 		} else {
-			subscribeToExp(project.expId, (data) => {
-				setProject(data as ExperimentData);
-			});
+			const eventSource = new EventSource(`/api/experiments/subscribe?expId=${project.expId}`);
+			eventSource.onmessage = (event) => {
+				if (event.data !== 'heartbeat' && event.data) {
+					setProject(JSON.parse(event.data) as ExperimentData);
+				}
+
+			}
 		}
 	}, [editingCanceled, originalProjectName, project.expId]);
 
@@ -171,7 +177,7 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 							<span
 								className="hidden sm:inline-block sm:align-middle sm:h-screen"
 								aria-hidden="true">
-							&#8203;
+								&#8203;
 							</span>
 
 							<div
@@ -233,7 +239,7 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 					className='inline-flex items-center justify-center px-4 py-2 border border-transparent shadow-sm text-sm font-medium rounded-md text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 xl:w-full'
 					onClick={openDeleteModal}
 				>
-        		Delete Experiment
+					Delete Experiment
 				</button>
 			</div>
 			<div className='sm:hidden'>
@@ -282,7 +288,7 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 					null
 				}
 				<p className='flex text-gray-500 text-sm space-x-2'>
-					<span>Uploaded at {new Date(project['created']).toLocaleString()}</span>
+					<span>Uploaded at {new Date(Number(project['created'])).toLocaleString()}</span>
 				</p>
 				{project['startedAtEpochMillis'] ?
 					<p className='flex text-gray-500 text-sm space-x-2'>
@@ -301,3 +307,4 @@ export const ExperimentListing = ({ projectinit, onCopyExperiment, onDownloadRes
 	);
 };
 
+