From 31e8304b0477b1ea9e28622474ddbc9fb3d51207 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 30 Sep 2024 20:38:55 -0400 Subject: [PATCH 01/67] update flask app and mongo module --- apps/backend/app.py | 4 +++- apps/backend/modules/mongo.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 27af335b..84730976 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -6,7 +6,7 @@ from flask import Flask, Response, request, jsonify from kubernetes import client, config import pymongo -from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection +from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments from spawn_runner import create_job, create_job_object flaskApp = Flask(__name__) @@ -29,6 +29,8 @@ ) # connect to the glados database gladosDB = mongoClient["gladosdb"] +# call the function to check if the documents for default experiments exist +check_insert_default_experiments(mongoClient) # setup the mongo collections experimentsCollection = gladosDB.experiments diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 99eafce8..b3bad45e 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -40,4 +40,15 @@ def upload_log_file(experimentId: str, contents: str, mongoClient): resultId = logCollection.insert_one(logFileEntry).inserted_id return resultId except Exception as err: - raise Exception("Encountered error while storing log file in MongoDB") from err \ No newline at end of file + raise Exception("Encountered error while storing log file in MongoDB") from err + +def check_insert_default_experiments(mongoClient: pymongo.MongoClient): + defaultExperimentCollection = mongoClient["gladosdb"].defaultExperiments + count = defaultExperimentCollection.count_documents({"name": "addNums.py"}) + if count == 0: + # this means that the addNums document does not exist + print("adding add nums!!!!!!") + defaultExperimentCollection.insert_one({"name": "addNums.py", + "type": "python", + "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNums.py" + }) \ No newline at end of file From c3b2d4d03011b324b0af5fcb850fb0b0fd792f07 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 30 Sep 2024 20:45:59 -0400 Subject: [PATCH 02/67] Update mongo.py --- apps/backend/modules/mongo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index b3bad45e..686497ef 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -43,6 +43,11 @@ def upload_log_file(experimentId: str, contents: str, mongoClient): raise Exception("Encountered error while storing log file in MongoDB") from err def check_insert_default_experiments(mongoClient: pymongo.MongoClient): + try: + verify_mongo_connection(mongoClient) + except: + # wait somehow... + check_insert_default_experiments(mongoClient) defaultExperimentCollection = mongoClient["gladosdb"].defaultExperiments count = defaultExperimentCollection.count_documents({"name": "addNums.py"}) if count == 0: From 43a5d9d0363a8846dfc5c04e13024cc9a4500e27 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 30 Sep 2024 21:17:42 -0400 Subject: [PATCH 03/67] add all python default experiments --- apps/backend/app.py | 5 ++++- apps/backend/modules/mongo.py | 33 +++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 84730976..ca2ccab1 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -7,6 +7,7 @@ from kubernetes import client, config import pymongo from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments +import threading from spawn_runner import create_job, create_job_object flaskApp = Flask(__name__) @@ -30,7 +31,9 @@ # connect to the glados database gladosDB = mongoClient["gladosdb"] # call the function to check if the documents for default experiments exist -check_insert_default_experiments(mongoClient) +# start that in a different thread so that it can do its thing in peace +addDefaultExpsThread = threading.Thread(target=verify_mongo_connection, args={mongoClient}) +addDefaultExpsThread.start() # setup the mongo collections experimentsCollection = gladosDB.experiments diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 686497ef..7952142f 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -3,14 +3,14 @@ from pymongo.errors import ConnectionFailure from bson import Binary -def verify_mongo_connection(mongoClient): +def verify_mongo_connection(mongoClient: pymongo.MongoClient): try: mongoClient.admin.command('ping') except ConnectionFailure as err: # just use a generic exception raise Exception("MongoDB server not available/unreachable") from err -def upload_experiment_aggregated_results(experimentId: str, results: str, mongoClient): +def upload_experiment_aggregated_results(experimentId: str, results: str, mongoClient: pymongo.MongoClient): experimentResultEntry = {"_id": experimentId, "resultContent": results} # Get the results connection resultsCollection = mongoClient["gladosdb"].results @@ -23,7 +23,7 @@ def upload_experiment_aggregated_results(experimentId: str, results: str, mongoC # Change to generic exception raise Exception("Encountered error while storing aggregated results in MongoDB") from err -def upload_experiment_zip(experimentId: str, encoded: Binary, mongoClient): +def upload_experiment_zip(experimentId: str, encoded: Binary, mongoClient: pymongo.MongoClient): experimentZipEntry = {"_id": experimentId, "fileContent": encoded} zipCollection = mongoClient["gladosdb"].zips try: @@ -33,7 +33,7 @@ def upload_experiment_zip(experimentId: str, encoded: Binary, mongoClient): except Exception as err: raise Exception("Encountered error while storing results zip in MongoDB") from err -def upload_log_file(experimentId: str, contents: str, mongoClient): +def upload_log_file(experimentId: str, contents: str, mongoClient: pymongo.MongoClient): logFileEntry = {"_id": experimentId, "fileContent": contents} logCollection = mongoClient["gladosdb"].logs try: @@ -49,11 +49,20 @@ def check_insert_default_experiments(mongoClient: pymongo.MongoClient): # wait somehow... check_insert_default_experiments(mongoClient) defaultExperimentCollection = mongoClient["gladosdb"].defaultExperiments - count = defaultExperimentCollection.count_documents({"name": "addNums.py"}) - if count == 0: - # this means that the addNums document does not exist - print("adding add nums!!!!!!") - defaultExperimentCollection.insert_one({"name": "addNums.py", - "type": "python", - "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNums.py" - }) \ No newline at end of file + experiments = [ + # python experiments + {"name": "addNums.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNums.py"}, + {"name": "addNumsFailsOnXis1Yis5.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsFailsOnXis1Yis5.py"}, + {"name": "addNumsTimeOutOnXis1Yis5.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsTimeOutOnXis1Yis5.py"}, + {"name": "addNumsTimed.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsTimed.py"}, + {"name": "addNumsWithConstants.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsWithConstants.py"}, + {"name": "alwaysFail.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/alwaysFail.py"}, + {"name": "genetic_algorithm.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/genetic_algorithm.py"} + # C experiments + # Java experiments + ] + + for exp in experiments: + count = defaultExperimentCollection.count_documents({"name": exp["name"]}) + if count == 0: + defaultExperimentCollection.insert_one(exp) \ No newline at end of file From c3b8e40827ab5e744a195eeefa8b3e60049033d4 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 30 Sep 2024 23:19:43 -0400 Subject: [PATCH 04/67] Update mongo.py --- apps/backend/modules/mongo.py | 45 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 7952142f..188486a9 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -43,26 +43,29 @@ def upload_log_file(experimentId: str, contents: str, mongoClient: pymongo.Mongo raise Exception("Encountered error while storing log file in MongoDB") from err def check_insert_default_experiments(mongoClient: pymongo.MongoClient): + # this gets run on its own thread, so let it try to enter the default experiments + def insertExperiments(): + defaultExperimentCollection = mongoClient["gladosdb"].defaultExperiments + experiments = [ + # python experiments + {"name": "addNums.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNums.py"}, + {"name": "addNumsFailsOnXis1Yis5.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsFailsOnXis1Yis5.py"}, + {"name": "addNumsTimeOutOnXis1Yis5.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsTimeOutOnXis1Yis5.py"}, + {"name": "addNumsTimed.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsTimed.py"}, + {"name": "addNumsWithConstants.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsWithConstants.py"}, + {"name": "alwaysFail.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/alwaysFail.py"}, + {"name": "genetic_algorithm.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/genetic_algorithm.py"} + # C experiments + # Java experiments + ] + + for exp in experiments: + count = defaultExperimentCollection.count_documents({"name": exp["name"]}) + if count == 0: + defaultExperimentCollection.insert_one(exp) + try: - verify_mongo_connection(mongoClient) + insertExperiments() except: - # wait somehow... - check_insert_default_experiments(mongoClient) - defaultExperimentCollection = mongoClient["gladosdb"].defaultExperiments - experiments = [ - # python experiments - {"name": "addNums.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNums.py"}, - {"name": "addNumsFailsOnXis1Yis5.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsFailsOnXis1Yis5.py"}, - {"name": "addNumsTimeOutOnXis1Yis5.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsTimeOutOnXis1Yis5.py"}, - {"name": "addNumsTimed.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsTimed.py"}, - {"name": "addNumsWithConstants.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/addNumsWithConstants.py"}, - {"name": "alwaysFail.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/alwaysFail.py"}, - {"name": "genetic_algorithm.py", "type": "python", "url": "https://raw.githubusercontent.com/AutomatingSciencePipeline/Monorepo/refs/heads/main/example_experiments/python/genetic_algorithm.py"} - # C experiments - # Java experiments - ] - - for exp in experiments: - count = defaultExperimentCollection.count_documents({"name": exp["name"]}) - if count == 0: - defaultExperimentCollection.insert_one(exp) \ No newline at end of file + # keep trying + insertExperiments() \ No newline at end of file From f34be2cd68efa8f04335a06c1e173d0e6e05ff34 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 30 Sep 2024 23:30:37 -0400 Subject: [PATCH 05/67] Update mongo.py --- apps/backend/modules/mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 188486a9..66e1d559 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -68,4 +68,4 @@ def insertExperiments(): insertExperiments() except: # keep trying - insertExperiments() \ No newline at end of file + check_insert_default_experiments(mongoClient) \ No newline at end of file From dec25f3ea11b9e67ac28a37ab976bf3497aa3f71 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 30 Sep 2024 23:36:40 -0400 Subject: [PATCH 06/67] Update app.py --- apps/backend/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index ca2ccab1..05127cf1 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -32,7 +32,7 @@ gladosDB = mongoClient["gladosdb"] # call the function to check if the documents for default experiments exist # start that in a different thread so that it can do its thing in peace -addDefaultExpsThread = threading.Thread(target=verify_mongo_connection, args={mongoClient}) +addDefaultExpsThread = threading.Thread(target=check_insert_default_experiments, args={mongoClient}) addDefaultExpsThread.start() # setup the mongo collections From 77c7bdaa9113e2eaa3c25a1f80cd590af953a626 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 1 Oct 2024 14:42:43 -0400 Subject: [PATCH 07/67] fix log message --- apps/runner/modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/runner/modules/utils.py b/apps/runner/modules/utils.py index 6b6f9285..ad1fed10 100644 --- a/apps/runner/modules/utils.py +++ b/apps/runner/modules/utils.py @@ -52,7 +52,7 @@ def upload_experiment_zip(experiment: ExperimentData, encoded: Binary): "experimentId": experiment.expId, "encoded": base64.b64encode(encoded).decode("utf-8") } - _callBackend(url, payload, "inserted result csv into mongodb with id") + _callBackend(url, payload, "inserted zip into mongodb with id") def upload_experiment_log(experimentId: DocumentId): filePath = get_filepath_for_experiment_log(experimentId) From 09e04f0090be18b0482da027e39ebc6451573b4f Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 19:36:14 -0400 Subject: [PATCH 08/67] work on backend and frontend --- apps/backend/app.py | 8 ++++++++ .../AddExperiment/stepComponents/DispatchStep.tsx | 13 +++++++++++++ installation_scripts/Deploy_Cluster.sh | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 05127cf1..edb8b192 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -96,6 +96,14 @@ def check_mongo(): return Response(status=200) except Exception: return Response(status=500) + +@flaskApp.post("/uploadExperimentFile") +def upload_experiment_file(): + json = request.get_json() + file = bson.Binary(base64.b64decode(json['file'])) + userId = json['user'] + print(file) + print(userId) if __name__ == '__main__': flaskApp.run() diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 9c33dcfe..8e519a06 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -5,6 +5,9 @@ import { Group, Text } from '@mantine/core'; import { useAuth } from '../../../../firebase/fbAuth'; import { Upload, FileCode } from 'tabler-icons-react'; import { useState } from 'react'; +import { getEnvVar } from '../../../../utils/env'; + +const BACKEND_PORT = getEnvVar() const SUPPORTED_FILE_TYPES = { 'text/plain': ['.py'], @@ -22,6 +25,16 @@ export const DispatchStep = ({ id, form, ...props }) => { const onDropFile = (files: Parameters[0]) => { setLoading(true); console.log('Submitting Experiment'); + // First we need to write the new experiment to the database + // Store that in JSON + const testResponse = fetch(`http://glados-service-backend:${BACKEND_PORT}/uploadExperimentFile`, { + method: 'POST', + headers: new Headers({ 'Content-Type': 'application/json' }), + credentials: 'same-origin', + body: JSON.stringify({ "file": files[0], "user": userId }), + }); + console.log(testResponse); + submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); const uploadResponse = await uploadExec(expId, files[0]); diff --git a/installation_scripts/Deploy_Cluster.sh b/installation_scripts/Deploy_Cluster.sh index 37177fac..cc636d42 100644 --- a/installation_scripts/Deploy_Cluster.sh +++ b/installation_scripts/Deploy_Cluster.sh @@ -3,7 +3,7 @@ # This script is meant to be run on the server to update the code and restart the docker containers echo "Restarting cubed" -sudo kubeadm reset +sudo kubeadm reset --cri-socket /var/run/containerd/containerd.sock rm -f $HOME/.kube/config sudo kubeadm init --config ../kube_config/kubeadm-config.yaml From bab74625b813b6a34e52d89cc96b55bf15316faf Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 19:40:15 -0400 Subject: [PATCH 09/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 8e519a06..c20f2827 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -7,7 +7,7 @@ import { Upload, FileCode } from 'tabler-icons-react'; import { useState } from 'react'; import { getEnvVar } from '../../../../utils/env'; -const BACKEND_PORT = getEnvVar() +const BACKEND_PORT = getEnvVar("BACKEND_PORT") const SUPPORTED_FILE_TYPES = { 'text/plain': ['.py'], From 0675d15216d3f6c167005e714d2fd2ff5eeffc94 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 19:55:39 -0400 Subject: [PATCH 10/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index c20f2827..bb5c1a10 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -5,9 +5,9 @@ import { Group, Text } from '@mantine/core'; import { useAuth } from '../../../../firebase/fbAuth'; import { Upload, FileCode } from 'tabler-icons-react'; import { useState } from 'react'; -import { getEnvVar } from '../../../../utils/env'; -const BACKEND_PORT = getEnvVar("BACKEND_PORT") +const BACKEND_PORT = process.env.BACKEND_PORT; +console.log(BACKEND_PORT); const SUPPORTED_FILE_TYPES = { 'text/plain': ['.py'], From 48083e9c52b1ea1c02243708d3431eab40cd629c Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 20:03:27 -0400 Subject: [PATCH 11/67] Update Dev_pull_repo.sh --- development_scripts/Dev_pull_repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development_scripts/Dev_pull_repo.sh b/development_scripts/Dev_pull_repo.sh index 8c2893b0..dc7d62fe 100644 --- a/development_scripts/Dev_pull_repo.sh +++ b/development_scripts/Dev_pull_repo.sh @@ -13,4 +13,4 @@ git checkout development git pull origin development sed -i 's/glados-backend:main/glados-backend:development/g' ./kubernetes_init/backend/deployment-backend.yaml && echo 'Sucessfully renamed backend image!' -sed -i 's/glados-frontend:mainglados-frontend:development/g' ./kubernetes_init/frontend/deployment-frontend.yaml && echo 'Sucessfully renamed frontend image!' +sed -i 's/glados-frontend:main/glados-frontend:development/g' ./kubernetes_init/frontend/deployment-frontend.yaml && echo 'Sucessfully renamed frontend image!' From 986a47dfcf67d2274b8d282cfe15f867b59be740 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 20:09:50 -0400 Subject: [PATCH 12/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index bb5c1a10..8f33c25a 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -27,6 +27,7 @@ export const DispatchStep = ({ id, form, ...props }) => { console.log('Submitting Experiment'); // First we need to write the new experiment to the database // Store that in JSON + console.log("Testing backend upload..."); const testResponse = fetch(`http://glados-service-backend:${BACKEND_PORT}/uploadExperimentFile`, { method: 'POST', headers: new Headers({ 'Content-Type': 'application/json' }), From 741ff0ff36b6a6d21920cb3532c22023ca440cc2 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 20:15:50 -0400 Subject: [PATCH 13/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 02d96932..d618038d 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -18,6 +18,7 @@ spec: containers: - name: container-test-frontend image: gladospipeline/glados-frontend:main + imagePullPolicy: Always ports: - containerPort: 0000 # hostPort: 0000 From 26295b1b1856d0e0dc06a7558f9f083ab4063acb Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 20:21:57 -0400 Subject: [PATCH 14/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 8f33c25a..74e7507d 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -6,7 +6,7 @@ import { useAuth } from '../../../../firebase/fbAuth'; import { Upload, FileCode } from 'tabler-icons-react'; import { useState } from 'react'; -const BACKEND_PORT = process.env.BACKEND_PORT; +const BACKEND_PORT = process.env; console.log(BACKEND_PORT); const SUPPORTED_FILE_TYPES = { From ca095141c695572f9b9aa27dc06606e1ec92536c Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 20:38:31 -0400 Subject: [PATCH 15/67] fix --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 6 ++++-- kubernetes_init/frontend/deployment-frontend.yaml | 7 +++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 74e7507d..ed6fabd2 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -5,9 +5,9 @@ import { Group, Text } from '@mantine/core'; import { useAuth } from '../../../../firebase/fbAuth'; import { Upload, FileCode } from 'tabler-icons-react'; import { useState } from 'react'; +import { env } from 'process'; + -const BACKEND_PORT = process.env; -console.log(BACKEND_PORT); const SUPPORTED_FILE_TYPES = { 'text/plain': ['.py'], @@ -21,6 +21,8 @@ const SUPPORTED_FILE_TYPES = { export const DispatchStep = ({ id, form, ...props }) => { const { userId } = useAuth(); const [loading, setLoading] = useState(false); + const BACKEND_PORT = env.BACKEND_PORT; + console.log(BACKEND_PORT); const onDropFile = (files: Parameters[0]) => { setLoading(true); diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index d618038d..5af46f79 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -19,9 +19,6 @@ spec: - name: container-test-frontend image: gladospipeline/glados-frontend:main imagePullPolicy: Always - ports: - - containerPort: 0000 - # hostPort: 0000 env: - name: BACKEND_PORT valueFrom: @@ -58,4 +55,6 @@ spec: secretKeyRef: name: secret-env key: MONGODB_PASSWORD - \ No newline at end of file + ports: + - containerPort: 0000 + # hostPort: 0000 \ No newline at end of file From 5da615934be5f3cdc91a3bea4059f0ce5ce37878 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 21:07:19 -0400 Subject: [PATCH 16/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index ed6fabd2..0c14fc82 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -21,7 +21,7 @@ const SUPPORTED_FILE_TYPES = { export const DispatchStep = ({ id, form, ...props }) => { const { userId } = useAuth(); const [loading, setLoading] = useState(false); - const BACKEND_PORT = env.BACKEND_PORT; + const BACKEND_PORT = env; console.log(BACKEND_PORT); const onDropFile = (files: Parameters[0]) => { From 4a18c8243d04e7901c24d03d3f890386334a3573 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 21:18:14 -0400 Subject: [PATCH 17/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 0c14fc82..9d090c8c 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -21,7 +21,7 @@ const SUPPORTED_FILE_TYPES = { export const DispatchStep = ({ id, form, ...props }) => { const { userId } = useAuth(); const [loading, setLoading] = useState(false); - const BACKEND_PORT = env; + const BACKEND_PORT = env.GLADOS_SERVICE_BACKEND_SERVICE_PORT; console.log(BACKEND_PORT); const onDropFile = (files: Parameters[0]) => { From 49fbc7870be2d428592bd66fcec1018c49a678cd Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 21:23:39 -0400 Subject: [PATCH 18/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 5af46f79..6b004c4b 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -56,5 +56,5 @@ spec: name: secret-env key: MONGODB_PASSWORD ports: - - containerPort: 0000 + - containerPort: 3000 # hostPort: 0000 \ No newline at end of file From b27ffb347319bb747b9099113d6a0eae15c40c34 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 21:26:56 -0400 Subject: [PATCH 19/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 6b004c4b..5af46f79 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -56,5 +56,5 @@ spec: name: secret-env key: MONGODB_PASSWORD ports: - - containerPort: 3000 + - containerPort: 0000 # hostPort: 0000 \ No newline at end of file From 8d13f384d194c23332836ffbe9a31f31be45b965 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 21:48:01 -0400 Subject: [PATCH 20/67] stupid env file --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- kubernetes_init/frontend/deployment-frontend.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 9d090c8c..ed6fabd2 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -21,7 +21,7 @@ const SUPPORTED_FILE_TYPES = { export const DispatchStep = ({ id, form, ...props }) => { const { userId } = useAuth(); const [loading, setLoading] = useState(false); - const BACKEND_PORT = env.GLADOS_SERVICE_BACKEND_SERVICE_PORT; + const BACKEND_PORT = env.BACKEND_PORT; console.log(BACKEND_PORT); const onDropFile = (files: Parameters[0]) => { diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 5af46f79..19539fb4 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -20,7 +20,7 @@ spec: image: gladospipeline/glados-frontend:main imagePullPolicy: Always env: - - name: BACKEND_PORT + - name: NEXT_PUBLIC_BACKEND_PORT valueFrom: secretKeyRef: name: secret-env From 626640db8229fe20b92b386a51cfb0539f44c392 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Wed, 2 Oct 2024 21:59:17 -0400 Subject: [PATCH 21/67] hard code --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index ed6fabd2..db82d2e3 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -5,9 +5,6 @@ import { Group, Text } from '@mantine/core'; import { useAuth } from '../../../../firebase/fbAuth'; import { Upload, FileCode } from 'tabler-icons-react'; import { useState } from 'react'; -import { env } from 'process'; - - const SUPPORTED_FILE_TYPES = { 'text/plain': ['.py'], @@ -21,8 +18,8 @@ const SUPPORTED_FILE_TYPES = { export const DispatchStep = ({ id, form, ...props }) => { const { userId } = useAuth(); const [loading, setLoading] = useState(false); - const BACKEND_PORT = env.BACKEND_PORT; - console.log(BACKEND_PORT); + //TODO: Fix the environment variable, its broken and won't work here for some reason + const BACKEND_PORT = "5050"; const onDropFile = (files: Parameters[0]) => { setLoading(true); From 4a1f8d1a08c53de7a1f17804e4b1b01428499902 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Thu, 3 Oct 2024 14:11:57 -0400 Subject: [PATCH 22/67] try to use github cache --- .github/workflows/publish-frontend-docker-development.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-frontend-docker-development.yml b/.github/workflows/publish-frontend-docker-development.yml index 982b29e0..225101f8 100644 --- a/.github/workflows/publish-frontend-docker-development.yml +++ b/.github/workflows/publish-frontend-docker-development.yml @@ -53,6 +53,8 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 From 8e577b1f14be5df2239d590a49998deb80dd1516 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Thu, 3 Oct 2024 14:12:51 -0400 Subject: [PATCH 23/67] try again --- .github/workflows/publish-frontend-docker-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-frontend-docker-development.yml b/.github/workflows/publish-frontend-docker-development.yml index 225101f8..5bce6db5 100644 --- a/.github/workflows/publish-frontend-docker-development.yml +++ b/.github/workflows/publish-frontend-docker-development.yml @@ -46,7 +46,7 @@ jobs: - name: Build and push Docker image id: push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: context: ./apps/frontend file: ./apps/frontend/frontend.Dockerfile From f5b679620f551eaf306ede5611bc36f8a82ebf82 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Thu, 3 Oct 2024 14:21:13 -0400 Subject: [PATCH 24/67] add buildx for caching --- .github/workflows/publish-frontend-docker-development.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-frontend-docker-development.yml b/.github/workflows/publish-frontend-docker-development.yml index 5bce6db5..353a7f1d 100644 --- a/.github/workflows/publish-frontend-docker-development.yml +++ b/.github/workflows/publish-frontend-docker-development.yml @@ -32,6 +32,9 @@ jobs: with: ref: development + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log in to Docker Hub uses: docker/login-action@v2 with: From 520ddab189d1170535c688868ca05017022c0bba Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Thu, 3 Oct 2024 14:25:37 -0400 Subject: [PATCH 25/67] change to only run on change to frontend --- .github/workflows/publish-frontend-docker-development.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-frontend-docker-development.yml b/.github/workflows/publish-frontend-docker-development.yml index 353a7f1d..0d43a306 100644 --- a/.github/workflows/publish-frontend-docker-development.yml +++ b/.github/workflows/publish-frontend-docker-development.yml @@ -11,6 +11,8 @@ name: Publish Dev Frontend Docker image on: push: + paths: + - 'apps/frontend/**' branches: ['development'] env: From be422a900dd7219c81f41c149e07e233c95a3372 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Thu, 3 Oct 2024 14:27:31 -0400 Subject: [PATCH 26/67] update all docker build and push --- .github/workflows/publish-backend-docker-development.yml | 2 ++ .github/workflows/publish-backend-docker.yml | 2 ++ .github/workflows/publish-frontend-docker.yml | 2 ++ .github/workflows/publish-runner-docker-development.yml | 2 ++ .github/workflows/publish-runner-docker.yml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/.github/workflows/publish-backend-docker-development.yml b/.github/workflows/publish-backend-docker-development.yml index 9d6fa3be..07c9ea5c 100644 --- a/.github/workflows/publish-backend-docker-development.yml +++ b/.github/workflows/publish-backend-docker-development.yml @@ -11,6 +11,8 @@ name: Publish Dev Backend Docker image on: push: + paths: + - 'apps/backend/**' branches: ['development'] env: diff --git a/.github/workflows/publish-backend-docker.yml b/.github/workflows/publish-backend-docker.yml index d804cd2c..4ff1c55e 100644 --- a/.github/workflows/publish-backend-docker.yml +++ b/.github/workflows/publish-backend-docker.yml @@ -11,6 +11,8 @@ name: Publish Production Backend Docker image on: push: + paths: + - 'apps/backend/**' branches: ['main'] env: diff --git a/.github/workflows/publish-frontend-docker.yml b/.github/workflows/publish-frontend-docker.yml index 91c15198..c2c2ab1f 100644 --- a/.github/workflows/publish-frontend-docker.yml +++ b/.github/workflows/publish-frontend-docker.yml @@ -11,6 +11,8 @@ name: Publish Production Frontend Docker image on: push: + paths: + - 'apps/frontend/**' branches: ['main'] env: diff --git a/.github/workflows/publish-runner-docker-development.yml b/.github/workflows/publish-runner-docker-development.yml index 8b9a81ed..a81c4e38 100644 --- a/.github/workflows/publish-runner-docker-development.yml +++ b/.github/workflows/publish-runner-docker-development.yml @@ -11,6 +11,8 @@ name: Publish Dev Runner Docker image on: push: + paths: + - 'apps/runner/**' branches: ['development'] env: diff --git a/.github/workflows/publish-runner-docker.yml b/.github/workflows/publish-runner-docker.yml index 4621dc03..80dcfb01 100644 --- a/.github/workflows/publish-runner-docker.yml +++ b/.github/workflows/publish-runner-docker.yml @@ -11,6 +11,8 @@ name: Publish Production Runner Docker image on: push: + paths: + - 'apps/runner/**' branches: ['main'] env: From db5b9f6a569d103527503a2172cd633c2835f454 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 15:26:50 -0400 Subject: [PATCH 27/67] update deploy script and app.py --- apps/backend/app.py | 1 + installation_scripts/Deploy_Cluster.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index edb8b192..e3f6dfff 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -104,6 +104,7 @@ def upload_experiment_file(): userId = json['user'] print(file) print(userId) + return Response(status=200) if __name__ == '__main__': flaskApp.run() diff --git a/installation_scripts/Deploy_Cluster.sh b/installation_scripts/Deploy_Cluster.sh index cc636d42..488e40cb 100644 --- a/installation_scripts/Deploy_Cluster.sh +++ b/installation_scripts/Deploy_Cluster.sh @@ -3,7 +3,7 @@ # This script is meant to be run on the server to update the code and restart the docker containers echo "Restarting cubed" -sudo kubeadm reset --cri-socket /var/run/containerd/containerd.sock +sudo kubeadm reset --cri-socket unix:///var/run/containerd/containerd.sock rm -f $HOME/.kube/config sudo kubeadm init --config ../kube_config/kubeadm-config.yaml From 7600e0c0c88451aff4b7601c2c3bfcb027ac66ac Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 15:32:31 -0400 Subject: [PATCH 28/67] add debugging to dispatch step --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index db82d2e3..a95366cd 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -27,11 +27,13 @@ export const DispatchStep = ({ id, form, ...props }) => { // First we need to write the new experiment to the database // Store that in JSON console.log("Testing backend upload..."); + var jsonPayload = JSON.stringify({ "file": files[0], "user": userId }); + console.log(jsonPayload); const testResponse = fetch(`http://glados-service-backend:${BACKEND_PORT}/uploadExperimentFile`, { method: 'POST', headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', - body: JSON.stringify({ "file": files[0], "user": userId }), + body: jsonPayload, }); console.log(testResponse); From b5bb14340732c8f5e77c390d6fb188057c1df939 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 15:36:47 -0400 Subject: [PATCH 29/67] update docker file for faster build maybe --- .../stepComponents/DispatchStep.tsx | 3 +- apps/frontend/frontend.Dockerfile | 49 +++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index a95366cd..1f78e5e1 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -20,12 +20,13 @@ export const DispatchStep = ({ id, form, ...props }) => { const [loading, setLoading] = useState(false); //TODO: Fix the environment variable, its broken and won't work here for some reason const BACKEND_PORT = "5050"; + console.log(BACKEND_PORT); const onDropFile = (files: Parameters[0]) => { setLoading(true); console.log('Submitting Experiment'); // First we need to write the new experiment to the database - // Store that in JSON + // Store that in JSON console.log("Testing backend upload..."); var jsonPayload = JSON.stringify({ "file": files[0], "user": userId }); console.log(jsonPayload); diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index 8b66053e..3fc64f80 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -1,13 +1,56 @@ +# FROM node:20.6 AS base + +# WORKDIR /app + +# COPY . /app + +# RUN npm install + +# RUN npm run build + +# EXPOSE $FRONTEND_WEBSERVER_PORT + +# CMD ["npm", "start"] + FROM node:20.6 AS base +FROM base AS deps + +RUN apk add --no-cache libc6-compat WORKDIR /app -COPY . /app +COPY package.json ./ + +RUN npm update && npm install + +# If you want yarn update and install uncomment the bellow -RUN npm install +# RUN yarn install && yarn upgrade + +FROM base AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY . . RUN npm run build +FROM base AS runner +WORKDIR /app + +ENV NODE_ENV production +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +COPY --from=builder /app/public ./public + +RUN mkdir .next +RUN chown nextjs:nodejs .next + +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static + +USER nextjs + EXPOSE $FRONTEND_WEBSERVER_PORT -CMD ["npm", "start"] \ No newline at end of file +CMD ["node", "server.js"] \ No newline at end of file From 43197a6c805160ec3dd8090cd37b1bcb97c95df8 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 15:40:09 -0400 Subject: [PATCH 30/67] Update frontend.Dockerfile --- apps/frontend/frontend.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index 3fc64f80..d3113d45 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -12,7 +12,7 @@ # CMD ["npm", "start"] -FROM node:20.6 AS base +FROM node:20-alpine AS base FROM base AS deps From 0cdcec3504ed80c3c8b8d07fb05105dcd45bd204 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 15:50:09 -0400 Subject: [PATCH 31/67] revert dispatch step, I broke it --- .../stepComponents/DispatchStep.tsx | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 1f78e5e1..9c33dcfe 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -18,26 +18,10 @@ const SUPPORTED_FILE_TYPES = { export const DispatchStep = ({ id, form, ...props }) => { const { userId } = useAuth(); const [loading, setLoading] = useState(false); - //TODO: Fix the environment variable, its broken and won't work here for some reason - const BACKEND_PORT = "5050"; - console.log(BACKEND_PORT); const onDropFile = (files: Parameters[0]) => { setLoading(true); console.log('Submitting Experiment'); - // First we need to write the new experiment to the database - // Store that in JSON - console.log("Testing backend upload..."); - var jsonPayload = JSON.stringify({ "file": files[0], "user": userId }); - console.log(jsonPayload); - const testResponse = fetch(`http://glados-service-backend:${BACKEND_PORT}/uploadExperimentFile`, { - method: 'POST', - headers: new Headers({ 'Content-Type': 'application/json' }), - credentials: 'same-origin', - body: jsonPayload, - }); - console.log(testResponse); - submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); const uploadResponse = await uploadExec(expId, files[0]); From b43c029bd645f4690d260fffdf88c33e15d8c5c1 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 17:17:25 -0400 Subject: [PATCH 32/67] update dockerfile and yaml for frontend --- apps/frontend/frontend.Dockerfile | 5 ++++- kubernetes_init/frontend/deployment-frontend.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index d3113d45..ce47a8a2 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -23,6 +23,9 @@ COPY package.json ./ RUN npm update && npm install +# Install this to optimize images +RUN npm i sharp + # If you want yarn update and install uncomment the bellow # RUN yarn install && yarn upgrade @@ -37,7 +40,7 @@ RUN npm run build FROM base AS runner WORKDIR /app -ENV NODE_ENV production +ENV NODE_ENV=production RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 19539fb4..5af46f79 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -20,7 +20,7 @@ spec: image: gladospipeline/glados-frontend:main imagePullPolicy: Always env: - - name: NEXT_PUBLIC_BACKEND_PORT + - name: BACKEND_PORT valueFrom: secretKeyRef: name: secret-env From 9667d671958c92dc1d7c6ed3fa2150d95d61a202 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 17:50:10 -0400 Subject: [PATCH 33/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 5af46f79..02d96932 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -18,7 +18,9 @@ spec: containers: - name: container-test-frontend image: gladospipeline/glados-frontend:main - imagePullPolicy: Always + ports: + - containerPort: 0000 + # hostPort: 0000 env: - name: BACKEND_PORT valueFrom: @@ -55,6 +57,4 @@ spec: secretKeyRef: name: secret-env key: MONGODB_PASSWORD - ports: - - containerPort: 0000 - # hostPort: 0000 \ No newline at end of file + \ No newline at end of file From 958672716b672b0cbdaf9a65aed2cd9c78b18a34 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 18:00:55 -0400 Subject: [PATCH 34/67] Update db.ts --- apps/frontend/firebase/db.ts | 47 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/apps/frontend/firebase/db.ts b/apps/frontend/firebase/db.ts index f060bf88..e1207aeb 100644 --- a/apps/frontend/firebase/db.ts +++ b/apps/frontend/firebase/db.ts @@ -5,6 +5,9 @@ import { collection, setDoc, doc, query, where, onSnapshot } from 'firebase/fire import { getDownloadURL, getStorage, ref, uploadBytes } from 'firebase/storage'; import { ExperimentData } from './db_types'; import { ResultsCsv, ProjectZip } from '../lib/mongodb_types'; +import { getEnvVar } from '../utils/env'; + +const BACKEND_PORT = getEnvVar('BACKEND_PORT'); export const DB_COLLECTION_EXPERIMENTS = 'Experiments'; @@ -49,23 +52,37 @@ export const submitExperiment = async (values: Partial, userId: return newExperimentDocument.id; }; - +//TODO: Refactor to use MongoDB export const uploadExec = async (id: ExperimentDocumentId, file) => { - const fileRef = ref(storage, `experiment${id}`); - return await uploadBytes(fileRef, file).then((snapshot) => { - console.log('Uploaded file. Updating doc...'); - const experimentRef = doc(db, DB_COLLECTION_EXPERIMENTS, id); - updateDoc(experimentRef, { - file: `experiment${id}`, - }).then(() => { - console.log(`Uploaded file for experiment ${id}`); - return true; - }).catch((error) => console.log('Upload doc error: ', error)); - return true; - }).catch((error) => { - console.log('Upload bytes error: ', error); - return false; + //Create the payload to send to the backend API + console.log("Inside uploadEXEC!"); + console.log(`File is: ${file}`); + + var url = `http://glados-service-backend:${BACKEND_PORT}/experiment` + var payload = JSON.stringify({"id": id, "file": file}); + const response = await fetch(``, { + method: 'POST', + headers: new Headers({ 'Content-Type': 'application/json' }), + body: payload, }); + console.log(response); + return response; + + // const fileRef = ref(storage, `experiment${id}`); + // return await uploadBytes(fileRef, file).then((snapshot) => { + // console.log('Uploaded file. Updating doc...'); + // const experimentRef = doc(db, DB_COLLECTION_EXPERIMENTS, id); + // updateDoc(experimentRef, { + // file: `experiment${id}`, + // }).then(() => { + // console.log(`Uploaded file for experiment ${id}`); + // return true; + // }).catch((error) => console.log('Upload doc error: ', error)); + // return true; + // }).catch((error) => { + // console.log('Upload bytes error: ', error); + // return false; + // }); }; const downloadArbitraryFile = (url: string, name: string) => { From a72d1aaf3233e5a3f54010be5dacb64736960205 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Thu, 3 Oct 2024 18:07:19 -0400 Subject: [PATCH 35/67] try to make public --- apps/frontend/firebase/db.ts | 2 +- kubernetes_init/frontend/deployment-frontend.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/frontend/firebase/db.ts b/apps/frontend/firebase/db.ts index e1207aeb..0fdaa134 100644 --- a/apps/frontend/firebase/db.ts +++ b/apps/frontend/firebase/db.ts @@ -7,7 +7,7 @@ import { ExperimentData } from './db_types'; import { ResultsCsv, ProjectZip } from '../lib/mongodb_types'; import { getEnvVar } from '../utils/env'; -const BACKEND_PORT = getEnvVar('BACKEND_PORT'); +const BACKEND_PORT = getEnvVar('NEXT_PUBLIC_BACKEND_PORT'); export const DB_COLLECTION_EXPERIMENTS = 'Experiments'; diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 02d96932..fcf2a280 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -27,6 +27,11 @@ spec: secretKeyRef: name: secret-env key: BACKEND_PORT + - name: NEXT_PUBLIC_BACKEND_PORT + valueFrom: + secretKeyRef: + name: secret-env + key: BACKEND_PORT - name: FRONTEND_WEBSERVER_PORT valueFrom: secretKeyRef: From f625177cafcfd169637ca9fa539b0f53db27667d Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Mon, 7 Oct 2024 15:19:39 -0400 Subject: [PATCH 36/67] add new api call --- .../stepComponents/DispatchStep.tsx | 7 ++++- apps/frontend/lib/mongodb.ts | 1 + apps/frontend/pages/api/files/uploadFile.tsx | 31 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 apps/frontend/pages/api/files/uploadFile.tsx diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 9c33dcfe..f7529c6c 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -24,7 +24,12 @@ export const DispatchStep = ({ id, form, ...props }) => { console.log('Submitting Experiment'); submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); - const uploadResponse = await uploadExec(expId, files[0]); + const uploadResponse = await fetch('/api/files/uploadFile', { + method: 'POST', + headers: new Headers({ 'Content-Type': 'application/json' }), + credentials: 'same-origin', + body: JSON.stringify({file: files[0]}) + }) if (uploadResponse) { console.log(`Handing experiment ${expId} to the backend`); const response = await fetch(`/api/experiments/${expId}`, { diff --git a/apps/frontend/lib/mongodb.ts b/apps/frontend/lib/mongodb.ts index 884b58e6..57dedfcb 100644 --- a/apps/frontend/lib/mongodb.ts +++ b/apps/frontend/lib/mongodb.ts @@ -15,6 +15,7 @@ export const DB_NAME = 'gladosdb'; export const COLLECTION_LOGS = 'logs'; export const COLLECTION_ZIPS = 'zips'; export const COLLECTION_RESULTS_CSVS = 'results'; +export const COLLECTION_EXPERIMENT_FILES = 'files'; let client: MongoClient; let clientPromise: Promise = new Promise((success) => { diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx new file mode 100644 index 00000000..05d954da --- /dev/null +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -0,0 +1,31 @@ +import clientPromise, { DB_NAME, COLLECTION_RESULTS_CSVS, COLLECTION_EXPERIMENT_FILES } from '../../../lib/mongodb'; +import { NextApiHandler } from 'next'; +import { ResultsCsv } from '../../../lib/mongodb_types'; + +const mongoFileUploader: NextApiHandler = async (req, res) => { + const { fileToUpload } = req.query; + if (typeof fileToUpload !== 'string'){ + res.status(400).json({response: 'File is not in the correct format'} as any); + return; + } + + try{ + const client = await clientPromise; + const db = client.db(DB_NAME); + + // Create the doc we want to write + const doc = {"file": Buffer.from(fileToUpload, 'base64')}; + const newDoc = await db.collection(COLLECTION_EXPERIMENT_FILES).insertOne(doc); + + res.status(200).json({response: `Successfully wrote file to document ${newDoc.insertedId}`} as any); + return; + } + catch (error){ + const message = "Failed to upload experiment file!"; + console.error("Error writing experiment file."); + res.status(500).json({ response: message } as any); + } +} + + +export default mongoFileUploader; From 1abc958f174e325c0a267381f4470399a7485605 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Mon, 7 Oct 2024 15:26:51 -0400 Subject: [PATCH 37/67] undo some changes --- apps/frontend/firebase/db.ts | 47 ++++++------------- apps/frontend/pages/api/files/uploadFile.tsx | 1 - .../frontend/deployment-frontend.yaml | 5 -- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/apps/frontend/firebase/db.ts b/apps/frontend/firebase/db.ts index 0fdaa134..f060bf88 100644 --- a/apps/frontend/firebase/db.ts +++ b/apps/frontend/firebase/db.ts @@ -5,9 +5,6 @@ import { collection, setDoc, doc, query, where, onSnapshot } from 'firebase/fire import { getDownloadURL, getStorage, ref, uploadBytes } from 'firebase/storage'; import { ExperimentData } from './db_types'; import { ResultsCsv, ProjectZip } from '../lib/mongodb_types'; -import { getEnvVar } from '../utils/env'; - -const BACKEND_PORT = getEnvVar('NEXT_PUBLIC_BACKEND_PORT'); export const DB_COLLECTION_EXPERIMENTS = 'Experiments'; @@ -52,37 +49,23 @@ export const submitExperiment = async (values: Partial, userId: return newExperimentDocument.id; }; -//TODO: Refactor to use MongoDB + export const uploadExec = async (id: ExperimentDocumentId, file) => { - //Create the payload to send to the backend API - console.log("Inside uploadEXEC!"); - console.log(`File is: ${file}`); - - var url = `http://glados-service-backend:${BACKEND_PORT}/experiment` - var payload = JSON.stringify({"id": id, "file": file}); - const response = await fetch(``, { - method: 'POST', - headers: new Headers({ 'Content-Type': 'application/json' }), - body: payload, + const fileRef = ref(storage, `experiment${id}`); + return await uploadBytes(fileRef, file).then((snapshot) => { + console.log('Uploaded file. Updating doc...'); + const experimentRef = doc(db, DB_COLLECTION_EXPERIMENTS, id); + updateDoc(experimentRef, { + file: `experiment${id}`, + }).then(() => { + console.log(`Uploaded file for experiment ${id}`); + return true; + }).catch((error) => console.log('Upload doc error: ', error)); + return true; + }).catch((error) => { + console.log('Upload bytes error: ', error); + return false; }); - console.log(response); - return response; - - // const fileRef = ref(storage, `experiment${id}`); - // return await uploadBytes(fileRef, file).then((snapshot) => { - // console.log('Uploaded file. Updating doc...'); - // const experimentRef = doc(db, DB_COLLECTION_EXPERIMENTS, id); - // updateDoc(experimentRef, { - // file: `experiment${id}`, - // }).then(() => { - // console.log(`Uploaded file for experiment ${id}`); - // return true; - // }).catch((error) => console.log('Upload doc error: ', error)); - // return true; - // }).catch((error) => { - // console.log('Upload bytes error: ', error); - // return false; - // }); }; const downloadArbitraryFile = (url: string, name: string) => { diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 05d954da..0ad3e4dd 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -1,6 +1,5 @@ import clientPromise, { DB_NAME, COLLECTION_RESULTS_CSVS, COLLECTION_EXPERIMENT_FILES } from '../../../lib/mongodb'; import { NextApiHandler } from 'next'; -import { ResultsCsv } from '../../../lib/mongodb_types'; const mongoFileUploader: NextApiHandler = async (req, res) => { const { fileToUpload } = req.query; diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index fcf2a280..02d96932 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -27,11 +27,6 @@ spec: secretKeyRef: name: secret-env key: BACKEND_PORT - - name: NEXT_PUBLIC_BACKEND_PORT - valueFrom: - secretKeyRef: - name: secret-env - key: BACKEND_PORT - name: FRONTEND_WEBSERVER_PORT valueFrom: secretKeyRef: From 5f6927354620e537e727653215fd14ed48a130e9 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Tue, 8 Oct 2024 12:34:35 -0400 Subject: [PATCH 38/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 02d96932..d618038d 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -18,6 +18,7 @@ spec: containers: - name: container-test-frontend image: gladospipeline/glados-frontend:main + imagePullPolicy: Always ports: - containerPort: 0000 # hostPort: 0000 From 3c40d913f319acaf76e59c63166d9c71ba4b61c0 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 14:26:02 -0400 Subject: [PATCH 39/67] test GridFS --- .../stepComponents/DispatchStep.tsx | 5 +- apps/frontend/pages/api/files/uploadFile.tsx | 58 ++++++++++++------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index f7529c6c..57034457 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -28,7 +28,10 @@ export const DispatchStep = ({ id, form, ...props }) => { method: 'POST', headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', - body: JSON.stringify({file: files[0]}) + body: JSON.stringify({ + fileToUpload: files[0], + experimentId: expId + }) }) if (uploadResponse) { console.log(`Handing experiment ${expId} to the backend`); diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 0ad3e4dd..dc515568 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -1,28 +1,44 @@ import clientPromise, { DB_NAME, COLLECTION_RESULTS_CSVS, COLLECTION_EXPERIMENT_FILES } from '../../../lib/mongodb'; import { NextApiHandler } from 'next'; +import { GridFSBucket } from 'mongodb'; +import { Readable } from 'stream'; const mongoFileUploader: NextApiHandler = async (req, res) => { - const { fileToUpload } = req.query; - if (typeof fileToUpload !== 'string'){ - res.status(400).json({response: 'File is not in the correct format'} as any); - return; - } - - try{ - const client = await clientPromise; - const db = client.db(DB_NAME); - - // Create the doc we want to write - const doc = {"file": Buffer.from(fileToUpload, 'base64')}; - const newDoc = await db.collection(COLLECTION_EXPERIMENT_FILES).insertOne(doc); - - res.status(200).json({response: `Successfully wrote file to document ${newDoc.insertedId}`} as any); - return; - } - catch (error){ - const message = "Failed to upload experiment file!"; - console.error("Error writing experiment file."); - res.status(500).json({ response: message } as any); + if (req.method === 'POST') { + const { fileToUpload, experimentId } = req.body; + + if (!fileToUpload || !experimentId) { + return res.status(400).json({ response: "Not enough arguments!" } as any); + } + + if (typeof fileToUpload !== 'string') { + res.status(400).json({ response: 'File is not in the correct format' } as any); + return; + } + + try { + const client = await clientPromise; + const db = client.db(DB_NAME); + const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' }); + + const readableStream = new Readable(); + readableStream.push(fileToUpload); + readableStream.push(null); + + readableStream. + pipe(bucket.openUploadStream(`experimentFile${experimentId}`, { + chunkSizeBytes: 1048576, + metadata: { field: 'expId', value: experimentId } + }) as any); + + res.status(200).json({ response: 'Successfully wrote file!' } as any); + return; + } + catch (error) { + const message = "Failed to upload experiment file!"; + console.error("Error writing experiment file."); + res.status(500).json({ response: message } as any); + } } } From a22e750510c64fc9ec45f777a19209f85ae7881e Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 14:27:34 -0400 Subject: [PATCH 40/67] force actions --- apps/frontend/pages/api/files/uploadFile.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index dc515568..00526af0 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -30,6 +30,7 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { chunkSizeBytes: 1048576, metadata: { field: 'expId', value: experimentId } }) as any); + res.status(200).json({ response: 'Successfully wrote file!' } as any); return; From 378be255941a401e5a9096833a6f20c078a0ae44 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 14:33:52 -0400 Subject: [PATCH 41/67] try different frontend dockerfile --- apps/frontend/frontend.Dockerfile | 70 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index ce47a8a2..c7c44bb5 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -1,59 +1,59 @@ -# FROM node:20.6 AS base +FROM node:20.6 AS base -# WORKDIR /app +WORKDIR /app -# COPY . /app +COPY . /app -# RUN npm install +RUN npm install -# RUN npm run build +RUN npm run build -# EXPOSE $FRONTEND_WEBSERVER_PORT +EXPOSE $FRONTEND_WEBSERVER_PORT -# CMD ["npm", "start"] +CMD ["npm", "start"] -FROM node:20-alpine AS base +# FROM node:20-alpine AS base -FROM base AS deps +# FROM base AS deps -RUN apk add --no-cache libc6-compat -WORKDIR /app +# RUN apk add --no-cache libc6-compat +# WORKDIR /app -COPY package.json ./ +# COPY package.json ./ -RUN npm update && npm install +# RUN npm update && npm install -# Install this to optimize images -RUN npm i sharp +# # Install this to optimize images +# RUN npm i sharp -# If you want yarn update and install uncomment the bellow +# # If you want yarn update and install uncomment the bellow -# RUN yarn install && yarn upgrade +# # RUN yarn install && yarn upgrade -FROM base AS builder -WORKDIR /app -COPY --from=deps /app/node_modules ./node_modules -COPY . . +# FROM base AS builder +# WORKDIR /app +# COPY --from=deps /app/node_modules ./node_modules +# COPY . . -RUN npm run build +# RUN npm run build -FROM base AS runner -WORKDIR /app +# FROM base AS runner +# WORKDIR /app -ENV NODE_ENV=production -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nextjs +# ENV NODE_ENV=production +# RUN addgroup --system --gid 1001 nodejs +# RUN adduser --system --uid 1001 nextjs -COPY --from=builder /app/public ./public +# COPY --from=builder /app/public ./public -RUN mkdir .next -RUN chown nextjs:nodejs .next +# RUN mkdir .next +# RUN chown nextjs:nodejs .next -COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static +# COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +# COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -USER nextjs +# USER nextjs -EXPOSE $FRONTEND_WEBSERVER_PORT +# EXPOSE $FRONTEND_WEBSERVER_PORT -CMD ["node", "server.js"] \ No newline at end of file +# CMD ["node", "server.js"] \ No newline at end of file From c290b50c401857e8ee5ff88e3216b1865dac6e7a Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 14:46:10 -0400 Subject: [PATCH 42/67] update dispatch step --- .../stepComponents/DispatchStep.tsx | 4 +- apps/frontend/frontend.Dockerfile | 70 +++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 57034457..3de2fd49 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -29,8 +29,8 @@ export const DispatchStep = ({ id, form, ...props }) => { headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', body: JSON.stringify({ - fileToUpload: files[0], - experimentId: expId + "fileToUpload": files[0], + "experimentId": expId }) }) if (uploadResponse) { diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index c7c44bb5..ce47a8a2 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -1,59 +1,59 @@ -FROM node:20.6 AS base +# FROM node:20.6 AS base -WORKDIR /app +# WORKDIR /app -COPY . /app +# COPY . /app -RUN npm install +# RUN npm install -RUN npm run build +# RUN npm run build -EXPOSE $FRONTEND_WEBSERVER_PORT +# EXPOSE $FRONTEND_WEBSERVER_PORT -CMD ["npm", "start"] +# CMD ["npm", "start"] -# FROM node:20-alpine AS base +FROM node:20-alpine AS base -# FROM base AS deps +FROM base AS deps -# RUN apk add --no-cache libc6-compat -# WORKDIR /app +RUN apk add --no-cache libc6-compat +WORKDIR /app -# COPY package.json ./ +COPY package.json ./ -# RUN npm update && npm install +RUN npm update && npm install -# # Install this to optimize images -# RUN npm i sharp +# Install this to optimize images +RUN npm i sharp -# # If you want yarn update and install uncomment the bellow +# If you want yarn update and install uncomment the bellow -# # RUN yarn install && yarn upgrade +# RUN yarn install && yarn upgrade -# FROM base AS builder -# WORKDIR /app -# COPY --from=deps /app/node_modules ./node_modules -# COPY . . +FROM base AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY . . -# RUN npm run build +RUN npm run build -# FROM base AS runner -# WORKDIR /app +FROM base AS runner +WORKDIR /app -# ENV NODE_ENV=production -# RUN addgroup --system --gid 1001 nodejs -# RUN adduser --system --uid 1001 nextjs +ENV NODE_ENV=production +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs -# COPY --from=builder /app/public ./public +COPY --from=builder /app/public ./public -# RUN mkdir .next -# RUN chown nextjs:nodejs .next +RUN mkdir .next +RUN chown nextjs:nodejs .next -# COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -# COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -# USER nextjs +USER nextjs -# EXPOSE $FRONTEND_WEBSERVER_PORT +EXPOSE $FRONTEND_WEBSERVER_PORT -# CMD ["node", "server.js"] \ No newline at end of file +CMD ["node", "server.js"] \ No newline at end of file From e7b619100c0b208f5a8475683add551554328233 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 14:58:25 -0400 Subject: [PATCH 43/67] Update frontend.Dockerfile --- apps/frontend/frontend.Dockerfile | 70 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index ce47a8a2..c7c44bb5 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -1,59 +1,59 @@ -# FROM node:20.6 AS base +FROM node:20.6 AS base -# WORKDIR /app +WORKDIR /app -# COPY . /app +COPY . /app -# RUN npm install +RUN npm install -# RUN npm run build +RUN npm run build -# EXPOSE $FRONTEND_WEBSERVER_PORT +EXPOSE $FRONTEND_WEBSERVER_PORT -# CMD ["npm", "start"] +CMD ["npm", "start"] -FROM node:20-alpine AS base +# FROM node:20-alpine AS base -FROM base AS deps +# FROM base AS deps -RUN apk add --no-cache libc6-compat -WORKDIR /app +# RUN apk add --no-cache libc6-compat +# WORKDIR /app -COPY package.json ./ +# COPY package.json ./ -RUN npm update && npm install +# RUN npm update && npm install -# Install this to optimize images -RUN npm i sharp +# # Install this to optimize images +# RUN npm i sharp -# If you want yarn update and install uncomment the bellow +# # If you want yarn update and install uncomment the bellow -# RUN yarn install && yarn upgrade +# # RUN yarn install && yarn upgrade -FROM base AS builder -WORKDIR /app -COPY --from=deps /app/node_modules ./node_modules -COPY . . +# FROM base AS builder +# WORKDIR /app +# COPY --from=deps /app/node_modules ./node_modules +# COPY . . -RUN npm run build +# RUN npm run build -FROM base AS runner -WORKDIR /app +# FROM base AS runner +# WORKDIR /app -ENV NODE_ENV=production -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nextjs +# ENV NODE_ENV=production +# RUN addgroup --system --gid 1001 nodejs +# RUN adduser --system --uid 1001 nextjs -COPY --from=builder /app/public ./public +# COPY --from=builder /app/public ./public -RUN mkdir .next -RUN chown nextjs:nodejs .next +# RUN mkdir .next +# RUN chown nextjs:nodejs .next -COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static +# COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +# COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -USER nextjs +# USER nextjs -EXPOSE $FRONTEND_WEBSERVER_PORT +# EXPOSE $FRONTEND_WEBSERVER_PORT -CMD ["node", "server.js"] \ No newline at end of file +# CMD ["node", "server.js"] \ No newline at end of file From 7e2af28936cdd8b78c80980bfd2ed7b8e2762185 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 15:08:01 -0400 Subject: [PATCH 44/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 3de2fd49..0dd02bc3 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -24,6 +24,10 @@ export const DispatchStep = ({ id, form, ...props }) => { console.log('Submitting Experiment'); submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); + console.log(JSON.stringify({ + "fileToUpload": files[0], + "experimentId": expId + })); const uploadResponse = await fetch('/api/files/uploadFile', { method: 'POST', headers: new Headers({ 'Content-Type': 'application/json' }), From 3635a3daf64c3d5ae8a0d49ef75b2e86ab48ba19 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 15:38:41 -0400 Subject: [PATCH 45/67] update frontend --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 4 ++-- apps/frontend/pages/api/files/uploadFile.tsx | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 0dd02bc3..1b1b02d3 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -25,7 +25,7 @@ export const DispatchStep = ({ id, form, ...props }) => { submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); console.log(JSON.stringify({ - "fileToUpload": files[0], + "fileToUpload": files[0].arrayBuffer, "experimentId": expId })); const uploadResponse = await fetch('/api/files/uploadFile', { @@ -33,7 +33,7 @@ export const DispatchStep = ({ id, form, ...props }) => { headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', body: JSON.stringify({ - "fileToUpload": files[0], + "fileToUpload": files[0]., "experimentId": expId }) }) diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 00526af0..8807b1a7 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -11,11 +11,6 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { return res.status(400).json({ response: "Not enough arguments!" } as any); } - if (typeof fileToUpload !== 'string') { - res.status(400).json({ response: 'File is not in the correct format' } as any); - return; - } - try { const client = await clientPromise; const db = client.db(DB_NAME); @@ -30,7 +25,7 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { chunkSizeBytes: 1048576, metadata: { field: 'expId', value: experimentId } }) as any); - + res.status(200).json({ response: 'Successfully wrote file!' } as any); return; From 8ca85bba1f6dc5d1165fd899a6c0f589f32fa055 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 20:59:13 -0400 Subject: [PATCH 46/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 1b1b02d3..e7829642 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -33,7 +33,7 @@ export const DispatchStep = ({ id, form, ...props }) => { headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', body: JSON.stringify({ - "fileToUpload": files[0]., + "fileToUpload": files[0], "experimentId": expId }) }) From 5aa4c5eecd528b7eae21020ea4356578dcc9499e Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 21:13:08 -0400 Subject: [PATCH 47/67] update to api --- .../AddExperiment/stepComponents/DispatchStep.tsx | 14 ++++++++++++-- apps/frontend/pages/api/files/uploadFile.tsx | 3 +-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index e7829642..30d49773 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -25,7 +25,7 @@ export const DispatchStep = ({ id, form, ...props }) => { submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); console.log(JSON.stringify({ - "fileToUpload": files[0].arrayBuffer, + "fileToUpload": arrayBufferToBase64(files[0].arrayBuffer), "experimentId": expId })); const uploadResponse = await fetch('/api/files/uploadFile', { @@ -33,7 +33,7 @@ export const DispatchStep = ({ id, form, ...props }) => { headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', body: JSON.stringify({ - "fileToUpload": files[0], + "fileToUpload": arrayBufferToBase64(files[0].arrayBuffer), "experimentId": expId }) }) @@ -110,3 +110,13 @@ export const DispatchStep = ({ id, form, ...props }) => { ); }; + +function arrayBufferToBase64(buffer) { + let binary = ''; + const bytes = new Uint8Array(buffer); + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]); + } + return Buffer.from(binary).toString("base64"); +} + diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 8807b1a7..b7eba384 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -15,7 +15,7 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { const client = await clientPromise; const db = client.db(DB_NAME); const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' }); - + const readableStream = new Readable(); readableStream.push(fileToUpload); readableStream.push(null); @@ -38,5 +38,4 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { } } - export default mongoFileUploader; From 5eded84073091b043c7f78b6675581eea240cb2a Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 21:27:05 -0400 Subject: [PATCH 48/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 30d49773..8d5f0ee5 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -33,7 +33,7 @@ export const DispatchStep = ({ id, form, ...props }) => { headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', body: JSON.stringify({ - "fileToUpload": arrayBufferToBase64(files[0].arrayBuffer), + "fileToUpload": arrayBufferToBase64(await files[0].arrayBuffer()), "experimentId": expId }) }) From 0010141c7451286156353a131c2845bac93abe49 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Tue, 15 Oct 2024 21:33:39 -0400 Subject: [PATCH 49/67] Update DispatchStep.tsx --- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 8d5f0ee5..69284f2f 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -25,7 +25,7 @@ export const DispatchStep = ({ id, form, ...props }) => { submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); console.log(JSON.stringify({ - "fileToUpload": arrayBufferToBase64(files[0].arrayBuffer), + "fileToUpload": arrayBufferToBase64(await files[0].arrayBuffer()), "experimentId": expId })); const uploadResponse = await fetch('/api/files/uploadFile', { From 71e28f315d1a8e26e8e271708450f2a466c7d3a1 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 12:12:33 -0400 Subject: [PATCH 50/67] clean up and backend --- apps/backend/app.py | 15 +++++------- apps/backend/modules/mongo.py | 23 ++++++++++++++++++- .../stepComponents/DispatchStep.tsx | 4 ---- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index e3f6dfff..eb72c1e1 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -1,4 +1,5 @@ """Module that uses flask to host endpoints for the backend""" +import threading import base64 from concurrent.futures import ProcessPoolExecutor import os @@ -6,8 +7,7 @@ from flask import Flask, Response, request, jsonify from kubernetes import client, config import pymongo -from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments -import threading +from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments, download_experiment_file from spawn_runner import create_job, create_job_object flaskApp = Flask(__name__) @@ -97,14 +97,11 @@ def check_mongo(): except Exception: return Response(status=500) -@flaskApp.post("/uploadExperimentFile") -def upload_experiment_file(): +@flaskApp.post("/downloadExpFile") +def download_exp_file(): json = request.get_json() - file = bson.Binary(base64.b64decode(json['file'])) - userId = json['user'] - print(file) - print(userId) - return Response(status=200) + experimentId = json['experimentId'] + return {'contents': download_experiment_file(experimentId, mongoClient)} if __name__ == '__main__': flaskApp.run() diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 66e1d559..79820f3c 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -2,6 +2,7 @@ import pymongo from pymongo.errors import ConnectionFailure from bson import Binary +from gridfs import GridFSBucket def verify_mongo_connection(mongoClient: pymongo.MongoClient): try: @@ -68,4 +69,24 @@ def insertExperiments(): insertExperiments() except: # keep trying - check_insert_default_experiments(mongoClient) \ No newline at end of file + check_insert_default_experiments(mongoClient) + +def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): + # we are going to have to get the binary data from mongo here + # setup the bucket + db = mongoClient["gladosDb"] + bucket = GridFSBucket(db, bucket_name='fileBucket') + files = bucket.find({'metadata.expId': expId}) + num_files = 0 + file_name = "" + for file in files: + num_files += 1 + if num_files > 1: + raise Exception("There are more than 1 file for a single experiment!") + file_name = file['filename'] + if file_name == "": + raise Exception("No file found!") + file = bucket.open_download_stream_by_name(file_name) + contents = file.read() + return contents + \ No newline at end of file diff --git a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 69284f2f..36aee042 100644 --- a/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -24,10 +24,6 @@ export const DispatchStep = ({ id, form, ...props }) => { console.log('Submitting Experiment'); submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); - console.log(JSON.stringify({ - "fileToUpload": arrayBufferToBase64(await files[0].arrayBuffer()), - "experimentId": expId - })); const uploadResponse = await fetch('/api/files/uploadFile', { method: 'POST', headers: new Headers({ 'Content-Type': 'application/json' }), From d135738a6a9e9dcd6bd903b583a4cfa966a7d191 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 13:10:06 -0400 Subject: [PATCH 51/67] add download file --- apps/backend/app.py | 5 ++--- apps/backend/modules/mongo.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index eb72c1e1..1300becf 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -97,10 +97,9 @@ def check_mongo(): except Exception: return Response(status=500) -@flaskApp.post("/downloadExpFile") +@flaskApp.get("/downloadExpFile") def download_exp_file(): - json = request.get_json() - experimentId = json['experimentId'] + experimentId = request.args.get('expId', default='', type=str) return {'contents': download_experiment_file(experimentId, mongoClient)} if __name__ == '__main__': diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 79820f3c..ff1450b2 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -1,3 +1,4 @@ +import base64 import json import pymongo from pymongo.errors import ConnectionFailure @@ -88,5 +89,5 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): raise Exception("No file found!") file = bucket.open_download_stream_by_name(file_name) contents = file.read() - return contents + return base64.b64encode(contents) \ No newline at end of file From f4e4c7ab488029557dbfe77f9b99e017fe7a5617 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 13:18:24 -0400 Subject: [PATCH 52/67] update backend --- apps/backend/app.py | 7 +++++-- .../flows/AddExperiment/stepComponents/DispatchStep.tsx | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 1300becf..2a5cb7dd 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -99,8 +99,11 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): - experimentId = request.args.get('expId', default='', type=str) - return {'contents': download_experiment_file(experimentId, mongoClient)} + try: + experimentId = request.args.get('expId', default='', type=str) + return {'contents': download_experiment_file(experimentId, mongoClient)} + except Exception: + return Response(status=500) if __name__ == '__main__': flaskApp.run() diff --git a/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index 280fa63e..ea0df7d6 100644 --- a/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -34,7 +34,7 @@ export const DispatchStep = ({ id, form, ...props }) => { "fileToUpload": arrayBufferToBase64(await files[0].arrayBuffer()), "experimentId": expId }) - }) + }); if (uploadResponse) { console.log(`Handing experiment ${expId} to the backend`); const response = await fetch(`/api/experiments/${expId}`, { From 80684017b14c1caa55b4c7681450d48bf3cbbac8 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 13:30:19 -0400 Subject: [PATCH 53/67] add debugging info --- apps/backend/app.py | 7 ++----- apps/backend/modules/mongo.py | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 2a5cb7dd..1300becf 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -99,11 +99,8 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): - try: - experimentId = request.args.get('expId', default='', type=str) - return {'contents': download_experiment_file(experimentId, mongoClient)} - except Exception: - return Response(status=500) + experimentId = request.args.get('expId', default='', type=str) + return {'contents': download_experiment_file(experimentId, mongoClient)} if __name__ == '__main__': flaskApp.run() diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index ff1450b2..1f7f214c 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -75,9 +75,11 @@ def insertExperiments(): def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): # we are going to have to get the binary data from mongo here # setup the bucket + print(expId) db = mongoClient["gladosDb"] bucket = GridFSBucket(db, bucket_name='fileBucket') files = bucket.find({'metadata.expId': expId}) + print(files) num_files = 0 file_name = "" for file in files: From 9fdb37cc5c10526aa44740d643422d5ac40886bf Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 13:36:42 -0400 Subject: [PATCH 54/67] Update app.py --- apps/backend/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 1300becf..3028fb04 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -99,8 +99,9 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): - experimentId = request.args.get('expId', default='', type=str) - return {'contents': download_experiment_file(experimentId, mongoClient)} + experiment_id = request.args.get('expId', default='', type=str) + print(experiment_id) + return {'contents': download_experiment_file(experiment_id, mongoClient)} if __name__ == '__main__': flaskApp.run() From 965842d8c4ab2b1631135c3fe582dc195c19a0d7 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 13:42:04 -0400 Subject: [PATCH 55/67] Update app.py --- apps/backend/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 3028fb04..b991c3a5 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -100,7 +100,7 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): experiment_id = request.args.get('expId', default='', type=str) - print(experiment_id) + raise Exception(experiment_id) return {'contents': download_experiment_file(experiment_id, mongoClient)} if __name__ == '__main__': From 266c815c76f9df14a3ccd6d1b4d36e73212c48e7 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 13:53:55 -0400 Subject: [PATCH 56/67] change upload file --- apps/backend/app.py | 1 - apps/frontend/pages/api/files/uploadFile.tsx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index b991c3a5..0e21cdd9 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -100,7 +100,6 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): experiment_id = request.args.get('expId', default='', type=str) - raise Exception(experiment_id) return {'contents': download_experiment_file(experiment_id, mongoClient)} if __name__ == '__main__': diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index b7eba384..ab5aece8 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -23,7 +23,7 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { readableStream. pipe(bucket.openUploadStream(`experimentFile${experimentId}`, { chunkSizeBytes: 1048576, - metadata: { field: 'expId', value: experimentId } + metadata: { expId: experimentId } }) as any); From be50845180d3283f2e01c96d5f134ce800af8f54 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 14:09:17 -0400 Subject: [PATCH 57/67] Update mongo.py --- apps/backend/modules/mongo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 1f7f214c..0a6b4969 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -78,7 +78,7 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): print(expId) db = mongoClient["gladosDb"] bucket = GridFSBucket(db, bucket_name='fileBucket') - files = bucket.find({'metadata.expId': expId}) + files = bucket.find({"metadata.expId": expId}).to_list() print(files) num_files = 0 file_name = "" @@ -86,7 +86,7 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): num_files += 1 if num_files > 1: raise Exception("There are more than 1 file for a single experiment!") - file_name = file['filename'] + file_name = file.filename if file_name == "": raise Exception("No file found!") file = bucket.open_download_stream_by_name(file_name) From d526ad671b1bb3cbc4745eb793a942bbcb1f660f Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 14:30:13 -0400 Subject: [PATCH 58/67] finish backend --- apps/backend/app.py | 7 +++++-- apps/backend/modules/mongo.py | 7 ++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index 0e21cdd9..ad7238ee 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -99,8 +99,11 @@ def check_mongo(): @flaskApp.get("/downloadExpFile") def download_exp_file(): - experiment_id = request.args.get('expId', default='', type=str) - return {'contents': download_experiment_file(experiment_id, mongoClient)} + try: + experiment_id = request.args.get('expId', default='', type=str) + return {'contents': download_experiment_file(experiment_id, mongoClient)} + except Exception: + return Response(status=500) if __name__ == '__main__': flaskApp.run() diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 0a6b4969..63a58aee 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -1,4 +1,3 @@ -import base64 import json import pymongo from pymongo.errors import ConnectionFailure @@ -75,11 +74,9 @@ def insertExperiments(): def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): # we are going to have to get the binary data from mongo here # setup the bucket - print(expId) - db = mongoClient["gladosDb"] + db = mongoClient["gladosdb"] bucket = GridFSBucket(db, bucket_name='fileBucket') files = bucket.find({"metadata.expId": expId}).to_list() - print(files) num_files = 0 file_name = "" for file in files: @@ -91,5 +88,5 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): raise Exception("No file found!") file = bucket.open_download_stream_by_name(file_name) contents = file.read() - return base64.b64encode(contents) + return contents.decode("utf-8") \ No newline at end of file From 622d444d3c384e8228033e15d4d73dadd5c1ed6d Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 14:58:47 -0400 Subject: [PATCH 59/67] Update runner.py --- apps/runner/runner.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 10a29364..636a4c84 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -6,6 +6,9 @@ import json import time import typing +import base64 + +import requests import firebase_admin from firebase_admin import credentials from firebase_admin import firestore, storage @@ -193,8 +196,16 @@ def download_experiment_files(experiment: ExperimentData): experiment.file = filepath explogger.info(f"Downloading {filepath} to ExperimentFiles/{experiment.expId}/{filepath}") try: - filedata = firebaseBucket.blob(filepath) - filedata.download_to_filename(filepath) + # filedata = firebaseBucket.blob(filepath) + # filedata.download_to_filename(filepath) + # try to call the backend to download + url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?={experiment.expId}' + response = requests.get(url) + file_contents = response.json()["contents"] + # write the file contents to file path + with open(filepath) as file: + file.write(file_contents) + except Exception as err: explogger.error(f"Error {err} occurred while trying to download experiment file") raise GladosInternalError('Failed to download experiment files') from err From fbe99e1c5cdd19eef0b468b712ea96c3e7e50371 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 15:03:57 -0400 Subject: [PATCH 60/67] Update runner.py --- apps/runner/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 636a4c84..1016384a 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -199,7 +199,7 @@ def download_experiment_files(experiment: ExperimentData): # filedata = firebaseBucket.blob(filepath) # filedata.download_to_filename(filepath) # try to call the backend to download - url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?={experiment.expId}' + url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' response = requests.get(url) file_contents = response.json()["contents"] # write the file contents to file path From dd16bde9b4b77c9c8849fa8e013122aa1387f2f5 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 15:16:08 -0400 Subject: [PATCH 61/67] change backend to use dev --- apps/backend/job-runner.yaml | 2 +- apps/runner/runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/backend/job-runner.yaml b/apps/backend/job-runner.yaml index 0e7efb1b..06b050d4 100644 --- a/apps/backend/job-runner.yaml +++ b/apps/backend/job-runner.yaml @@ -10,7 +10,7 @@ spec: spec: containers: - name: runner - image: gladospipeline/glados-runner:main + image: gladospipeline/glados-runner:development imagePullPolicy: Always command: [] env: diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 1016384a..434696a8 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -200,7 +200,7 @@ def download_experiment_files(experiment: ExperimentData): # filedata.download_to_filename(filepath) # try to call the backend to download url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' - response = requests.get(url) + response = requests.get(url, timeout=60) file_contents = response.json()["contents"] # write the file contents to file path with open(filepath) as file: From 4f617f4dc5f981f72864bb6f25c3c3fb72a3ea5c Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 16 Oct 2024 16:08:30 -0400 Subject: [PATCH 62/67] Update runner.py --- apps/runner/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 434696a8..2bad43b2 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -203,7 +203,7 @@ def download_experiment_files(experiment: ExperimentData): response = requests.get(url, timeout=60) file_contents = response.json()["contents"] # write the file contents to file path - with open(filepath) as file: + with open(filepath, "x") as file: file.write(file_contents) except Exception as err: From 6035a5656e4513ee38fea445ce4523b0db598dae Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Fri, 18 Oct 2024 21:02:16 -0400 Subject: [PATCH 63/67] Update runner.py --- apps/runner/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 2bad43b2..81d4bc99 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -201,7 +201,7 @@ def download_experiment_files(experiment: ExperimentData): # try to call the backend to download url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' response = requests.get(url, timeout=60) - file_contents = response.json()["contents"] + file_contents = str(base64.b64decode(response.json()["contents"]), "utf-8") # write the file contents to file path with open(filepath, "x") as file: file.write(file_contents) From 849769148f282a3dee643904d5649f28c8a5df81 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Fri, 18 Oct 2024 21:02:46 -0400 Subject: [PATCH 64/67] Update runner.py --- apps/runner/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 81d4bc99..1355288e 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -201,7 +201,7 @@ def download_experiment_files(experiment: ExperimentData): # try to call the backend to download url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' response = requests.get(url, timeout=60) - file_contents = str(base64.b64decode(response.json()["contents"]), "utf-8") + file_contents = base64.b64decode(response.json()["contents"]).decode() # write the file contents to file path with open(filepath, "x") as file: file.write(file_contents) From c23c681d68526b610748b1482c3287e39fbb67e8 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Fri, 18 Oct 2024 21:11:46 -0400 Subject: [PATCH 65/67] remove firebase bucket --- apps/runner/runner.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 1355288e..11f054a3 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -42,7 +42,6 @@ firebaseCredentials = credentials.Certificate(json.loads(_get_env(ENV_FIREBASE_CREDENTIALS))) firebaseApp = firebase_admin.initialize_app(firebaseCredentials) firebaseDb = firestore.client() -firebaseBucket = storage.bucket("gladosbase.appspot.com") syslogger.info("GLADOS Runner Started") @@ -196,8 +195,6 @@ def download_experiment_files(experiment: ExperimentData): experiment.file = filepath explogger.info(f"Downloading {filepath} to ExperimentFiles/{experiment.expId}/{filepath}") try: - # filedata = firebaseBucket.blob(filepath) - # filedata.download_to_filename(filepath) # try to call the backend to download url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' response = requests.get(url, timeout=60) @@ -231,12 +228,6 @@ def remove_downloaded_directory(experimentId: DocumentId): def upload_experiment_results(experiment: ExperimentData): explogger.info('Uploading Experiment Results...') - try: - uploadBlob = firebaseBucket.blob(f"results/result{experiment.expId}.csv") - uploadBlob.upload_from_filename('results.csv') - except Exception as err: - raise DatabaseConnectionError("Error uploading aggregated experiment results to firebase") from err - explogger.info('Uploading to MongoDB') verify_mongo_connection() @@ -260,13 +251,6 @@ def upload_experiment_results(experiment: ExperimentData): except Exception as err: raise GladosInternalError("Error preparing experiment results zip") from err - # TODO remove firebase usage once transition to mongo file storage is complete - try: - uploadBlob = firebaseBucket.blob(f"results/result{experiment.expId}.zip") - uploadBlob.upload_from_filename('ResultCsvs.zip') - except Exception as err: - raise DatabaseConnectionError("Error uploading experiment results zip to firebase") from err - upload_experiment_zip(experiment, encoded) def post_process_experiment(experiment: ExperimentData): From adfef61a7e2e66e5e150353eeb0736ce696495f7 Mon Sep 17 00:00:00 2001 From: Riley Windsor Date: Fri, 18 Oct 2024 21:20:00 -0400 Subject: [PATCH 66/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index d618038d..1cfda10c 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: container-test-frontend - image: gladospipeline/glados-frontend:main + image: gladospipeline/glados-frontend:development imagePullPolicy: Always ports: - containerPort: 0000 From a590f4bdd917edd26f4b8e701dbe51b377a35e43 Mon Sep 17 00:00:00 2001 From: Riley Windsor <95257011+rhit-windsors@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:03:35 -0400 Subject: [PATCH 67/67] Update deployment-frontend.yaml --- kubernetes_init/frontend/deployment-frontend.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes_init/frontend/deployment-frontend.yaml b/kubernetes_init/frontend/deployment-frontend.yaml index 1cfda10c..52c912a6 100644 --- a/kubernetes_init/frontend/deployment-frontend.yaml +++ b/kubernetes_init/frontend/deployment-frontend.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: container-test-frontend - image: gladospipeline/glados-frontend:development + image: gladospipeline/glados-frontend:main imagePullPolicy: Always ports: - containerPort: 0000 @@ -58,4 +58,4 @@ spec: secretKeyRef: name: secret-env key: MONGODB_PASSWORD - \ No newline at end of file +