From 7aef515c783ae8aac287ee49b494e6fc45290235 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 13:06:09 -0500 Subject: [PATCH 1/8] Revert "Merge pull request #343 from AutomatingSciencePipeline/jar-hotfix" This reverts commit c999ae790b987c1e54606ffa4b45e79906e7b9c8, reversing changes made to 1aab315d4c2f6605b976f041fee8e55b9bc02b5d. --- apps/runner/runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index bc4dffa7..1eae3946 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -173,10 +173,8 @@ def determine_experiment_file_type(filepath: str): filetype = ExperimentType.UNKNOWN if 'Python script' in rawfiletype or 'python3' in rawfiletype: filetype = ExperimentType.PYTHON - elif 'ELF 64-bit LSB' in rawfiletype: - filetype = ExperimentType.C - else: - # check if it is a jar file, they can have multiple rawfiletypes + elif 'Zip archive data' in rawfiletype: + # check for META-INF/MANIFEST.MF in file try: file = open(filepath, "r") contents = file.read() @@ -186,6 +184,8 @@ def determine_experiment_file_type(filepath: str): except FileNotFoundError as e: explogger.error(f"{filepath} could not be read to determine if it is a jar file!") raise e + elif 'ELF 64-bit LSB' in rawfiletype: + filetype = ExperimentType.C explogger.info(f"Raw Filetype: {rawfiletype}\n Filtered Filetype: {filetype.value}") From 5d8950d6ce393c2fb928bff72544bddf7b227d29 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 13:06:14 -0500 Subject: [PATCH 2/8] Revert "Merge pull request #341 from AutomatingSciencePipeline/jar-file-type-fix" This reverts commit 1aab315d4c2f6605b976f041fee8e55b9bc02b5d, reversing changes made to 97e083a40d23cf7340b806ce1560ebce41cf8768. --- apps/runner/runner.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 1eae3946..11f054a3 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -173,17 +173,8 @@ def determine_experiment_file_type(filepath: str): filetype = ExperimentType.UNKNOWN if 'Python script' in rawfiletype or 'python3' in rawfiletype: filetype = ExperimentType.PYTHON - elif 'Zip archive data' in rawfiletype: - # check for META-INF/MANIFEST.MF in file - try: - file = open(filepath, "r") - contents = file.read() - if "META-INF/MANIFEST.MF" in contents: - filetype = ExperimentType.JAVA - file.close() - except FileNotFoundError as e: - explogger.error(f"{filepath} could not be read to determine if it is a jar file!") - raise e + elif 'Java archive data (JAR)' in rawfiletype: + filetype = ExperimentType.JAVA elif 'ELF 64-bit LSB' in rawfiletype: filetype = ExperimentType.C From 886175e9686d68aea2fdf1a73ed73a07cb374a65 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 13:29:29 -0500 Subject: [PATCH 3/8] update upload --- .../stepComponents/DispatchStep.tsx | 11 ++--- apps/frontend/package-lock.json | 45 +++++++++++++++++ apps/frontend/package.json | 2 + apps/frontend/pages/api/files/uploadFile.tsx | 49 ++++++++++++------- 4 files changed, 83 insertions(+), 24 deletions(-) diff --git a/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx b/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx index ea0df7d6..abec3085 100644 --- a/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx +++ b/apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx @@ -26,16 +26,15 @@ export const DispatchStep = ({ id, form, ...props }) => { console.log('Submitting Experiment'); submitExperiment(form.values, userId as string).then(async (expId) => { console.log(`Uploading file for ${expId}:`, files); + const formData = new FormData(); + formData.set("file", files[0]); + formData.set("expId", expId); const uploadResponse = await fetch('/api/files/uploadFile', { method: 'POST', - headers: new Headers({ 'Content-Type': 'application/json' }), credentials: 'same-origin', - body: JSON.stringify({ - "fileToUpload": arrayBufferToBase64(await files[0].arrayBuffer()), - "experimentId": expId - }) + body: formData }); - if (uploadResponse) { + if (uploadResponse.ok) { console.log(`Handing experiment ${expId} to the backend`); const response = await fetch(`/api/experiments/${expId}`, { method: 'POST', diff --git a/apps/frontend/package-lock.json b/apps/frontend/package-lock.json index 7ae2e655..75c06655 100644 --- a/apps/frontend/package-lock.json +++ b/apps/frontend/package-lock.json @@ -20,10 +20,12 @@ "@mantine/hooks": "^5.6.2", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", + "@types/formidable": "^3.4.5", "classnames": "^2.3.2", "dayjs": "^1.11.6", "encoding": "^0.1.13", "firebase": "^9.12.1", + "formidable": "^3.5.2", "joi": "^17.6.4", "mongodb": "^5.2.0", "next": "^13.4.19", @@ -1890,6 +1892,14 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.1.tgz", "integrity": "sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==" }, + "node_modules/@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/hoist-non-react-statics": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz", @@ -2404,6 +2414,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" + }, "node_modules/ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", @@ -2923,6 +2938,15 @@ "node": ">=6" } }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -3836,6 +3860,19 @@ "is-callable": "^1.1.3" } }, + "node_modules/formidable": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.2.tgz", + "integrity": "sha512-Jqc1btCy3QzRbJaICGwKcBfGWuLADRerLzDqi2NwSt/UkXLsHJw2TVResiaoBufHVHy9aSgClOHCeJsSsFLTbg==", + "dependencies": { + "dezalgo": "^1.0.4", + "hexoid": "^2.0.0", + "once": "^1.4.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, "node_modules/fraction.js": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.6.tgz", @@ -4133,6 +4170,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hexoid": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-2.0.0.tgz", + "integrity": "sha512-qlspKUK7IlSQv2o+5I7yhUd7TxlOG2Vr5LTa3ve2XSNVKAL/n/u/7KLvKmFNimomDIKvZFXWHv0T12mv7rT8Aw==", + "engines": { + "node": ">=8" + } + }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", diff --git a/apps/frontend/package.json b/apps/frontend/package.json index 802074de..e87f563d 100644 --- a/apps/frontend/package.json +++ b/apps/frontend/package.json @@ -23,10 +23,12 @@ "@mantine/hooks": "^5.6.2", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", + "@types/formidable": "^3.4.5", "classnames": "^2.3.2", "dayjs": "^1.11.6", "encoding": "^0.1.13", "firebase": "^9.12.1", + "formidable": "^3.5.2", "joi": "^17.6.4", "mongodb": "^5.2.0", "next": "^13.4.19", diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 5351d371..21afd29d 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -1,21 +1,33 @@ import clientPromise, { DB_NAME, COLLECTION_RESULTS_CSVS, COLLECTION_EXPERIMENT_FILES } from '../../../lib/mongodb'; -import { NextApiHandler } from 'next'; +import { NextApiHandler, NextApiRequest } from 'next'; import { GridFSBucket } from 'mongodb'; -import { Readable } from 'stream'; +import { Readable, Writable } from 'stream'; +import formidable, { Fields, Files } from "formidable"; +import fs from "fs"; export const config = { api: { - bodyParser: { - sizeLimit: '1000mb' - } + bodyParser: false }, - } +} + +// Helper function to parse form data with formidable +const parseForm = async (req: NextApiRequest): Promise<{ fields: Fields; files: Files }> => + new Promise((resolve, reject) => { + const form = formidable({ keepExtensions: true }); + form.parse(req, (err, fields, files) => { + if (err) reject(err); + resolve({ fields, files }); + }); + }); + const mongoFileUploader: NextApiHandler = async (req, res) => { if (req.method === 'POST') { - const { fileToUpload, experimentId } = req.body; + const { fields, files } = await parseForm(req); + const expId = Array.isArray(fields.id) ? fields.id[0] : fields.id; - if (!fileToUpload || !experimentId) { + if (!files.file || !expId) { return res.status(400).json({ response: "Not enough arguments!" } as any); } @@ -23,19 +35,20 @@ const mongoFileUploader: NextApiHandler = async (req, res) => { const client = await clientPromise; const db = client.db(DB_NAME); const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' }); - - const readableStream = new Readable(); - readableStream.push(fileToUpload); - readableStream.push(null); - readableStream. - pipe(bucket.openUploadStream(`experimentFile${experimentId}`, { - chunkSizeBytes: 1048576, - metadata: { expId: experimentId } - }) as any); + const file = Array.isArray(files.file) ? files.file[0] : files.file; + const fileStream = fs.createReadStream(file.filepath); + + // Upload the file to GridFS + const uploadStream = bucket.openUploadStream(file.originalFilename || "uploadedFile", { + metadata: { expId: expId }, + }) as Writable; - res.status(200).json({ response: 'Successfully wrote file!' } as any); + // Pipe the file stream to GridFS + fileStream.pipe(uploadStream).on("finish", () => { + res.status(200).json({ message: "File and ID uploaded successfully." } as any); + }); return; } catch (error) { From ba1e3a5506de3c13911302dbbffad05a176912a3 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 13:43:28 -0500 Subject: [PATCH 4/8] Update uploadFile.tsx --- apps/frontend/pages/api/files/uploadFile.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/frontend/pages/api/files/uploadFile.tsx b/apps/frontend/pages/api/files/uploadFile.tsx index 21afd29d..435a71d2 100644 --- a/apps/frontend/pages/api/files/uploadFile.tsx +++ b/apps/frontend/pages/api/files/uploadFile.tsx @@ -25,8 +25,8 @@ const parseForm = async (req: NextApiRequest): Promise<{ fields: Fields; files: const mongoFileUploader: NextApiHandler = async (req, res) => { if (req.method === 'POST') { const { fields, files } = await parseForm(req); - const expId = Array.isArray(fields.id) ? fields.id[0] : fields.id; - + const expId = Array.isArray(fields.expId) ? fields.expId[0] : fields.expId; + if (!files.file || !expId) { return res.status(400).json({ response: "Not enough arguments!" } as any); } From 031e5f7a683069400fd7dde283978394f523bd34 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 13:49:59 -0500 Subject: [PATCH 5/8] update backend --- apps/backend/app.py | 7 +++++-- apps/backend/modules/mongo.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/backend/app.py b/apps/backend/app.py index ad7238ee..c4152c9e 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -1,10 +1,11 @@ """Module that uses flask to host endpoints for the backend""" +import io import threading import base64 from concurrent.futures import ProcessPoolExecutor import os import bson -from flask import Flask, Response, request, jsonify +from flask import Flask, Response, request, jsonify, send_file from kubernetes import client, config import pymongo from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments, download_experiment_file @@ -101,7 +102,9 @@ def check_mongo(): def download_exp_file(): try: experiment_id = request.args.get('expId', default='', type=str) - return {'contents': download_experiment_file(experiment_id, mongoClient)} + file_data = download_experiment_file(experiment_id, mongoClient) + file_stream = io.BytesIO(file_data) + return send_file(file_stream, as_attachment=True, download_name="experiment_file", mimetype="application/octet-stream") except Exception: return Response(status=500) diff --git a/apps/backend/modules/mongo.py b/apps/backend/modules/mongo.py index 63a58aee..e3785c30 100644 --- a/apps/backend/modules/mongo.py +++ b/apps/backend/modules/mongo.py @@ -88,5 +88,5 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient): raise Exception("No file found!") file = bucket.open_download_stream_by_name(file_name) contents = file.read() - return contents.decode("utf-8") + return contents \ No newline at end of file From dc441e44dcf519f9fc9874761bc97c4e3ae0fb6d Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 13:56:30 -0500 Subject: [PATCH 6/8] update runner --- apps/backend/job-runner.yaml | 2 +- apps/runner/runner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/backend/job-runner.yaml b/apps/backend/job-runner.yaml index cbd45305..0ff57460 100644 --- a/apps/backend/job-runner.yaml +++ b/apps/backend/job-runner.yaml @@ -10,7 +10,7 @@ spec: spec: containers: - name: runner - image: gladospipeline/glados-runner:main + image: gladospipeline/glados-runner:development imagePullPolicy: Always command: [] env: diff --git a/apps/runner/runner.py b/apps/runner/runner.py index 11f054a3..d617a210 100644 --- a/apps/runner/runner.py +++ b/apps/runner/runner.py @@ -198,9 +198,9 @@ def download_experiment_files(experiment: ExperimentData): # try to call the backend to download url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}' response = requests.get(url, timeout=60) - file_contents = base64.b64decode(response.json()["contents"]).decode() + file_contents = response.content # write the file contents to file path - with open(filepath, "x") as file: + with open(filepath, "xb") as file: file.write(file_contents) except Exception as err: From c13dbb2c616335a906d0e375a15c6b8a6346bda2 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 14:08:16 -0500 Subject: [PATCH 7/8] Update frontend.Dockerfile --- apps/frontend/frontend.Dockerfile | 70 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/apps/frontend/frontend.Dockerfile b/apps/frontend/frontend.Dockerfile index c7c44bb5..ce47a8a2 100644 --- a/apps/frontend/frontend.Dockerfile +++ b/apps/frontend/frontend.Dockerfile @@ -1,59 +1,59 @@ -FROM node:20.6 AS base +# FROM node:20.6 AS base -WORKDIR /app +# WORKDIR /app -COPY . /app +# COPY . /app -RUN npm install +# RUN npm install -RUN npm run build +# RUN npm run build -EXPOSE $FRONTEND_WEBSERVER_PORT +# EXPOSE $FRONTEND_WEBSERVER_PORT -CMD ["npm", "start"] +# CMD ["npm", "start"] -# FROM node:20-alpine AS base +FROM node:20-alpine AS base -# FROM base AS deps +FROM base AS deps -# RUN apk add --no-cache libc6-compat -# WORKDIR /app +RUN apk add --no-cache libc6-compat +WORKDIR /app -# COPY package.json ./ +COPY package.json ./ -# RUN npm update && npm install +RUN npm update && npm install -# # Install this to optimize images -# RUN npm i sharp +# Install this to optimize images +RUN npm i sharp -# # If you want yarn update and install uncomment the bellow +# If you want yarn update and install uncomment the bellow -# # RUN yarn install && yarn upgrade +# RUN yarn install && yarn upgrade -# FROM base AS builder -# WORKDIR /app -# COPY --from=deps /app/node_modules ./node_modules -# COPY . . +FROM base AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY . . -# RUN npm run build +RUN npm run build -# FROM base AS runner -# WORKDIR /app +FROM base AS runner +WORKDIR /app -# ENV NODE_ENV=production -# RUN addgroup --system --gid 1001 nodejs -# RUN adduser --system --uid 1001 nextjs +ENV NODE_ENV=production +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs -# COPY --from=builder /app/public ./public +COPY --from=builder /app/public ./public -# RUN mkdir .next -# RUN chown nextjs:nodejs .next +RUN mkdir .next +RUN chown nextjs:nodejs .next -# COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -# COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -# USER nextjs +USER nextjs -# EXPOSE $FRONTEND_WEBSERVER_PORT +EXPOSE $FRONTEND_WEBSERVER_PORT -# CMD ["node", "server.js"] \ No newline at end of file +CMD ["node", "server.js"] \ No newline at end of file From d5e0c6f618133a189a101ec615c09e9def034c09 Mon Sep 17 00:00:00 2001 From: rhit-windsors Date: Wed, 6 Nov 2024 14:20:15 -0500 Subject: [PATCH 8/8] Update job-runner.yaml --- apps/backend/job-runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backend/job-runner.yaml b/apps/backend/job-runner.yaml index 0ff57460..cbd45305 100644 --- a/apps/backend/job-runner.yaml +++ b/apps/backend/job-runner.yaml @@ -10,7 +10,7 @@ spec: spec: containers: - name: runner - image: gladospipeline/glados-runner:development + image: gladospipeline/glados-runner:main imagePullPolicy: Always command: [] env: