Merge pull request #347 from AutomatingSciencePipeline/development

Restore Java Functionality

apps/backend/app.py

@@ -1,10 +1,11 @@
 """Module that uses flask to host endpoints for the backend"""
+import io
 import threading
 import base64
 from concurrent.futures import ProcessPoolExecutor
 import os
 import bson
-from flask import Flask, Response, request, jsonify
+from flask import Flask, Response, request, jsonify, send_file
 from kubernetes import client, config
 import pymongo
 from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments, download_experiment_file
@@ -101,7 +102,9 @@ def check_mongo():
 def download_exp_file():
     try:
         experiment_id = request.args.get('expId', default='', type=str)
-        return {'contents': download_experiment_file(experiment_id, mongoClient)}
+        file_data = download_experiment_file(experiment_id, mongoClient)
+        file_stream = io.BytesIO(file_data)
+        return send_file(file_stream, as_attachment=True, download_name="experiment_file", mimetype="application/octet-stream")
     except Exception:
         return Response(status=500)
 

apps/backend/modules/mongo.py

@@ -88,5 +88,5 @@ def download_experiment_file(expId: str, mongoClient: pymongo.MongoClient):
         raise Exception("No file found!")
     file = bucket.open_download_stream_by_name(file_name)
     contents = file.read()
-    return contents.decode("utf-8")
+    return contents
 

apps/frontend/app/components/flows/AddExperiment/stepComponents/DispatchStep.tsx

@@ -26,16 +26,15 @@ export const DispatchStep = ({ id, form, ...props }) => {
 		console.log('Submitting Experiment');
 		submitExperiment(form.values, userId as string).then(async (expId) => {
 			console.log(`Uploading file for ${expId}:`, files);
+			const formData = new FormData();
+			formData.set("file", files[0]);
+			formData.set("expId", expId);
 			const uploadResponse = await fetch('/api/files/uploadFile', {
 				method: 'POST',
-				headers: new Headers({ 'Content-Type': 'application/json' }),
 				credentials: 'same-origin',
-				body: JSON.stringify({
-					"fileToUpload": arrayBufferToBase64(await files[0].arrayBuffer()),
-					"experimentId": expId
-				})
+				body: formData
 			});
-			if (uploadResponse) {
+			if (uploadResponse.ok) {
 				console.log(`Handing experiment ${expId} to the backend`);
 				const response = await fetch(`/api/experiments/${expId}`, {
 					method: 'POST',

apps/frontend/frontend.Dockerfile

@@ -1,59 +1,59 @@
-FROM node:20.6 AS base
+# FROM node:20.6 AS base
 
-WORKDIR /app
+# WORKDIR /app
 
-COPY . /app
+# COPY . /app
 
-RUN npm install
+# RUN npm install
 
-RUN npm run build
+# RUN npm run build
 
-EXPOSE $FRONTEND_WEBSERVER_PORT
+# EXPOSE $FRONTEND_WEBSERVER_PORT
 
-CMD ["npm", "start"]
+# CMD ["npm", "start"]
 
-# FROM node:20-alpine AS base
+FROM node:20-alpine AS base
 
-# FROM base AS deps
+FROM base AS deps
 
-# RUN apk add --no-cache libc6-compat
-# WORKDIR /app
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
 
-# COPY package.json ./
+COPY package.json ./
 
-# RUN npm update && npm install
+RUN npm update && npm install
 
-# # Install this to optimize images
-# RUN npm i sharp
+# Install this to optimize images
+RUN npm i sharp
 
-# # If you want yarn update and  install uncomment the bellow
+# If you want yarn update and  install uncomment the bellow
 
-# # RUN yarn install &&  yarn upgrade
+# RUN yarn install &&  yarn upgrade
 
-# FROM base AS builder
-# WORKDIR /app
-# COPY --from=deps /app/node_modules ./node_modules
-# COPY . .
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY . .
 
-# RUN npm run build
+RUN npm run build
 
-# FROM base AS runner
-# WORKDIR /app
+FROM base AS runner
+WORKDIR /app
 
-# ENV NODE_ENV=production
-# RUN addgroup --system --gid 1001 nodejs
-# RUN adduser --system --uid 1001 nextjs
+ENV NODE_ENV=production
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
 
-# COPY --from=builder /app/public ./public
+COPY --from=builder /app/public ./public
 
-# RUN mkdir .next
-# RUN chown nextjs:nodejs .next
+RUN mkdir .next
+RUN chown nextjs:nodejs .next
 
-# COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
-# COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
+COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
+COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 
-# USER nextjs
+USER nextjs
 
-# EXPOSE $FRONTEND_WEBSERVER_PORT
+EXPOSE $FRONTEND_WEBSERVER_PORT
 
-# CMD ["node", "server.js"]
+CMD ["node", "server.js"]

apps/frontend/package.json

@@ -23,10 +23,12 @@
 		"@mantine/hooks": "^5.6.2",
 		"@testing-library/jest-dom": "^5.16.5",
 		"@testing-library/react": "^13.4.0",
+		"@types/formidable": "^3.4.5",
 		"classnames": "^2.3.2",
 		"dayjs": "^1.11.6",
 		"encoding": "^0.1.13",
 		"firebase": "^9.12.1",
+		"formidable": "^3.5.2",
 		"joi": "^17.6.4",
 		"mongodb": "^5.2.0",
 		"next": "^13.4.19",

apps/frontend/pages/api/files/uploadFile.tsx

@@ -1,41 +1,54 @@
 import clientPromise, { DB_NAME, COLLECTION_RESULTS_CSVS, COLLECTION_EXPERIMENT_FILES } from '../../../lib/mongodb';
-import { NextApiHandler } from 'next';
+import { NextApiHandler, NextApiRequest } from 'next';
 import { GridFSBucket } from 'mongodb';
-import { Readable } from 'stream';
+import { Readable, Writable } from 'stream';
+import formidable, { Fields, Files } from "formidable";
+import fs from "fs";
 
 export const config = {
     api: {
-      bodyParser: {
-        sizeLimit: '1000mb'
-      }
+        bodyParser: false
     },
-  }
+}
+
+// Helper function to parse form data with formidable
+const parseForm = async (req: NextApiRequest): Promise<{ fields: Fields; files: Files }> =>
+    new Promise((resolve, reject) => {
+        const form = formidable({ keepExtensions: true });
+        form.parse(req, (err, fields, files) => {
+            if (err) reject(err);
+            resolve({ fields, files });
+        });
+    });
+
 
 const mongoFileUploader: NextApiHandler<string> = async (req, res) => {
     if (req.method === 'POST') {
-        const { fileToUpload, experimentId } = req.body;
-
-        if (!fileToUpload || !experimentId) {
+        const { fields, files } = await parseForm(req);
+        const expId = Array.isArray(fields.expId) ? fields.expId[0] : fields.expId;
+
+        if (!files.file || !expId) {
             return res.status(400).json({ response: "Not enough arguments!" } as any);
         }
 
         try {
             const client = await clientPromise;
             const db = client.db(DB_NAME);
             const bucket = new GridFSBucket(db, { bucketName: 'fileBucket' });
-
-            const readableStream = new Readable();
-            readableStream.push(fileToUpload);
-            readableStream.push(null);
 
-            readableStream.
-                pipe(bucket.openUploadStream(`experimentFile${experimentId}`, {
-                    chunkSizeBytes: 1048576,
-                    metadata: { expId: experimentId }
-                }) as any);
+            const file = Array.isArray(files.file) ? files.file[0] : files.file;
+            const fileStream = fs.createReadStream(file.filepath);
+
+            // Upload the file to GridFS
+            const uploadStream = bucket.openUploadStream(file.originalFilename || "uploadedFile", {
+                metadata: { expId: expId },
+            }) as Writable;
 
 
-            res.status(200).json({ response: 'Successfully wrote file!' } as any);
+            // Pipe the file stream to GridFS
+            fileStream.pipe(uploadStream).on("finish", () => {
+                res.status(200).json({ message: "File and ID uploaded successfully." } as any);
+            });
             return;
         }
         catch (error) {

apps/runner/runner.py

@@ -173,19 +173,10 @@ def determine_experiment_file_type(filepath: str):
     filetype = ExperimentType.UNKNOWN
     if 'Python script' in rawfiletype or 'python3' in rawfiletype:
         filetype = ExperimentType.PYTHON
+    elif 'Java archive data (JAR)' in rawfiletype:
+        filetype = ExperimentType.JAVA
     elif 'ELF 64-bit LSB' in rawfiletype:
         filetype = ExperimentType.C
-    else:
-        # check if it is a jar file, they can have multiple rawfiletypes
-        try:
-            file = open(filepath, "r")
-            contents = file.read()
-            if "META-INF/MANIFEST.MF" in contents:
-                filetype = ExperimentType.JAVA
-            file.close()
-        except FileNotFoundError as e:
-            explogger.error(f"{filepath} could not be read to determine if it is a jar file!")
-            raise e
 
     explogger.info(f"Raw Filetype: {rawfiletype}\n Filtered Filetype: {filetype.value}")
 
@@ -207,9 +198,9 @@ def download_experiment_files(experiment: ExperimentData):
         # try to call the backend to download
         url = f'http://glados-service-backend:{os.getenv("BACKEND_PORT")}/downloadExpFile?expId={experiment.expId}'
         response = requests.get(url, timeout=60)
-        file_contents = base64.b64decode(response.json()["contents"]).decode()
+        file_contents = response.content
         # write the file contents to file path
-        with open(filepath, "x") as file:
+        with open(filepath, "xb") as file:
             file.write(file_contents)
 
     except Exception as err: