Skip to content

Commit

Permalink
Merge branch 'master' into dependabot/npm_and_yarn/lab/webapp/minimis…
Browse files Browse the repository at this point in the history 
…t-and-minimist-and-mkdirp-1.2.8
  • Loading branch information
@jay-m-dev
jay-m-dev committed Sep 28, 2023
2 parents c72be65 + 81e10dc commit cca7d59
Show file tree
Hide file tree
Showing 11 changed files with 391 additions and 160 deletions.
3 changes: 3 additions & 0 deletions lab/db.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ db.bind("batches");
db.bind("users");
db.bind("datasets");
db.bind("settings");
db.bind("chats");
db.bind("chatlogs");
db.bind("executions");

// Promisify all methods
Object.keys(mongoskin).forEach((key) => {
Expand Down
88 changes: 87 additions & 1 deletion lab/lab.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ const assert = require("assert");
const openaiRouter = require('./routes/openai');
const chatapiRouter = require('./routes/chatapi');
const execapiRouter = require('./routes/execapi');
const { deleteFilesFromGridstore } = require('./labutils');

/***************
* Enums
Expand Down Expand Up @@ -601,6 +602,7 @@ app.get("/api/v1/:collection/:id", (req, res, next) => {

// Update existing entry
app.put("/api/v1/:collection/:id", jsonParser, (req, res, next) => {

delete req.body._id; // Delete ID (will not update otherwise)
req.collection.updateByIdAsync(req.params.id, {
$set: req.body
Expand All @@ -626,6 +628,91 @@ app.put("/api/v1/:collection/:id", jsonParser, (req, res, next) => {
});
});

app.delete('/api/v1/datasets/:id', async (req, res, next) => {
const result = {};
let files = [];
let query = '';
try {
const dataset_id = db.toObjectID(req.params.id);
let dataset = await db.datasets.findByIdAsync(dataset_id);

if (dataset == null) {
return res.status(404).send({ message: 'dataset ' + req.params.id + ' not found'});
}

const dataset_file_id = db.toObjectID(dataset.files[0]._id);
files.push(...dataset.files);

// experiments
query = { $or: [
{"_dataset_id": {"$eq": dataset_id}},
{"_dataset_file_id": {"$eq": dataset_file_id}},
]}

let experiments = await db.experiments.find(query).toArrayAsync();
let runningExp = experiments.find(exp => exp._status == 'running');

if (runningExp) {
return res.status(409).send({message: 'cannot delete dataset, experiments running'});
}

let experimentIds = []; // maybe I don't need this one.
experiments.forEach(exp => {
experimentIds.push(exp._id);
files.push(...exp.files);
})

// chats
query = { $or: [
{"_dataset_id": {"$eq": dataset_id}},
{"_experiment_id": {"$in": experimentIds}}
]}
let chats = await db.chats.find(query).toArrayAsync();
let chatIds = [];
let chatlogIds = [];
chats.forEach(chat => {
chatIds.push(chat._id);
chatlogIds.push(...chat.chatlogs);
})

// executions
query = { $or: [
{"_dataset_id": {"$eq": dataset_id}},
{"_dataset_file_id": {"$eq": dataset_file_id}},
{"_experiment_id": {"$in": experimentIds}}
]}
let executions = await db.executions.find(query).toArrayAsync();
executions.forEach(exec => {
files.push(...exec.files);
})

// *** DELETE ***
result.datasetCount = await db.datasets.removeByIdAsync(dataset_id);
if (experiments.length > 0) {
result.experimentCount = await db.experiments.removeAsync({'_id': { '$in': experimentIds }});
console.log('experiment count');
}
if (chatIds.length > 0) {
result.chatlogCount = (await db.chatlogs.removeAsync({'_id': { '$in': chatlogIds }}));
console.log('chatlogs deleted');
result.chatCount = (await db.chats.removeAsync({'_id': { '$in': chatIds }}));
console.log('chats deleted');
}
result.fileCount = await deleteFilesFromGridstore(files);

// temp values
// result.datasets = dataset;
// result.experiments = experiments;
// result.chats = chats;
// result.executions = executions;
// result.files = files;

res.send(result);
} catch (err) {
next(err);
}
});

// Delete existing entry
app.delete("/api/v1/:collection/:id", (req, res, next) => {
req.collection.removeByIdAsync(req.params.id)
Expand All @@ -643,7 +730,6 @@ app.delete("/api/v1/:collection/:id", (req, res, next) => {
});



// Experiment page
app.get("/api/v1/experiments/:id", (req, res, next) => {
db.experiments.findByIdAsync(req.params.id)
Expand Down
27 changes: 27 additions & 0 deletions lab/labutils.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
const db = require("./db").db;

async function deleteFilesFromGridstore(files) {
try {
let filesP = [];
let filesDeleted = 0;

for (let i = 0; i < files.length; i++) {
let gfs = new db.GridStore(db, files[i]._id, 'w', {
promiseLibrary: Promise
});
filesP.push(gfs.unlinkAsync().then(() => {
filesDeleted++;
}));
}

await Promise.all(filesP);
return filesDeleted;
} catch (err) {
console.log(err);
throw err;
}
}

module.exports = {
deleteFilesFromGridstore
}
97 changes: 0 additions & 97 deletions lab/routes/execapi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,103 +120,6 @@ router.post('/executions', async (req, res, next) => {
});


router.post('/executions_old', async (req, res, next) => {
if (req.body.src_code == null) {
return res.status(400).json({ message: 'No src_code provided' });
}

// This should not be needed in the code run. The client should take the
// execution_id returned by this enpoint and write it to the next chatlog.
// // this will be the chatlog_id of the chatlog that is requesting this run.
// // the next chatlog will be the one with the results of this run. The client
// // will need to save the execution_id returned by this endpoint in the next
// // chatlog.
// if (req.body.chatlog_id != null) {
// return res.status(400).json({ message: 'no chatlog_id provided' });
// }

// create a new execution
let execution = new Execution({
src_code: req.body.src_code,
status: 'submitted',
result: null,
files: []
});

if (req.body.dataset_file_id != null) {
execution._dataset_file_id = req.body.dataset_file_id;
} else if (req.body.dataset_id != null) {
execution._dataset_id = req.body.dataset_id;
let dataset = await getDatasetById(req.body.dataset_id);
if (dataset != null) {
execution._dataset_file_id = dataset.files[0]._id;
}
}

if (req.body.experiment_id != null) {
execution._experiment_id = req.body.experiment_id;
}

try {
const newExecution = await execution.save();
execution._id = newExecution._id;
} catch (err) {
return res.status(500).json({ message: err.message });
}

// make folder if not available yet:
// let tmppath = path.join(process.env.CODE_RUN_PATH, request.execution_id.toString());
let tmppath = path.join(process.env.CODE_RUN_PATH, execution._id.toString());
// make tmp folder if it is not available
if (!fs.existsSync(tmppath)) fs.mkdirSync(tmppath, { recursive: true });

// find machines that could handle the project
// this may need revision, submitting experiments checks the machine capacity
// but the capacity is tied to each algorithm. Not sure how to handle this yet.
let machines;
try {
machines = await Machine.find({}, { address: 1 });
if (machines.length == 0) {
return res.status(400).json({ message: 'No machines available' });
}
// call the machine api
let result = await fetch(machines[0].address + '/code/run', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(execution)
});
result = await result.json();

// update the execution status
execution.status = result.exec_results.status;
execution.result = result.exec_results.result;

// add any generated files in tmppath to the execution.files array
const files = await uploadExecFiles(execution._id, tmppath);
execution.files = files;

const updatedExecution = await execution.save();

// delete the tmp folder
fs.rmdir(tmppath, { recursive: true }, (err) => {
if (err) {
console.error(err);
} else {
console.log(tmppath + ' folder deleted');
}
});

res.send(execution);
}
catch (err) {
console.error(err);
return res.status(500).json({ message: err.message });
}
});


router.post('/executions/install', async (req, res, next) => {

if (req.body.command != 'install' && req.body.command != 'freeze') {
Expand Down
63 changes: 35 additions & 28 deletions ...webapp/src/components/Datasets/components/DatasetCard/components/DatasetActions/index.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,21 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
(Autogenerated header, do not modify)
*/
import React from 'react';
import { Loader, Dimmer, Checkbox, Popup, Dropdown, Icon } from 'semantic-ui-react';
import React from "react";
import {
Loader,
Dimmer,
Checkbox,
Popup,
Dropdown,
Icon,
} from "semantic-ui-react";

function DatasetActions({ dataset, recommender, toggleAI }) {
const onToggleAI = () => {
const aiState = dataset.ai;
const aiNextState = aiState === 'off' || aiState === 'finished' ? 'requested' : 'off';
const aiNextState =
aiState === "off" || aiState === "finished" ? "requested" : "off";

toggleAI(dataset._id, aiNextState);

Expand All @@ -47,53 +55,52 @@ function DatasetActions({ dataset, recommender, toggleAI }) {
// buildNewExpPopup

localStorage.setItem("addNewPopup", "true");
localStorage.setItem('aiTooglePopup', 'true');
localStorage.setItem('buildNewExpPopup', 'true');


localStorage.setItem("aiTooglePopup", "true");
localStorage.setItem("buildNewExpPopup", "true");
};

// ai states: on, queueing, requested, off, finished
// recommender states: initializing, disabled, active

// if the recommender is in an off or initializing state, disable the ai toggle
const hasMetadata = dataset.has_metadata;
const aiLabelText = 'AI';

const recState = recommender.status;
const aiState = dataset.ai;
const aiLabelText = "AI";

const aiInitializing = (recState === 'initializing') ? true : false;
const aiDisabled = (recState === 'running') ? dataset.isTogglingAI : true;
const aiIsChecked = (aiState === 'off' || aiState === 'finished') ? false : true;
const recState = recommender.status;
const aiState = dataset.ai;

const aiLabelClass = `ai-label ${(aiIsChecked) ? 'bright' : 'dim' }`;
const aiToggleClass = `ai-switch ${(aiState === 'on' || aiState === 'queuing') ? 'active' : aiState }`;
const aiInitializing = recState === "initializing" ? true : false;
const aiDisabled = recState === "running" ? dataset.isTogglingAI : true;
const aiIsChecked =
aiState === "off" || aiState === "finished" ? false : true;

const aiPopupContent = (recState === 'running') ? `AI ${aiState}` : `AI recommender ${recState}`;
const aiLabelClass = `ai-label ${aiIsChecked ? "bright" : "dim"}`;
const aiToggleClass = `ai-switch ${
aiState === "on" || aiState === "queuing" ? "active" : aiState
}`;

const dropdownIcon = <Icon inverted color="grey" size="large" name="caret down" />;
const aiPopupContent =
recState === "running" ? `AI ${aiState}` : `AI recommender ${recState}`;

const dropdownIcon = (
<Icon inverted color="grey" size="large" name="caret down" />
);

if (aiInitializing) {
return (
<span>
<span>
<span className={aiLabelClass}>
{aiLabelText}
</span>
<Loader size='small' active inline indeterminate />
<span className={aiLabelClass}>{aiLabelText}</span>
<Loader size="small" active inline indeterminate />
</span>
</span>
</span>
);
} else {
return (
<span>
{hasMetadata &&
{hasMetadata && (
<span>
<span className={aiLabelClass}>
{aiLabelText}
</span>
<span className={aiLabelClass}>{aiLabelText}</span>
<Popup
content={aiPopupContent}
size="small"
Expand All @@ -109,7 +116,7 @@ function DatasetActions({ dataset, recommender, toggleAI }) {
}
/>
</span>
}
)}
{/*<Dropdown pointing="top right" icon={dropdownIcon}>
<Dropdown.Menu>
<Dropdown.Item icon="file" text="Replace file" />
Expand Down
Loading

0 comments on commit cca7d59

Please sign in to comment.