Skip to content
This repository has been archived by the owner on Jun 4, 2024. It is now read-only.

Implement API to create a grid to store results of a scheduled query #444

Merged
merged 5 commits into from
May 28, 2018
Merged

Implement API to create a grid to store results of a scheduled query #444

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
26ec584
PlotlyAPI: use lowercase filenames
n-riesco May 26, 2018
c26fa9d
settings: add getCredentials(username)
n-riesco May 26, 2018
b3504bf
PlotlyAPI: refactor
n-riesco May 27, 2018
3aa1c34
routes: extend entrypoint POST queries
n-riesco May 27, 2018
728888b
PlotlyAPI: rename getGridMetadat to getGridMeta
n-riesco May 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 72 additions & 7 deletions backend/persistent/QueryScheduler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
import {
getCurrentUser,
getGridMetadata,
newGrid,
updateGrid
} from './plotly-api.js';

Expand All @@ -25,6 +26,7 @@ class QueryScheduler {
this.loadQueries = this.loadQueries.bind(this);
this.clearQuery = this.clearQuery.bind(this);
this.clearQueries = this.clearQueries.bind(this);
this.queryAndCreateGrid = this.queryAndCreateGrid.bind(this);
this.queryAndUpdateGrid = this.queryAndUpdateGrid.bind(this);

// this.job wraps this.queryAndUpdateGrid to avoid concurrent runs of the same job
Expand Down Expand Up @@ -119,7 +121,69 @@ class QueryScheduler {
Object.keys(this.queryJobs).forEach(this.clearQuery);
}

queryAndUpdateGrid (fid, uids, queryString, connectionId, requestor) {
queryAndCreateGrid(filename, query, connectionId, requestor) {
const {username, apiKey, accessToken} = getCredentials(requestor);
let startTime;

// Check if the user even exists
if (!username || !(apiKey || accessToken)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can simplify it:

if !(username || apiKey || accessToken) {

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tricky, but not equivalent: the latter doesn't catch the case when username is defined but both apiKey and accessToken are undefined.

/*
* Warning: The front end looks for "Unauthenticated" in this error message. Don't change it!
*/
const errorMessage = (
'Unauthenticated: Attempting to create a grid but the ' +
`authentication credentials for the user "${username}" do not exist.`
);
Logger.log(errorMessage, 0);
throw new Error(errorMessage);
}

// Check if the credentials are valid
return getCurrentUser(username).then(res => {
if (res.status !== 200) {
const errorMessage = (
`Unauthenticated: ${getSetting('PLOTLY_API_URL')} failed to identify ${username}.`
);
Logger.log(errorMessage, 0);
throw new Error(errorMessage);
}


startTime = process.hrtime();

Logger.log(`Querying "${query}" with connection ${connectionId} to create a new grid`, 2);
return Connections.query(query, getConnectionById(connectionId));

}).then(({rows, columnnames}) => {
Logger.log(`Query "${query}" took ${process.hrtime(startTime)[0]} seconds`, 2);
Logger.log('Create a new grid with new data', 2);
Logger.log(`First row: ${JSON.stringify(rows.slice(0, 1))}`, 2);

startTime = process.hrtime();

return newGrid(
filename,
columnnames,
rows,
requestor
);

}).then(res => {
Logger.log(`Request to Plotly for creating a grid took ${process.hrtime(startTime)[0]} seconds`, 2);

if (res.status !== 201) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be putting this into a constants file with actual codes? I noticed below we have 200, 401, etc. If we used names would make code easier to read

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it'd make the code less readable (in this context it's clear 201 is an HTTP status code).

Logger.log(`Error ${res.status} while creating a grid`, 2);
}

return res.json().then((json) => {
Logger.log(`Grid ${json.file.fid} has been updated.`, 2);
return json;
});
});

}

queryAndUpdateGrid(fid, uids, query, connectionId, requestor) {
const requestedDBConnections = getConnectionById(connectionId);
let startTime = process.hrtime();

Expand Down Expand Up @@ -156,21 +220,22 @@ class QueryScheduler {
throw new Error(errorMessage);
}

Logger.log(`Querying "${queryString}" with connection ${connectionId} to update grid ${fid}`, 2);
return Connections.query(queryString, requestedDBConnections);
Logger.log(`Querying "${query}" with connection ${connectionId} to update grid ${fid}`, 2);
return Connections.query(query, requestedDBConnections);

}).then(rowsAndColumns => {
}).then(({rows}) => {

Logger.log(`Query "${queryString}" took ${process.hrtime(startTime)[0]} seconds`, 2);
Logger.log(`Query "${query}" took ${process.hrtime(startTime)[0]} seconds`, 2);
Logger.log(`Updating grid ${fid} with new data`, 2);
Logger.log(
'First row: ' +
JSON.stringify(rowsAndColumns.rows.slice(0, 1)),
JSON.stringify(rows.slice(0, 1)),
2);

startTime = process.hrtime();

return updateGrid(
rowsAndColumns.rows,
rows,
fid,
uids,
requestor
Expand Down
75 changes: 49 additions & 26 deletions backend/routes.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -643,42 +643,65 @@ export default class Servers {
return next();
});

// register or overwrite a query
// register/update a query (and create/update a grid)
/*
* TODO - Updating a query should be a PATCH or PUT under
* the endpoint `/queries/:fid`
*/
server.post('/queries', function postQueriesHandler(req, res, next) {
// Make the query and update the user's grid
const {fid, uids, query, connectionId, requestor} = req.params;
const {filename, fid, uids, query, connectionId, requestor} = req.params;

// If a filename has been provided,
// make the query and create a new grid
if (filename) {
return that.queryScheduler.queryAndCreateGrid(
filename, query, connectionId, requestor
)
.then((newGridResponse) => {
const queryObject = {
...req.params,
fid: newGridResponse.file.fid,
uids: newGridResponse.file.cols.map(col => col.uid)
};
that.queryScheduler.scheduleQuery(queryObject);
res.json(201, queryObject);
return next();
})
.catch(onError);
}

// If a grid fid has been provided,
// check the user has permission to edit,
// make the query and update the grid
if (fid) {
return checkWritePermissions(fid, requestor).then(function () {
return that.queryScheduler.queryAndUpdateGrid(
fid, uids, query, connectionId, requestor
);
})
.then(() => {
let status;
if (getQuery(req.params.fid)) {
// TODO - Technically, this should be
// under the endpoint `/queries/:fid`
status = 200;
} else {
status = 201;
}
that.queryScheduler.scheduleQuery(req.params);
res.json(status, {});
return next();
})
.catch(onError);
}

// Check that the user has permission to edit the grid
return onError(new Error('Bad request'));

checkWritePermissions(fid, requestor)
.then(function nowQueryAndUpdateGrid() {
return that.queryScheduler.queryAndUpdateGrid(
fid, uids, query, connectionId, requestor
);
})
.then(function returnSuccess() {
let status;
if (getQuery(req.params.fid)) {
// TODO - Technically, this should be
// under the endpoint `/queries/:fid`
status = 200;
} else {
status = 201;
}
that.queryScheduler.scheduleQuery(req.params);
res.json(status, {});
return next();
})
.catch(function returnError(error) {
function onError(error) {
Logger.log(error, 0);
res.json(400, {error: {message: error.message}});
return next();
});

}
});

// delete a query
Expand Down
7 changes: 4 additions & 3 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"test-unit-all": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --bail --full-trace --timeout 90000 --compilers js:babel-register --recursive test/**/*.spec.js",
"test-unit-all-watch": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --bail --full-trace --timeout 90000 --compilers js:babel-register --recursive test/**/*.spec.js --watch",
"test-unit-watch": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --bail --full-trace --timeout 90000 --watch --compilers js:babel-register ",
"test-unit-athena": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.athena.spec.js",
"test-unit-certificates": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/certificates.spec.js",
"test-unit-csv": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.csv.spec.js",
"test-unit-datastores": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/Datastores.spec.js",
Expand All @@ -39,14 +40,14 @@
"test-unit-ibmdb": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.ibmdb.spec.js",
"test-unit-impala": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.impala.spec.js",
"test-unit-livy": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.livy.spec.js",
"test-unit-athena": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.athena.spec.js",
"test-unit-mock": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.mock.spec.js",
"test-unit-oauth2": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.oauth2.spec.js",
"test-unit-oracle": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.oracle.spec.js",
"test-unit-oracle:node": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.oracle.spec.js",
"test-unit-plotly": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/plotly-api.spec.js",
"test-unit-scheduler": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/QueryScheduler.spec.js",
"test-unit-queries": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.queries.spec.js",
"test-unit-routes": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/routes.spec.js",
"test-unit-mock": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.mock.spec.js",
"test-unit-scheduler": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/QueryScheduler.spec.js",
"pack": "cross-env NODE_ENV=production electron-builder --publish=never",
"package": "cross-env NODE_ENV=production node -r babel-register package.js",
"package-all": "yarn run package -- --all",
Expand Down
Loading