From 53a4eae4b0ea75812e258b403e2087ed9a07301e Mon Sep 17 00:00:00 2001 From: John Calderon Date: Mon, 6 Mar 2023 09:17:04 -0500 Subject: [PATCH 01/16] read local yaml file --- skyline-vscode/package.json | 5 + skyline-vscode/react-ui/package-lock.json | 52 ++- skyline-vscode/react-ui/package.json | 1 + skyline-vscode/react-ui/public/dummy.yaml | 19 + skyline-vscode/react-ui/src/App.js | 2 + .../react-ui/src/sections/ProviderPanel.js | 336 +++++++++--------- skyline-vscode/react-ui/src/utils/parsers.js | 15 + skyline-vscode/src/extension.ts | 2 + skyline-vscode/src/skyline_session.ts | 15 +- 9 files changed, 270 insertions(+), 177 deletions(-) create mode 100644 skyline-vscode/react-ui/public/dummy.yaml create mode 100644 skyline-vscode/react-ui/src/utils/parsers.js diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index 59e7720..f67ba80 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -34,6 +34,11 @@ "type": "number", "default": 60120, "description": "Specifies the port of the profiler." + }, + "skyline.providers":{ + "type": "string", + "default": "/home/john/Documents/centml/misc/dummy.yaml", + "description": "location of additional cloud providers" } } } diff --git a/skyline-vscode/react-ui/package-lock.json b/skyline-vscode/react-ui/package-lock.json index a69e29a..51d8a43 100644 --- a/skyline-vscode/react-ui/package-lock.json +++ b/skyline-vscode/react-ui/package-lock.json @@ -1660,6 +1660,14 @@ "resolve-from": "^5.0.0" }, "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -1674,6 +1682,15 @@ "path-exists": "^4.0.0" } }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -3671,12 +3688,9 @@ "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" }, "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "aria-query": { "version": "5.0.2", @@ -8815,12 +8829,11 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" } }, "jsdom": { @@ -11787,6 +11800,14 @@ "color-convert": "^1.9.0" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -11861,6 +11882,15 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "nth-check": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", diff --git a/skyline-vscode/react-ui/package.json b/skyline-vscode/react-ui/package.json index ca60b0e..498fd19 100644 --- a/skyline-vscode/react-ui/package.json +++ b/skyline-vscode/react-ui/package.json @@ -11,6 +11,7 @@ "@testing-library/react": "^12.1.2", "@testing-library/user-event": "^13.5.0", "bootstrap": "^5.1.3", + "js-yaml": "^4.1.0", "less": "^4.1.2", "react": "^17.0.2", "react-bootstrap": "^2.1.2", diff --git a/skyline-vscode/react-ui/public/dummy.yaml b/skyline-vscode/react-ui/public/dummy.yaml new file mode 100644 index 0000000..a9daab6 --- /dev/null +++ b/skyline-vscode/react-ui/public/dummy.yaml @@ -0,0 +1,19 @@ +--- + doe: "a deer, a female deer" + ray: "a drop of golden sun" + pi: 3.14159 + xmas: true + french-hens: 3 + calling-birds: + - huey + - dewey + - louie + - fred + xmas-fifth-day: + calling-birds: four + french-hens: 3 + golden-rings: 5 + partridges: + count: 1 + location: "a pear tree" + turtle-doves: two \ No newline at end of file diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index ae41567..63d946a 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -66,6 +66,7 @@ function App() { const [numIterations, setNumIterations] = useState(100000); App.vscodeApi = vscodeApi; + console.log(vscodeApi); const resetApp = function () { setErrorText(""); @@ -102,6 +103,7 @@ function App() { window.addEventListener("message", (event) => { if (event.data["message_type"] === "connection") { setConnectionStatus(event.data["status"]); + console.log("IM HERE",event.data["file"]); } else if (event.data["message_type"] === "analysis") { processAnalysisState(event.data); } else if (event.data["message_type"] === "text_change") { diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.js index 4b87e03..c1d0c9f 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.js @@ -25,6 +25,8 @@ import { currencyFormat, } from "../utils/utils"; +import {loadYamlFile} from '../utils/parsers'; + const highlightColor = "#9b59b6"; const normalSize = 200; const highlightSize = 280; @@ -58,7 +60,7 @@ const populate_initial_data = (habitatData) => { const ProviderPanel = ({ numIterations, habitatData }) => { const [providerPanelSettings, setProviderPanelSettings] = useState({ - plotData: populate_initial_data(habitatData), + plotData: null, nearest: null, clicked: null, maxNumGpu: 0, @@ -170,171 +172,183 @@ const ProviderPanel = ({ numIterations, habitatData }) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [numIterations]); + useEffect(async()=>{ + const data = await loadYamlFile(); + console.log(data) + setProviderPanelSettings(prevState=>({ + ...prevState, + plotData: populate_initial_data(habitatData) + })) + // eslint-disable-next-line react-hooks/exhaustive-deps + },[]) + return ( <> + {providerPanelSettings.plotData && cloudInstances && gpuPropertyList && cloudProviders ?
- Providers - - - - - - -
-
Filter by provider
- - handleFilterChange("provider", e.target.value) - } - > - - {Object.keys(cloudProviders).map((provider, index) => { - return ( - - ); - })} - -
- - -
-
Filter by GPU
- - handleFilterChange("gpu", e.target.value) - } - > - - {[...gpuList].map((gpu, index) => { - return ( - - ); - })} - -
- - -
-
Filter Max Number of GPUs:
- - {MAX_GPU.map((numgpu) => ( - - handleFilterChange( - "numGpus ", - e.currentTarget.value - ) - } - > - {numgpu === 0 ? "all" : numgpu} - - ))} - -
- - - - - + Providers + + + + + + +
+
Filter by provider
+ + handleFilterChange("provider", e.target.value) + } + > + + {Object.keys(cloudProviders).map((provider, index) => { + return ( + + ); + })} + +
+ + +
+
Filter by GPU
+ + handleFilterChange("gpu", e.target.value) + } + > + + {[...gpuList].map((gpu, index) => { + return ( + + ); + })} + +
+ + +
+
Filter Max Number of GPUs:
+ + {MAX_GPU.map((numgpu) => ( + + handleFilterChange( + "numGpus ", + e.currentTarget.value + ) + } + > + {numgpu === 0 ? "all" : numgpu} + + ))} + +
+ + + + - - -
- Deployment Plan - {providerPanelSettings.clicked && ( - - - - - - - - - -

- {providerPanelSettings.clicked.info.instance} -

- - {`Estimated Cost: ${currencyFormat( - providerPanelSettings.estimated_cost - )}`} + + + +
+ Deployment Plan + {providerPanelSettings.clicked && ( + + + + + + + + + +

+ {providerPanelSettings.clicked.info.instance} +

+ + {`Estimated Cost: ${currencyFormat( + providerPanelSettings.estimated_cost + )}`} + +

+ + {`Estimated Training Time: ${numberFormat( + providerPanelSettings.estimated_time + )} Hours`} -

- - {`Estimated Training Time: ${numberFormat( - providerPanelSettings.estimated_time - )} Hours`} - -

- - - - - - - - - - - - - - - - - - -
GPUNum. GPUVRAM
{providerPanelSettings.clicked.info.gpu} - {providerPanelSettings.clicked.info.ngpus} - {providerPanelSettings.clicked.vmem} GB
- - - - )} - {providerPanelSettings.clicked == null && ( - - - - Select a configuration. - - - - )} -
- - - -
+

+ + + + + + + + + + + + + + + + + + +
GPUNum. GPUVRAM
{providerPanelSettings.clicked.info.gpu} + {providerPanelSettings.clicked.info.ngpus} + {providerPanelSettings.clicked.vmem} GB
+ + + + )} + {providerPanelSettings.clicked == null && ( + + + + Select a configuration. + + + + )} +
+ + + + + :
Loading
} ); }; diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js new file mode 100644 index 0000000..d8830ca --- /dev/null +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -0,0 +1,15 @@ +/** + * Yaml parser + */ + +import { load } from "js-yaml"; + +export const loadYamlFile = async() => { + let result = null; + await fetch('dummy.yaml') + .then((response) => response.text()) + .then((yamlString) => load(yamlString)) + .then((data) => result=data) + .catch((e) => console.log(e)); +return result; +}; diff --git a/skyline-vscode/src/extension.ts b/skyline-vscode/src/extension.ts index 3a31113..f630d24 100644 --- a/skyline-vscode/src/extension.ts +++ b/skyline-vscode/src/extension.ts @@ -12,6 +12,7 @@ export function activate(context: vscode.ExtensionContext) { let disposable = vscode.commands.registerCommand('skyline-vscode.cmd_begin_analyze', () => { let vsconfig = vscode.workspace.getConfiguration('skyline'); + console.log("GAAAAAAAAAAAAAAAAAA",vsconfig); let options: vscode.OpenDialogOptions = { canSelectFiles: false, @@ -44,6 +45,7 @@ export function activate(context: vscode.ExtensionContext) { projectRoot: uri[0].fsPath, addr: vsconfig.address, port: vsconfig.port, + providers: vsconfig.providers, webviewPanel: panel }; diff --git a/skyline-vscode/src/skyline_session.ts b/skyline-vscode/src/skyline_session.ts index 590a687..1ae925f 100644 --- a/skyline-vscode/src/skyline_session.ts +++ b/skyline-vscode/src/skyline_session.ts @@ -16,6 +16,7 @@ export interface SkylineSessionOptions { projectRoot: string; addr: string; port: number; + providers: string; webviewPanel: vscode.WebviewPanel } @@ -28,6 +29,7 @@ export class SkylineSession { connection: Socket; port: number; addr: string; + providers:string; seq_num: number; last_length: number; message_buffer: Uint8Array; @@ -55,15 +57,17 @@ export class SkylineSession { reactProjectRoot: string; constructor(options: SkylineSessionOptions, environ: SkylineEnvironment) { - console.log("SkylineSession instantiated"); + console.log("SkylineSession instantiated !!!!"); this.resetBackendConnection = false; this.connection = new Socket(); this.connection.on('connect', this.on_connect.bind(this)); this.connection.on('data', this.on_data.bind(this)); this.connection.on('close', this.on_close_connection.bind(this)); - this.port = options.port - this.addr = options.addr + this.port = options.port; + this.addr = options.addr; + this.providers = options.providers; + console.log("port",this.port,"address", this.addr,"file",this.providers); this.seq_num = 0; this.last_length = -1; @@ -106,7 +110,8 @@ export class SkylineSession { on_connect() { let connectionMessage = { "message_type": "connection", - "status": true + "status": true, + "file": this.providers }; this.webviewPanel.webview.postMessage(connectionMessage); } @@ -361,7 +366,7 @@ export class SkylineSession { Skyline - + From 19aecdad816d7c4b1c3949f3bcb188740dbcfa37 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Wed, 15 Mar 2023 16:07:24 -0400 Subject: [PATCH 02/16] completed unit testing --- skyline-vscode/package.json | 5 +- skyline-vscode/react-ui/package-lock.json | 328 +++++++++----- skyline-vscode/react-ui/package.json | 8 +- .../react-ui/public/data/providers.yaml | 146 +++++++ skyline-vscode/react-ui/public/dummy.yaml | 19 - skyline-vscode/react-ui/src/App.js | 7 +- .../react-ui/src/components/ScatterGraph.js | 2 +- .../react-ui/src/data/properties.js | 23 + skyline-vscode/react-ui/src/data/providers.js | 283 ------------ .../react-ui/src/sections/DeploymentTab.js | 3 +- .../src/sections/DeploymentTab.test.js | 40 +- .../react-ui/src/sections/ProviderPanel.js | 409 +++++++++--------- .../src/sections/ProviderPanel.test.js | 29 +- skyline-vscode/react-ui/src/utils/parsers.js | 91 +++- .../react-ui/src/utils/parsers.test.js | 91 ++++ skyline-vscode/src/extension.ts | 1 - skyline-vscode/src/skyline_session.ts | 19 +- skyline-vscode/src/utils.ts | 15 + 18 files changed, 859 insertions(+), 660 deletions(-) create mode 100644 skyline-vscode/react-ui/public/data/providers.yaml delete mode 100644 skyline-vscode/react-ui/public/dummy.yaml create mode 100644 skyline-vscode/react-ui/src/data/properties.js delete mode 100644 skyline-vscode/react-ui/src/data/providers.js create mode 100644 skyline-vscode/react-ui/src/utils/parsers.test.js diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index f67ba80..a65567c 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -35,9 +35,9 @@ "default": 60120, "description": "Specifies the port of the profiler." }, - "skyline.providers":{ + "skyline.providers": { "type": "string", - "default": "/home/john/Documents/centml/misc/dummy.yaml", + "default": "", "description": "location of additional cloud providers" } } @@ -67,6 +67,7 @@ }, "dependencies": { "google-protobuf": "^3.18.0", + "js-yaml": "^4.1.0", "ts-protoc-gen": "^0.15.0" } } diff --git a/skyline-vscode/react-ui/package-lock.json b/skyline-vscode/react-ui/package-lock.json index 51d8a43..e9987f2 100644 --- a/skyline-vscode/react-ui/package-lock.json +++ b/skyline-vscode/react-ui/package-lock.json @@ -5,9 +5,10 @@ "requires": true, "dependencies": { "@adobe/css-tools": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.0.1.tgz", - "integrity": "sha512-+u76oB43nOHrF4DDWRLWDCtci7f3QJoEBigemIdIeTi1ODqjx6Tad9NCVnPRwewWlKkVab5PlK8DCtPTyX7S8g==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.2.0.tgz", + "integrity": "sha512-E09FiIft46CmH5Qnjb0wsW54/YQd69LsxeKUOWawmws1XWvyFGURnAChH0mlr7YPFR1ofwvUQfcL0J3lMxXqPA==", + "dev": true }, "@ampproject/remapping": { "version": "2.2.0", @@ -1933,11 +1934,12 @@ } }, "@jest/expect-utils": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.4.1.tgz", - "integrity": "sha512-w6YJMn5DlzmxjO00i9wu2YSozUYRBhIoJ6nQwpMYcBMtiqMGJm1QBzOf6DDgRao8dbtpDoaqLg6iiQTvv0UHhQ==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.5.0.tgz", + "integrity": "sha512-fmKzsidoXQT2KwnrwE0SQq3uj8Z763vzR8LnLBwC2qYWEFpjX8daRsk6rHUM1QvNlEW/UJXNXm59ztmJJWs2Mg==", + "dev": true, "requires": { - "jest-get-type": "^29.2.0" + "jest-get-type": "^29.4.3" } }, "@jest/fake-timers": { @@ -2190,9 +2192,10 @@ } }, "@jest/schemas": { - "version": "29.4.0", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.4.0.tgz", - "integrity": "sha512-0E01f/gOZeNTG76i5eWWSupvSHaIINrTie7vCyjiYFKgzNdyEGd12BUv4oNBFHOqlHDbtoJi3HrQ38KCC90NsQ==", + "version": "29.4.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.4.3.tgz", + "integrity": "sha512-VLYKXQmtmuEz6IxJsrZwzG9NvtkQsWNnWMsKxqWNu3+CnfzJQhp0WDDKWLVV9hLKr0l3SLLFRqcYHjhtyuDVxg==", + "dev": true, "requires": { "@sinclair/typebox": "^0.25.16" } @@ -2327,11 +2330,12 @@ } }, "@jest/types": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.4.1.tgz", - "integrity": "sha512-zbrAXDUOnpJ+FMST2rV7QZOgec8rskg2zv8g2ajeqitp4tvZiyqTCYXANrKsM+ryj5o+LI+ZN2EgU9drrkiwSA==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.5.0.tgz", + "integrity": "sha512-qbu7kN6czmVRc3xWFQcAN03RAUamgppVUdXrvl1Wr3jlNF93o9mJbGcDWrwGB6ht44u7efB1qCFgVQmca24Uog==", + "dev": true, "requires": { - "@jest/schemas": "^29.4.0", + "@jest/schemas": "^29.4.3", "@types/istanbul-lib-coverage": "^2.0.0", "@types/istanbul-reports": "^3.0.0", "@types/node": "*", @@ -2343,6 +2347,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -2552,9 +2557,10 @@ "integrity": "sha512-LwzQKA4vzIct1zNZzBmRKI9QuNpLgTQMEjsQLf3BXuGYb3QPTP4Yjf6mkdX+X1mYttZ808QpOwAzZjv28kq7DA==" }, "@sinclair/typebox": { - "version": "0.25.21", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.21.tgz", - "integrity": "sha512-gFukHN4t8K4+wVC+ECqeqwzBDeFeTzBXroBTqE6vcWrQGbEUpHO7LYdG0f4xnvYq4VOEwITSlHlp0JBAIFMS/g==" + "version": "0.25.24", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz", + "integrity": "sha512-XJfwUVUKDHF5ugKwIcxEgc9k8b7HbznCp6eUfWgu710hMPNIO4aw4/zB5RogDQz8nd6gyCDpU9O/m6qYEWY6yQ==", + "dev": true }, "@sinonjs/commons": { "version": "1.8.6", @@ -2693,13 +2699,14 @@ } }, "@testing-library/dom": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.17.1.tgz", - "integrity": "sha512-KnH2MnJUzmFNPW6RIKfd+zf2Wue8mEKX0M3cpX6aKl5ZXrJM1/c/Pc8c2xDNYQCnJO48Sm5ITbMXgqTr3h4jxQ==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.20.0.tgz", + "integrity": "sha512-d9ULIT+a4EXLX3UU8FBjauG9NnsZHkHztXoIcTsOKoOw030fyjheN9svkTULjJxtYag9DZz5Jz5qkWZDPxTFwA==", + "dev": true, "requires": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", - "@types/aria-query": "^4.2.0", + "@types/aria-query": "^5.0.1", "aria-query": "^5.0.0", "chalk": "^4.1.0", "dom-accessibility-api": "^0.5.9", @@ -2711,6 +2718,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -2722,6 +2730,7 @@ "version": "5.16.5", "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-5.16.5.tgz", "integrity": "sha512-N5ixQ2qKpi5OLYfwQmUb/5mSV9LneAcaUfp32pn4yCnpb8r/Yz0pXFPck21dIicKmi+ta5WRAknkZCfA8refMA==", + "dev": true, "requires": { "@adobe/css-tools": "^4.0.1", "@babel/runtime": "^7.9.2", @@ -2735,19 +2744,20 @@ } }, "@testing-library/react": { - "version": "12.1.5", - "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-12.1.5.tgz", - "integrity": "sha512-OfTXCJUFgjd/digLUuPxa0+/3ZxsQmE7ub9kcbW/wi96Bh3o/p5vrETcBGfP17NWPGqeYYl5LTRpwyGoMC4ysg==", + "version": "12.1.2", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-12.1.2.tgz", + "integrity": "sha512-ihQiEOklNyHIpo2Y8FREkyD1QAea054U0MVbwH1m8N9TxeFz+KoJ9LkqoKqJlzx2JDm56DVwaJ1r36JYxZM05g==", + "dev": true, "requires": { "@babel/runtime": "^7.12.5", - "@testing-library/dom": "^8.0.0", - "@types/react-dom": "<18.0.0" + "@testing-library/dom": "^8.0.0" } }, "@testing-library/user-event": { "version": "13.5.0", "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-13.5.0.tgz", "integrity": "sha512-5Kwtbo3Y/NowpkbRuSepbyMFkZmHgD+vPzYB/RJ4oxt5Gj/avFFBYjhw27cqSVPVw/3a67NK1PbiIr9k4Gwmdg==", + "dev": true, "requires": { "@babel/runtime": "^7.12.5" } @@ -2763,9 +2773,10 @@ "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==" }, "@types/aria-query": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-4.2.2.tgz", - "integrity": "sha512-HnYpAE1Y6kRyKM/XkEuiRQhTHvkzMBurTHnpFLYLBGPIylZNPs9jJcuOOYWxPLJCSEtmZT0Y8rHDokKN7rRTig==" + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.1.tgz", + "integrity": "sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==", + "dev": true }, "@types/babel__core": { "version": "7.1.19", @@ -2979,9 +2990,10 @@ } }, "@types/jest": { - "version": "29.0.0", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.0.0.tgz", - "integrity": "sha512-X6Zjz3WO4cT39Gkl0lZ2baFRaEMqJl5NC1OjElkwtNzAlbkr2K/WJXkBkH5VP0zx4Hgsd2TZYdOEfvp2Dxia+Q==", + "version": "29.4.1", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.4.1.tgz", + "integrity": "sha512-zDQSWXG+ZkEvs2zFFMszePhx4euKz+Yt3Gg1P+RHjfJBinTTr6L2DEyovO4V/WrKXuF0Dgn56GWGZPDa6TW9eQ==", + "dev": true, "requires": { "expect": "^29.0.0", "pretty-format": "^29.0.0" @@ -2990,14 +3002,16 @@ "ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==" + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true }, "pretty-format": { - "version": "29.0.2", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.0.2.tgz", - "integrity": "sha512-wp3CdtUa3cSJVFn3Miu5a1+pxc1iPIQTenOAn+x5erXeN1+ryTcLesV5pbK/rlW5EKwp27x38MoYfNGaNXDDhg==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.5.0.tgz", + "integrity": "sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw==", + "dev": true, "requires": { - "@jest/schemas": "^29.0.0", + "@jest/schemas": "^29.4.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } @@ -3005,7 +3019,8 @@ "react-is": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "dev": true } } }, @@ -3069,14 +3084,6 @@ "csstype": "^3.0.2" } }, - "@types/react-dom": { - "version": "17.0.17", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-17.0.17.tgz", - "integrity": "sha512-VjnqEmqGnasQKV0CWLevqMTXBYG9GbwuE6x3VetERLh0cq2LTptFE73MrQi2S7GkKXCf2GgwItB/melLnxfnsg==", - "requires": { - "@types/react": "^17" - } - }, "@types/react-transition-group": { "version": "4.4.5", "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.5.tgz", @@ -3137,6 +3144,7 @@ "version": "5.14.5", "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz", "integrity": "sha512-SBwbxYoyPIvxHbeHxTZX2Pe/74F/tX2/D3mMvzabdeJ25bBojfW0TyB8BHrbq/9zaaKICJZjLP+8r6AeZMFCuQ==", + "dev": true, "requires": { "@types/jest": "*" } @@ -3695,7 +3703,8 @@ "aria-query": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.0.2.tgz", - "integrity": "sha512-eigU3vhqSO+Z8BKDnVLN/ompjhf3pYzecKXz8+whRy+9gZu8n1TCGfwzQUUPnqdHl9ax1Hr9031orZ+UOEYr7Q==" + "integrity": "sha512-eigU3vhqSO+Z8BKDnVLN/ompjhf3pYzecKXz8+whRy+9gZu8n1TCGfwzQUUPnqdHl9ax1Hr9031orZ+UOEYr7Q==", + "dev": true }, "array-flatten": { "version": "2.1.2", @@ -4041,6 +4050,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -4181,6 +4195,15 @@ "node-int64": "^0.4.0" } }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -4259,6 +4282,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -4600,6 +4624,15 @@ "yaml": "^1.10.0" } }, + "cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "dev": true, + "requires": { + "node-fetch": "2.6.7" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -4770,7 +4803,8 @@ "css.escape": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", - "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==" + "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", + "dev": true }, "cssdb": { "version": "7.0.1", @@ -5060,9 +5094,10 @@ "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==" }, "diff-sequences": { - "version": "29.3.1", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.3.1.tgz", - "integrity": "sha512-hlM3QR272NXCi4pq+N4Kok4kOp6EsgOM3ZSpJI7Da3UAs+Ttsi8MRmB6trM/lhyzUxGfOgnpkHtgqm5Q/CTcfQ==" + "version": "29.4.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.4.3.tgz", + "integrity": "sha512-ofrBgwpPhCD85kMKtE9RYFFq6OC1A89oW2vvgWZNCwxrUpRUILopY7lsYyMDSjc8g6U6aiO0Qubg6r4Wgt5ZnA==", + "dev": true }, "dir-glob": { "version": "3.0.1", @@ -5101,7 +5136,8 @@ "dom-accessibility-api": { "version": "0.5.14", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.14.tgz", - "integrity": "sha512-NMt+m9zFMPZe0JcY9gN224Qvk6qLIdqex29clBvc/y75ZBX9YA9wNK3frsYvu2DI1xcCIwxwnX+TlsJ2DSOADg==" + "integrity": "sha512-NMt+m9zFMPZe0JcY9gN224Qvk6qLIdqex29clBvc/y75ZBX9YA9wNK3frsYvu2DI1xcCIwxwnX+TlsJ2DSOADg==", + "dev": true }, "dom-converter": { "version": "0.2.0", @@ -5859,15 +5895,16 @@ "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==" }, "expect": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/expect/-/expect-29.4.1.tgz", - "integrity": "sha512-OKrGESHOaMxK3b6zxIq9SOW8kEXztKff/Dvg88j4xIJxur1hspEbedVkR3GpHe5LO+WB2Qw7OWN0RMTdp6as5A==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.5.0.tgz", + "integrity": "sha512-yM7xqUrCO2JdpFo4XpM82t+PJBFybdqoQuJLDGeDX2ij8NZzqRHyu3Hp188/JX7SWqud+7t4MUdvcgGBICMHZg==", + "dev": true, "requires": { - "@jest/expect-utils": "^29.4.1", - "jest-get-type": "^29.2.0", - "jest-matcher-utils": "^29.4.1", - "jest-message-util": "^29.4.1", - "jest-util": "^29.4.1" + "@jest/expect-utils": "^29.5.0", + "jest-get-type": "^29.4.3", + "jest-matcher-utils": "^29.5.0", + "jest-message-util": "^29.5.0", + "jest-util": "^29.5.0" } }, "express": { @@ -6713,6 +6750,11 @@ "harmony-reflect": "^1.4.6" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "ignore": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", @@ -6762,7 +6804,8 @@ "indent-string": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==" + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true }, "inflight": { "version": "1.0.6", @@ -7441,31 +7484,34 @@ } }, "jest-diff": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.4.1.tgz", - "integrity": "sha512-uazdl2g331iY56CEyfbNA0Ut7Mn2ulAG5vUaEHXycf1L6IPyuImIxSz4F0VYBKi7LYIuxOwTZzK3wh5jHzASMw==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.5.0.tgz", + "integrity": "sha512-LtxijLLZBduXnHSniy0WMdaHjmQnt3g5sa16W4p0HqukYTTsyTW3GD1q41TyGl5YFXj/5B2U6dlh5FM1LIMgxw==", + "dev": true, "requires": { "chalk": "^4.0.0", - "diff-sequences": "^29.3.1", - "jest-get-type": "^29.2.0", - "pretty-format": "^29.4.1" + "diff-sequences": "^29.4.3", + "jest-get-type": "^29.4.3", + "pretty-format": "^29.5.0" }, "dependencies": { "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "pretty-format": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.4.1.tgz", - "integrity": "sha512-dt/Z761JUVsrIKaY215o1xQJBGlSmTx/h4cSqXqjHLnU1+Kt+mavVE7UgqJJO5ukx5HjSswHfmXz4LjS2oIJfg==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.5.0.tgz", + "integrity": "sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw==", + "dev": true, "requires": { - "@jest/schemas": "^29.4.0", + "@jest/schemas": "^29.4.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" }, @@ -7473,14 +7519,16 @@ "ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==" + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true } } }, "react-is": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "dev": true } } }, @@ -7668,10 +7716,21 @@ } } }, + "jest-fetch-mock": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/jest-fetch-mock/-/jest-fetch-mock-3.0.3.tgz", + "integrity": "sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw==", + "dev": true, + "requires": { + "cross-fetch": "^3.0.4", + "promise-polyfill": "^8.1.3" + } + }, "jest-get-type": { - "version": "29.2.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.2.0.tgz", - "integrity": "sha512-uXNJlg8hKFEnDgFsrCjznB+sTxdkuqiCL6zMgA75qEbAJjJYTs9XPrvDctrEig2GDow22T/LvHgO57iJhXB/UA==" + "version": "29.4.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.4.3.tgz", + "integrity": "sha512-J5Xez4nRRMjk8emnTpWrlkyb9pfRQQanDrvWHhsR1+VUfbwxi30eVcZFlcdGInRibU4G5LwHXpI7IRHU0CY+gg==", + "dev": true }, "jest-haste-map": { "version": "27.5.1", @@ -7881,31 +7940,34 @@ } }, "jest-matcher-utils": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.4.1.tgz", - "integrity": "sha512-k5h0u8V4nAEy6lSACepxL/rw78FLDkBnXhZVgFneVpnJONhb2DhZj/Gv4eNe+1XqQ5IhgUcqj745UwH0HJmMnA==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.5.0.tgz", + "integrity": "sha512-lecRtgm/rjIK0CQ7LPQwzCs2VwW6WAahA55YBuI+xqmhm7LAaxokSB8C97yJeYyT+HvQkH741StzpU41wohhWw==", + "dev": true, "requires": { "chalk": "^4.0.0", - "jest-diff": "^29.4.1", - "jest-get-type": "^29.2.0", - "pretty-format": "^29.4.1" + "jest-diff": "^29.5.0", + "jest-get-type": "^29.4.3", + "pretty-format": "^29.5.0" }, "dependencies": { "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "pretty-format": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.4.1.tgz", - "integrity": "sha512-dt/Z761JUVsrIKaY215o1xQJBGlSmTx/h4cSqXqjHLnU1+Kt+mavVE7UgqJJO5ukx5HjSswHfmXz4LjS2oIJfg==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.5.0.tgz", + "integrity": "sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw==", + "dev": true, "requires": { - "@jest/schemas": "^29.4.0", + "@jest/schemas": "^29.4.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" }, @@ -7913,29 +7975,32 @@ "ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==" + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true } } }, "react-is": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "dev": true } } }, "jest-message-util": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.4.1.tgz", - "integrity": "sha512-H4/I0cXUaLeCw6FM+i4AwCnOwHRgitdaUFOdm49022YD5nfyr8C/DrbXOBEyJaj+w/y0gGJ57klssOaUiLLQGQ==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.5.0.tgz", + "integrity": "sha512-Kijeg9Dag6CKtIDA7O21zNTACqD5MD/8HfIV8pdD94vFyFuer52SigdC3IQMhab3vACxXMiFk+yMHNdbqtyTGA==", + "dev": true, "requires": { "@babel/code-frame": "^7.12.13", - "@jest/types": "^29.4.1", + "@jest/types": "^29.5.0", "@types/stack-utils": "^2.0.0", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "micromatch": "^4.0.4", - "pretty-format": "^29.4.1", + "pretty-format": "^29.5.0", "slash": "^3.0.0", "stack-utils": "^2.0.3" }, @@ -7944,17 +8009,19 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "pretty-format": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.4.1.tgz", - "integrity": "sha512-dt/Z761JUVsrIKaY215o1xQJBGlSmTx/h4cSqXqjHLnU1+Kt+mavVE7UgqJJO5ukx5HjSswHfmXz4LjS2oIJfg==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.5.0.tgz", + "integrity": "sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw==", + "dev": true, "requires": { - "@jest/schemas": "^29.4.0", + "@jest/schemas": "^29.4.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" }, @@ -7962,14 +8029,16 @@ "ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==" + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true } } }, "react-is": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "dev": true } } }, @@ -8452,11 +8521,12 @@ } }, "jest-util": { - "version": "29.4.1", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.4.1.tgz", - "integrity": "sha512-bQy9FPGxVutgpN4VRc0hk6w7Hx/m6L53QxpDreTZgJd9gfx/AV2MjyPde9tGyZRINAUrSv57p2inGBu2dRLmkQ==", + "version": "29.5.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.5.0.tgz", + "integrity": "sha512-RYMgG/MTadOr5t8KdhejfvUU82MxsCu5MF6KuDUHl+NuwzUt+Sm6jJWxTJVrDR1j5M/gJVCPKQEpWXY+yIQ6lQ==", + "dev": true, "requires": { - "@jest/types": "^29.4.1", + "@jest/types": "^29.5.0", "@types/node": "*", "chalk": "^4.0.0", "ci-info": "^3.2.0", @@ -8468,6 +8538,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -9070,9 +9141,10 @@ } }, "lz-string": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.4.4.tgz", - "integrity": "sha512-0ckx7ZHRPqb0oUm8zNr+90mtf9DQB60H1wMCjBtfi62Kl3a7JbHob6gA2bC+xRvZoOL+1hzUK8jeuEIQE8svEQ==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true }, "magic-string": { "version": "0.25.9", @@ -9173,7 +9245,8 @@ "min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", - "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==" + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true }, "mini-css-extract-plugin": { "version": "2.6.1", @@ -9306,6 +9379,39 @@ "tslib": "^2.0.3" } }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dev": true, + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, "node-forge": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", @@ -10422,6 +10528,12 @@ "asap": "~2.0.6" } }, + "promise-polyfill": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", + "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==", + "dev": true + }, "prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -10925,6 +11037,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", + "dev": true, "requires": { "indent-string": "^4.0.0", "strip-indent": "^3.0.0" @@ -11690,6 +11803,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, "requires": { "min-indent": "^1.0.0" } diff --git a/skyline-vscode/react-ui/package.json b/skyline-vscode/react-ui/package.json index 498fd19..6ef3bcf 100644 --- a/skyline-vscode/react-ui/package.json +++ b/skyline-vscode/react-ui/package.json @@ -7,10 +7,8 @@ "@fortawesome/fontawesome-svg-core": "^1.2.36", "@fortawesome/free-solid-svg-icons": "^5.15.4", "@fortawesome/react-fontawesome": "^0.1.17", - "@testing-library/jest-dom": "^5.16.1", - "@testing-library/react": "^12.1.2", - "@testing-library/user-event": "^13.5.0", "bootstrap": "^5.1.3", + "buffer": "^6.0.3", "js-yaml": "^4.1.0", "less": "^4.1.2", "react": "^17.0.2", @@ -47,7 +45,11 @@ ] }, "devDependencies": { + "@testing-library/jest-dom": "^5.16.5", + "@testing-library/react": "^12.1.2", + "@testing-library/user-event": "^13.5.0", "jest": "^27.5.1", + "jest-fetch-mock": "^3.0.3", "webpack-bundle-analyzer": "^4.8.0" } } diff --git a/skyline-vscode/react-ui/public/data/providers.yaml b/skyline-vscode/react-ui/public/data/providers.yaml new file mode 100644 index 0000000..5ee09c8 --- /dev/null +++ b/skyline-vscode/react-ui/public/data/providers.yaml @@ -0,0 +1,146 @@ + +#GPU Instances +#costs are 1 GPU * hr +#For Google we need to add the cost for the instance itself, a100 run on specific A2 Series +#while t4, p4, v100, and p100 run on General purposes N1 Series + +#References: +#FOR AWS: +#https://aws.amazon.com/ec2/instance-types/ +#https://aws.amazon.com/ec2/pricing/on-demand/ + +#FOR GOOGLE: +#https://cloud.google.com/compute/gpus-pricing +#https://cloud.google.com/products/calculator/ + +#FOR AZURE: +#https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-gpu +#https://azure.microsoft.com/en-ca/pricing/details/virtual-machines/windows/#pricing +# +--- + google: + name: "Google Cloud Platform" + logo: "resources/google.png" + color: "#ea4335" + instances: + - name: "a2-highgpu-1g" + gpu: "a100" + ngpus: 1 + cost: 3.67 + - name: "n1-standard-1" + gpu: "t4" + ngpus: 1 + cost: 0.4 + - name: "n1-standard-1" + gpu: "p4" + ngpus: 1 + cost: 0.65 + - name: "n1-standard-1" + gpu: "v100" + ngpus: 1 + cost: 2.53 + - name: "n1-standard-1" + gpu: "p100" + ngpus: 1 + cost: 1.51 + - name: "a2-highgpu-2g" + gpu: "a100" + ngpus: 2 + cost: 7.35 + - name: "n1-standard-1" + gpu: "t4" + ngpus: 2 + cost: 0.75 + - name: "n1-standard-1" + gpu: "p4" + ngpus: 2 + cost: 1.25 + - name: "n1-standard-1" + gpu: "v100" + ngpus: 2 + cost: 5.01 + - name: "n1-standard-1" + gpu: "p100" + ngpus: 2 + cost: 2.97 + - name: "a2-highgpu-4g" + gpu: "a100" + ngpus: 4 + cost: 14.7 + - name: "n1-standard-1" + gpu: "t4" + ngpus: 4 + cost: 1.45 + - name: "n1-standard-1" + gpu: "p4" + ngpus: 4 + cost: 2.45 + - name: "n1-standard-1" + gpu: "v100" + ngpus: 4 + cost: 9.97 + - name: "n1-standard-1" + gpu: "p100" + ngpus: 4 + cost: 5.89 + aws: + name: "Amazon Web Services" + logo: "resources/aws.png" + color: "#ff9900" + instances: + - name: "p3.2xlarge" + gpu: "v100" + ngpus: 1 + cost: 3.06 + - name: "g4dn.xlarge" + gpu: "t4" + ngpus: 1 + cost: 0.526 + - name: "g4dn.12xlarge" + gpu: "t4" + ngpus: 4 + cost: 3.912 + - name: "p3.8xlarge" + gpu: "v100" + ngpus: 4 + cost: 12.24 + azure: + name: "Microsoft Azure" + logo: "resources/azure.png" + color: "#008AD7" + instances: + - name: "NC6s v2" + gpu: "p100" + ngpus: 1 + cost: 2.07 + - name: "NC6s v3" + gpu: "v100" + ngpus: 1 + cost: 3.06 + - name: "NC4as T4 v3" + gpu: "t4" + ngpus: 1 + cost: 0.526 + - name: "NC12s v2" + gpu: "p100" + ngpus: 2 + cost: 4.14 + - name: "NC12s v3" + gpu: "v100" + ngpus: 2 + cost: 6.12 + - name: "NC24rs v2" + gpu: "p100" + ngpus: 4 + cost: 9.108 + - name: "NC24rs v3" + gpu: "v100" + ngpus: 4 + cost: 13.46 + - name: "NC64as T4 v3" + gpu: "t4" + ngpus: 4 + cost: 4.352 + + + \ No newline at end of file diff --git a/skyline-vscode/react-ui/public/dummy.yaml b/skyline-vscode/react-ui/public/dummy.yaml deleted file mode 100644 index a9daab6..0000000 --- a/skyline-vscode/react-ui/public/dummy.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- - doe: "a deer, a female deer" - ray: "a drop of golden sun" - pi: 3.14159 - xmas: true - french-hens: 3 - calling-birds: - - huey - - dewey - - louie - - fred - xmas-fifth-day: - calling-birds: four - french-hens: 3 - golden-rings: 5 - partridges: - count: 1 - location: "a pear tree" - turtle-doves: two \ No newline at end of file diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 63d946a..9c9b9aa 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -64,9 +64,9 @@ function App() { const [errorText, setErrorText] = useState(); const [connectionStatus, setConnectionStatus] = useState(false); const [numIterations, setNumIterations] = useState(100000); + const [externalData, setExternalData] = useState(null); App.vscodeApi = vscodeApi; - console.log(vscodeApi); const resetApp = function () { setErrorText(""); @@ -103,9 +103,11 @@ function App() { window.addEventListener("message", (event) => { if (event.data["message_type"] === "connection") { setConnectionStatus(event.data["status"]); - console.log("IM HERE",event.data["file"]); } else if (event.data["message_type"] === "analysis") { processAnalysisState(event.data); + } else if (event.data["message_type"] === "loaded_additional_providers") { + console.log("received loaded file",event.data["additionalProviders"]); + setExternalData(event.data["additionalProviders"]); } else if (event.data["message_type"] === "text_change") { setTextChanged(true); } else if (event.data["message_type"] === "error") { @@ -223,6 +225,7 @@ function App() { diff --git a/skyline-vscode/react-ui/src/components/ScatterGraph.js b/skyline-vscode/react-ui/src/components/ScatterGraph.js index bfbe5f2..8ab8fdf 100644 --- a/skyline-vscode/react-ui/src/components/ScatterGraph.js +++ b/skyline-vscode/react-ui/src/components/ScatterGraph.js @@ -14,7 +14,7 @@ import { } from "recharts"; import { calculate_training_time, currencyFormat } from "../utils/utils"; -import { gpuPropertyList } from "../data/providers"; +import { gpuPropertyList } from "../data/properties"; export const ProviderScatterGraph = ({ data, diff --git a/skyline-vscode/react-ui/src/data/properties.js b/skyline-vscode/react-ui/src/data/properties.js new file mode 100644 index 0000000..8325bb4 --- /dev/null +++ b/skyline-vscode/react-ui/src/data/properties.js @@ -0,0 +1,23 @@ +export const gpuPropertyList = [ + { name: "p100", vmem: 16, type: "server" }, + { name: "p4000", vmem: 8, type: "server" }, + { name: "rtx2070", vmem: 8, type: "consumer" }, + { name: "rtx2080ti", vmem: 11, type: "consumer" }, + { name: "t4", vmem: 16, type: "server" }, + { name: "v100", vmem: 16, type: "server" }, + { name: "a100", vmem: 40, type: "server" }, + { name: "rtx3090", vmem: 24, type: "consumer" }, + { name: "a40", vmem: 48, type: "server" }, + { name: "a4000", vmem: 16, type: "server" }, + { name: "rtx4000", vmem: 8, type: "consumer" }, +]; + +/** + * COLOR DATA FOR DEPLOYMENT SCATTER GRAPH + */ +export const deploymentScatterGraphColorSize = { + HIGHLIGHTCOLOR: "#9b59b6", + NORMALSIZE: 200, + HIGHLIGHTSIZE: 280, +}; + diff --git a/skyline-vscode/react-ui/src/data/providers.js b/skyline-vscode/react-ui/src/data/providers.js deleted file mode 100644 index 5ec8a13..0000000 --- a/skyline-vscode/react-ui/src/data/providers.js +++ /dev/null @@ -1,283 +0,0 @@ -/** - * GPU Instances - * costs are 1 GPU * hr - * For Google we need to add the cost for the instance itself, a100 run on specific A2 Series - * while t4, p4, v100, and p100 run on General purposes N1 Series - * - * References: - * FOR AWS: - * https://aws.amazon.com/ec2/instance-types/ - * https://aws.amazon.com/ec2/pricing/on-demand/ - * - * FOR GOOGLE: - * https://cloud.google.com/compute/gpus-pricing - * https://cloud.google.com/products/calculator/ - * - * FOR AZURE: - * https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-gpu - * https://azure.microsoft.com/en-ca/pricing/details/virtual-machines/windows/#pricing - * - */ -export const cloudInstances = [ - { - id: 0, - provider: "google", - ngpus: 1, - instance: "a2-highgpu-1g", - gpu: "a100", - cost: 2.93 + 0.74, - }, - { - id: 1, - provider: "google", - ngpus: 1, - instance: "n1-standard-1", - gpu: "t4", - cost: 0.35 + 0.05, - }, - { - id: 2, - provider: "google", - ngpus: 1, - instance: "n1-standard-1", - gpu: "p4", - cost: 0.6 + 0.05, - }, - { - id: 3, - provider: "google", - ngpus: 1, - instance: "n1-standard-1", - gpu: "v100", - cost: 2.48 + 0.05, - }, - { - id: 4, - provider: "google", - ngpus: 1, - instance: "n1-standard-1", - gpu: "p100", - cost: 1.46 + 0.05, - }, - { - id: 6, - provider: "aws", - ngpus: 1, - instance: "p3.2xlarge", - gpu: "v100", - cost: 3.06, - }, - { - id: 8, - provider: "aws", - ngpus: 1, - instance: "g4dn.xlarge", - gpu: "t4", - cost: 0.526, - }, - { - id: 10, - provider: "azure", - ngpus: 1, - instance: "NC6s v2", - gpu: "p100", - cost: 2.07, - }, - { - id: 11, - provider: "azure", - ngpus: 1, - instance: "NC6s v3", - gpu: "v100", - cost: 3.06, - }, - { - id: 12, - provider: "azure", - ngpus: 1, - instance: "NC4as T4 v3", - gpu: "t4", - cost: 0.526, - }, - - // { - // id: 13, - // provider: "centml", - // ngpus: 1, - // instance: "CentML", - // gpu: "rtx2080ti", - // cost: 0.2, - // }, - - // 2/4 GPU instances - - { - id: 14, - provider: "google", - ngpus: 2, - instance: "a2-highgpu-2g", - gpu: "a100", - cost: 5.87 + 1.48, - }, - { - id: 15, - provider: "google", - ngpus: 2, - instance: "n1-standard-1", - gpu: "t4", - cost: 2 * 0.35 + 0.05, - }, - { - id: 16, - provider: "google", - ngpus: 2, - instance: "n1-standard-1", - gpu: "p4", - cost: 2 * 0.6 + 0.05, - }, - { - id: 17, - provider: "google", - ngpus: 2, - instance: "n1-standard-1", - gpu: "v100", - cost: 2 * 2.48 + 0.05, - }, - { - id: 18, - provider: "google", - ngpus: 2, - instance: "n1-standard-1", - gpu: "p100", - cost: 2 * 1.46 + 0.05, - }, - { - id: 20, - provider: "aws", - ngpus: 4, - instance: "g4dn.12xlarge", - gpu: "t4", - cost: 3.912, - }, - - { - id: 21, - provider: "google", - ngpus: 4, - instance: "a2-highgpu-4g", - gpu: "a100", - cost: 11.74 + 2.96, - }, - { - id: 22, - provider: "google", - ngpus: 4, - instance: "n1-standard-1", - gpu: "t4", - cost: 4 * 0.35 + 0.05, - }, - { - id: 23, - provider: "google", - ngpus: 4, - instance: "n1-standard-1", - gpu: "p4", - cost: 4 * 0.6 + 0.05, - }, - { - id: 24, - provider: "google", - ngpus: 4, - instance: "n1-standard-1", - gpu: "v100", - cost: 4 * 2.48 + 0.05, - }, - { - id: 25, - provider: "google", - ngpus: 4, - instance: "n1-standard-1", - gpu: "p100", - cost: 4 * 1.46 + 0.05, - }, - { - id: 27, - provider: "aws", - ngpus: 4, - instance: "p3.8xlarge", - gpu: "v100", - cost: 12.24, - }, - { - id: 28, - provider: "azure", - ngpus: 2, - instance: "NC12s v2", - gpu: "p100", - cost: 4.14, - }, - { - id: 29, - provider: "azure", - ngpus: 4, - instance: "NC24rs v2", - gpu: "p100", - cost: 9.1080, - }, - { - id: 30, - provider: "azure", - ngpus: 2, - instance: "NC12s v3", - gpu: "v100", - cost: 6.12, - }, - { - id: 31, - provider: "azure", - ngpus: 4, - instance: "NC24rs v3", - gpu: "v100", - cost: 13.46, - }, - { - id: 32, - provider: "azure", - ngpus: 4, - instance: "NC64as T4 v3", - gpu: "t4", - cost: 4.352, - }, -]; - -export const cloudProviders = { - google: { - name: "Google Cloud Platform", - logo: "resources/google.png", - color: "#ea4335", - }, - azure: { - name: "Microsoft Azure", - logo: "resources/azure.png", - color: "#008AD7", - }, - aws: { - name: "Amazon Web Services", - logo: "resources/aws.png", - color: "#ff9900", - }, -} - -export const gpuPropertyList = [ - { name: "p100", vmem: 16, type: "server" }, - { name: "p4000", vmem: 8, type: "server" }, - { name: "rtx2070", vmem: 8, type: "consumer" }, - { name: "rtx2080ti", vmem: 11, type: "consumer" }, - { name: "t4", vmem: 16, type: "server" }, - { name: "v100", vmem: 16, type: "server" }, - { name: "a100", vmem: 40, type: "server" }, - { name: "rtx3090", vmem: 24, type: "consumer" }, - { name: "a40", vmem: 48, type: "server" }, - { name: "a4000", vmem: 16, type: "server" }, - { name: "rtx4000", vmem: 8, type: "consumer" } -]; diff --git a/skyline-vscode/react-ui/src/sections/DeploymentTab.js b/skyline-vscode/react-ui/src/sections/DeploymentTab.js index bd8eba7..f8aefd2 100644 --- a/skyline-vscode/react-ui/src/sections/DeploymentTab.js +++ b/skyline-vscode/react-ui/src/sections/DeploymentTab.js @@ -7,7 +7,7 @@ import Badge from "react-bootstrap/Badge"; import React from "react"; import { numberFormat } from "../utils/utils"; -const DeploymentTab = ({ numIterations, habitatData }) => { +const DeploymentTab = ({ numIterations, habitatData,additionalProviders }) => { return ( <> {habitatData.length === 0 ? ( @@ -32,6 +32,7 @@ const DeploymentTab = ({ numIterations, habitatData }) => { diff --git a/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js b/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js index f4e1c4b..c75d621 100644 --- a/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js +++ b/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js @@ -1,13 +1,14 @@ +import { enableFetchMocks } from "jest-fetch-mock"; +enableFetchMocks(); import { render, screen } from "@testing-library/react"; import DeploymentTab from "./DeploymentTab"; - const { ResizeObserver } = window; const numIterations = 10000; const data = [ ["source", 22.029312], ["P100", 14.069682], - ["P4000", 127.268085], // 27.268085 + ["P4000", 127.268085], ["RTX2070", 16.088268], ["RTX2080Ti", 11.826558], ["T4", 22.029312], @@ -19,13 +20,26 @@ const data = [ ["RTX4000", 20.2342], ]; +const yamlTestData = ` +--- + google: + name: "Google Cloud Platform" + logo: "resources/google.png" + color: "#ea4335" + instances: + - name: "a2-highgpu-1g" + gpu: "a100" + ngpus: 1 + cost: 3.67 +` + beforeEach(() => { delete window.ResizeObserver; window.ResizeObserver = jest.fn().mockImplementation(() => ({ observe: jest.fn(), unobserve: jest.fn(), disconnect: jest.fn(), - })); + })) }); afterEach(() => { @@ -59,17 +73,25 @@ test("Shows loading spinner when there is no habitat data", () => { expect(deploymentSection).toBeNull(); }); -test("Shows deployment target when there is habitat data", () => { +test("Shows deployment target when there is habitat data", async () => { + // ARRANGE - const { container } = render( - - ); + // Mock fetch response + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + text: jest.fn().mockResolvedValue(yamlTestData) + }) + + const { container } = render( + + ); // ASSERT - expect(screen.getByText(/deployment target/i)).toBeTruthy(); + expect(await screen.findByText(/deployment target/i)).toBeTruthy(); + const deploymentSection = screen.queryByText(/loading information/i); expect(deploymentSection).toBeNull(); expect( - container.querySelector(".recharts-responsive-container") // eslint-disable-line + container.querySelector(".recharts-responsive-container") // eslint-disable-line ).toBeTruthy(); }); diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.js index c1d0c9f..9754cea 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.js @@ -15,52 +15,21 @@ import Table from "react-bootstrap/Table"; import Form from "react-bootstrap/Form"; import { - cloudInstances, - gpuPropertyList, - cloudProviders, -} from "../data/providers"; + deploymentScatterGraphColorSize +} from "../data/properties"; import { calculate_training_time, numberFormat, currencyFormat, } from "../utils/utils"; -import {loadYamlFile} from '../utils/parsers'; +import { loadYamlFile } from "../utils/parsers"; -const highlightColor = "#9b59b6"; -const normalSize = 200; -const highlightSize = 280; - -const populate_initial_data = (habitatData) => { - let filtered_instances = []; - if (habitatData) { - for (const instance of cloudInstances) { - const found_in_habitat = habitatData.find( - (item) => item[0].toLowerCase() === instance.gpu.toLowerCase() - ); - const found_in_gpuPropertyList = gpuPropertyList.find( - (item) => - item.name.toLocaleLowerCase() === instance.gpu.toLocaleLowerCase() - ); - if (found_in_habitat && found_in_gpuPropertyList) { - filtered_instances.push({ - id: instance.id, - x: found_in_habitat[1], // msec - y: (instance.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration - info: instance, - vmem: found_in_gpuPropertyList.vmem, - fill: cloudProviders[instance.provider].color, - z: normalSize, - }); - } - } - } - return filtered_instances; -}; - -const ProviderPanel = ({ numIterations, habitatData }) => { +const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { const [providerPanelSettings, setProviderPanelSettings] = useState({ plotData: null, + initialData: null, + cloudProviders: null, nearest: null, clicked: null, maxNumGpu: 0, @@ -72,11 +41,11 @@ const ProviderPanel = ({ numIterations, habitatData }) => { const MAX_GPU = [1, 2, 4, 0]; // 0 is all - const initial_data = populate_initial_data(habitatData); + // const initial_data = populate_initial_data(habitatData); let gpuList = new Set(); - if (initial_data) { - initial_data.forEach((element) => { + if (providerPanelSettings.initialData) { + providerPanelSettings.initialData.forEach((element) => { gpuList.add(element.info.gpu.toLocaleLowerCase()); }); } @@ -97,7 +66,7 @@ const ProviderPanel = ({ numIterations, habitatData }) => { numGpuFilter = Number(value); } - const filtered_plot_data = initial_data + const filtered_plot_data = providerPanelSettings.initialData .filter((item) => { if (providerFilter !== "all") { return ( @@ -133,7 +102,7 @@ const ProviderPanel = ({ numIterations, habitatData }) => { const recalculateCost = (provider) => { if (provider == null) return; - const originalData = initial_data.find((item) => item.id === provider.id); + const originalData = providerPanelSettings.initialData.find((item) => item.id === provider.id); let totalHr = calculate_training_time(numIterations, originalData); // NEED YUBO FEEDBACK let totalCost = provider.info.cost * totalHr; @@ -153,14 +122,14 @@ const ProviderPanel = ({ numIterations, habitatData }) => { if (item.id === value.id) { return { ...item, - fill: highlightColor, - z: highlightSize, + fill: deploymentScatterGraphColorSize.HIGHLIGHTCOLOR, + z: deploymentScatterGraphColorSize.HIGHLIGHTSIZE, }; } return { ...item, - fill: cloudProviders[item.info.provider].color, - z: normalSize, + fill: providerPanelSettings.cloudProviders[item.info.provider].color, + z: deploymentScatterGraphColorSize.NORMALSIZE, }; }), })); @@ -172,183 +141,195 @@ const ProviderPanel = ({ numIterations, habitatData }) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [numIterations]); - useEffect(async()=>{ - const data = await loadYamlFile(); - console.log(data) - setProviderPanelSettings(prevState=>({ - ...prevState, - plotData: populate_initial_data(habitatData) - })) + useEffect(() => { + async function fetchData() { + const data = await loadYamlFile(habitatData, additionalProviders); + setProviderPanelSettings((prevState) => ({ + ...prevState, + plotData: data.instanceArray, + initialData: data.instanceArray, + cloudProviders: data.cloudProviders, + })); + } + fetchData(); // eslint-disable-next-line react-hooks/exhaustive-deps - },[]) - + }, []); return ( <> - {providerPanelSettings.plotData && cloudInstances && gpuPropertyList && cloudProviders ? -
- Providers - - - + {providerPanelSettings.plotData && providerPanelSettings.cloudProviders ? ( +
+ Providers + - - -
-
Filter by provider
- - handleFilterChange("provider", e.target.value) - } - > - - {Object.keys(cloudProviders).map((provider, index) => { - return ( - - ); - })} - -
- - -
-
Filter by GPU
- - handleFilterChange("gpu", e.target.value) - } - > - - {[...gpuList].map((gpu, index) => { - return ( - - ); - })} - -
- - -
-
Filter Max Number of GPUs:
- - {MAX_GPU.map((numgpu) => ( - + + + +
+
Filter by provider
+ + handleFilterChange("provider", e.target.value) } - name="radio" - size="sm" - value={numgpu} - checked={numgpu === providerPanelSettings.maxNumGpu} + > + + {Object.keys(providerPanelSettings.cloudProviders).map( + (provider, index) => { + return ( + + ); + } + )} + +
+ + +
+
Filter by GPU
+ - handleFilterChange( - "numGpus ", - e.currentTarget.value - ) + handleFilterChange("gpu", e.target.value) } > - {numgpu === 0 ? "all" : numgpu} - - ))} - -
- - - - - - - - -
- Deployment Plan - {providerPanelSettings.clicked && ( - - - - - - - All + {[...gpuList].map((gpu, index) => { + return ( + + ); + })} + +
+ + +
+
Filter Max Number of GPUs:
+ + {MAX_GPU.map((numgpu) => ( + - - -

- {providerPanelSettings.clicked.info.instance} -

- - {`Estimated Cost: ${currencyFormat( - providerPanelSettings.estimated_cost - )}`} - -

- - {`Estimated Training Time: ${numberFormat( - providerPanelSettings.estimated_time - )} Hours`} - -

- - - - - - - - - - - - - - - - - - -
GPUNum. GPUVRAM
{providerPanelSettings.clicked.info.gpu} - {providerPanelSettings.clicked.info.ngpus} - {providerPanelSettings.clicked.vmem} GB
- - - - )} - {providerPanelSettings.clicked == null && ( - - - - Select a configuration. - - - - )} -
- - - -
- :
Loading
} + name="radio" + size="sm" + value={numgpu} + checked={ + numgpu === providerPanelSettings.maxNumGpu + } + onChange={(e) => + handleFilterChange( + "numGpus ", + e.currentTarget.value + ) + } + > + {numgpu === 0 ? "all" : numgpu} + + ))} + +
+ + + + + + + + +
+ Deployment Plan + {providerPanelSettings.clicked && ( + + + + + + + + + +

+ {providerPanelSettings.clicked.info.instance} +

+ + {`Estimated Cost: ${currencyFormat( + providerPanelSettings.estimated_cost + )}`} + +

+ + {`Estimated Training Time: ${numberFormat( + providerPanelSettings.estimated_time + )} Hours`} + +

+ + + + + + + + + + + + + + + + + + +
GPUNum. GPUVRAM
+ {providerPanelSettings.clicked.info.gpu} + + {providerPanelSettings.clicked.info.ngpus} + {providerPanelSettings.clicked.vmem} GB
+ + + + )} + {providerPanelSettings.clicked == null && ( + + + + Select a configuration. + + + + )} +
+ + + +
+ ) : ( +
Loading
+ )} ); }; diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js index 042ae50..908c60e 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js @@ -1,3 +1,5 @@ +import { enableFetchMocks } from 'jest-fetch-mock' +enableFetchMocks() import { render, screen } from "@testing-library/react"; import ProviderPanel from "./ProviderPanel"; @@ -19,6 +21,19 @@ const data = [ ["RTX4000", 20.2342], ]; +const yamlTestData = ` +--- + google: + name: "Google Cloud Platform" + logo: "resources/google.png" + color: "#ea4335" + instances: + - name: "a2-highgpu-1g" + gpu: "a100" + ngpus: 1 + cost: 3.67 +` + beforeEach(() => { delete window.ResizeObserver; window.ResizeObserver = jest.fn().mockImplementation(() => ({ @@ -49,16 +64,22 @@ jest.mock("recharts", () => { }; }); -test("Shows a scatter chart", () => { +test("Shows a scatter chart", async() => { // ARRANGE + // Mock fetch response + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + text: jest.fn().mockResolvedValue(yamlTestData) + }) + const { container } = render( - + ); // ASSERT - expect(screen.getByText(/Providers/i)).toBeTruthy(); + expect(await screen.findByText(/Providers/i)).toBeTruthy(); expect( container.querySelector(".recharts-responsive-container") // eslint-disable-line ).toBeTruthy(); - expect(screen.getByText(/select a configuration/i)).toBeTruthy(); + expect(await screen.findByText(/select a configuration/i)).toBeTruthy(); }); diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index d8830ca..5ffa791 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -1,15 +1,88 @@ /** * Yaml parser */ - import { load } from "js-yaml"; +import { + gpuPropertyList, + deploymentScatterGraphColorSize, +} from "../data/properties"; + +let instanceID = 0; +let instanceArray = []; +let cloudProviders = {}; + +export const loadYamlFile = async (habitatData, additionalProviders) => { + const response = await fetch("data/providers.yaml"); + if (!response.ok) { + console.log("Could not read local file"); + } + const localDataText = await response.text(); + const localDataYaml = load(localDataText); + CloudProviderReader(localDataYaml, habitatData); + CloudProviderReader(additionalProviders, habitatData); + return { cloudProviders, instanceArray }; +}; -export const loadYamlFile = async() => { - let result = null; - await fetch('dummy.yaml') - .then((response) => response.text()) - .then((yamlString) => load(yamlString)) - .then((data) => result=data) - .catch((e) => console.log(e)); -return result; +const CloudProviderReader = (data, habitatData) => { + if (data && habitatData) { + for (const [key, value] of Object.entries(data)) { + const cloudProviderName = key.toLocaleLowerCase(); + if (!(cloudProviderName in cloudProviders)) { + cloudProviders[cloudProviderName] = { + name: value.name, + logo: value.logo, + color: value.color, + }; + } + for (const instanceData of value.instances) { + if ( + instanceData.ngpus && + Number.isInteger(instanceData.ngpus) && + instanceData.ngpus !== 0 && + instanceData.cost && + typeof instanceData.cost === "number" + ) { + const instanceAlreadyInArr = instanceArray.find( + (item) => + item.info.provider === cloudProviderName && + item.info.gpu === instanceData.gpu && + item.info.instance === instanceData.name && + item.info.ngpus === instanceData.ngpus + ); + if (instanceAlreadyInArr) { + instanceAlreadyInArr.info.cost = instanceData.cost; + instanceAlreadyInArr.y = + (instanceData.cost / 3.6e6) * instanceAlreadyInArr.x; + } else { + const found_in_habitat = habitatData.find( + (item) => item[0].toLowerCase() === instanceData.gpu.toLowerCase() + ); + const found_in_gpuPropertyList = gpuPropertyList.find( + (item) => + item.name.toLocaleLowerCase() === + instanceData.gpu.toLocaleLowerCase() + ); + if (found_in_habitat && found_in_gpuPropertyList) { + instanceArray.push({ + id: instanceID, + x: found_in_habitat[1], // msec + y: (instanceData.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration + info: { + instance: instanceData.name.toLocaleLowerCase(), + gpu: instanceData.gpu.toLocaleLowerCase(), + ngpus: instanceData.ngpus, + cost: instanceData.cost, + provider: cloudProviderName.toLocaleLowerCase(), + }, + vmem: found_in_gpuPropertyList.vmem, + fill: value.color, + z: deploymentScatterGraphColorSize.NORMALSIZE, + }); + instanceID += 1; + } + } + } + } + } + } }; diff --git a/skyline-vscode/react-ui/src/utils/parsers.test.js b/skyline-vscode/react-ui/src/utils/parsers.test.js new file mode 100644 index 0000000..cdbaa82 --- /dev/null +++ b/skyline-vscode/react-ui/src/utils/parsers.test.js @@ -0,0 +1,91 @@ +import { enableFetchMocks } from "jest-fetch-mock"; +enableFetchMocks(); + +const data = [ + ["source", 22.029312], + ["P100", 14.069682], + ["P4000", 127.268085], // 27.268085 + ["RTX2070", 16.088268], + ["RTX2080Ti", 11.826558], + ["T4", 22.029312], + ["V100", 10.182922], + ["A100", 10.068596], + ["RTX3090", 9.841998], + ["A40", 11.558072], + ["A4000", 14.67059], + ["RTX4000", 20.2342], +]; + +import { loadYamlFile } from "./parsers"; + +const yamlTestData = ` +--- + google: + name: "Google Cloud Platform" + logo: "resources/google.png" + color: "#ea4335" + instances: + - name: "a2-highgpu-1g" + gpu: "a100" + ngpus: 1 + cost: 3.67 +`; +const dataFromVSCodeExtension = { + aws: { + name: "Amazon Web Services", + logo: "resources/aws.png", + color: "#ff9900", + instances: [{ name: "p3.2xlarge", gpu: "v100", ngpus: 1, cost: 3.06 }], + }, +}; + +test("Parse yaml files and return list of cloud providers and instances", async () => { + jest.spyOn(global, "fetch").mockResolvedValue({ + ok: true, + text: jest.fn().mockResolvedValue(yamlTestData), + }); + const resp = await loadYamlFile(data, dataFromVSCodeExtension); + expect(resp.cloudProviders).toStrictEqual({ + google: { + name: "Google Cloud Platform", + logo: "resources/google.png", + color: "#ea4335", + }, + aws: { + name: "Amazon Web Services", + logo: "resources/aws.png", + color: "#ff9900", + }, + }); + expect(resp.instanceArray.length).toEqual(2); + expect(resp.instanceArray).toContainEqual({ + id: 0, + x: 10.068596, + y: 0.000010264374255555554, + info: { + instance: "a2-highgpu-1g", + gpu: "a100", + ngpus: 1, + cost: 3.67, + provider: "google", + }, + vmem: 40, + fill: "#ea4335", + z: 200, + }); + expect(resp.instanceArray).toContainEqual({ + id: 1, + x: 10.182922, + y: 0.0000086554837, + info: { + instance: "p3.2xlarge", + gpu: "v100", + ngpus: 1, + cost: 3.06, + provider: "aws", + }, + vmem: 16, + fill: "#ff9900", + z: 200, + }); +}); diff --git a/skyline-vscode/src/extension.ts b/skyline-vscode/src/extension.ts index f630d24..854006a 100644 --- a/skyline-vscode/src/extension.ts +++ b/skyline-vscode/src/extension.ts @@ -12,7 +12,6 @@ export function activate(context: vscode.ExtensionContext) { let disposable = vscode.commands.registerCommand('skyline-vscode.cmd_begin_analyze', () => { let vsconfig = vscode.workspace.getConfiguration('skyline'); - console.log("GAAAAAAAAAAAAAAAAAA",vsconfig); let options: vscode.OpenDialogOptions = { canSelectFiles: false, diff --git a/skyline-vscode/src/skyline_session.ts b/skyline-vscode/src/skyline_session.ts index 1ae925f..006981b 100644 --- a/skyline-vscode/src/skyline_session.ts +++ b/skyline-vscode/src/skyline_session.ts @@ -6,7 +6,7 @@ const fs = require('fs'); import {Socket} from 'net'; import { simpleDecoration } from './decorations'; -import { energy_component_type_mapping } from './utils'; +import { energy_component_type_mapping,yaml_loader } from './utils'; const crypto = require('crypto'); const resolve = require('path').resolve; @@ -57,7 +57,7 @@ export class SkylineSession { reactProjectRoot: string; constructor(options: SkylineSessionOptions, environ: SkylineEnvironment) { - console.log("SkylineSession instantiated !!!!"); + console.log("SkylineSession instantiated"); this.resetBackendConnection = false; this.connection = new Socket(); @@ -67,7 +67,6 @@ export class SkylineSession { this.port = options.port; this.addr = options.addr; this.providers = options.providers; - console.log("port",this.port,"address", this.addr,"file",this.providers); this.seq_num = 0; this.last_length = -1; @@ -84,6 +83,7 @@ export class SkylineSession { this.webviewPanel.onDidDispose(this.disconnect.bind(this)); this.webviewPanel.webview.html = this._getHtmlForWebview(); this.connect(); + this.load_extension_yaml_file(); vscode.workspace.onDidChangeTextDocument(this.on_text_change.bind(this)); this.restart_profiling = this.restart_profiling.bind(this); @@ -110,8 +110,7 @@ export class SkylineSession { on_connect() { let connectionMessage = { "message_type": "connection", - "status": true, - "file": this.providers + "status": true }; this.webviewPanel.webview.postMessage(connectionMessage); } @@ -165,6 +164,16 @@ export class SkylineSession { this.webviewPanel.webview.postMessage(errorEvent); } + load_extension_yaml_file() { + const extensionFile = yaml_loader(this.providers); + let loadedFile = { + "message_type": "loaded_additional_providers", + "additionalProviders": extensionFile, + }; + console.log("sending loaded file"); + this.webviewPanel.webview.postMessage(loadedFile); + } + webview_handle_message(msg: any) { console.log("webview_handle_message"); console.log(msg); diff --git a/skyline-vscode/src/utils.ts b/skyline-vscode/src/utils.ts index 9685250..f850756 100644 --- a/skyline-vscode/src/utils.ts +++ b/skyline-vscode/src/utils.ts @@ -1,3 +1,6 @@ +const yaml = require('js-yaml'); +const fs = require('fs'); + import {EnergyConsumptionComponentType} from './protobuf/innpv_pb'; export const energy_component_type_mapping = (code: number):string => { let result:string="INVALID"; @@ -13,4 +16,16 @@ export const energy_component_type_mapping = (code: number):string => { break; } return result; +} + +export const yaml_loader = (pathToFile:String):Object | null => { + let yamlFile:Object | null = null; + try { + const doc = yaml.load(fs.readFileSync(pathToFile, 'utf8')); + yamlFile = doc; + } catch (e) { + console.log("Could not read the file",e); + } + + return yamlFile; } \ No newline at end of file From 0e103e4f71f9197fe056fb1f04af6471ebf5fdf1 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 16 Mar 2023 12:15:23 -0400 Subject: [PATCH 03/16] added general yaml schema for external cloud providers | changes unit testing | changed parsers.js --- README.md | 3 +- schemas/CloudProvidersSchema.yaml | 52 ++++++++ skyline-vscode/react-ui/src/App.js | 1 - .../src/sections/DeploymentTab.test.js | 4 +- .../src/sections/ProviderPanel.test.js | 4 +- skyline-vscode/react-ui/src/utils/parsers.js | 116 ++++++++++-------- .../react-ui/src/utils/parsers.test.js | 48 +++++++- 7 files changed, 167 insertions(+), 61 deletions(-) create mode 100644 schemas/CloudProvidersSchema.yaml diff --git a/README.md b/README.md index 3f28727..7dce26c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ After installation, please make note of the path to the `skyline` binary. To do ## Usage example 1. Open one of the examples DNN project examples, i.e. [Resnet](https://github.com/CentML/skyline/tree/main/examples/resnet) from Skyline in VSCode -2. Press `Ctrl+Shift+P`, then select `Skyline` from the dropdown list. +2. (Optional) You can add other external cloud instances. Use the CloudProvidersSchema.yaml file as a reference. This file is located under the schemas folder. Once you have created your own yaml file, you can reference the path to this file in the extension settings in **providers** option. The file will be read when the extension is called. +3. Press `Ctrl+Shift+P`, then select `Skyline` from the dropdown list. 3. Click on `Begin Analysis`. ## Development Environment Setup diff --git a/schemas/CloudProvidersSchema.yaml b/schemas/CloudProvidersSchema.yaml new file mode 100644 index 0000000..874a87e --- /dev/null +++ b/schemas/CloudProvidersSchema.yaml @@ -0,0 +1,52 @@ +title: + Schema definition for cloud providers +description: | + This schema is intended to be used as a template to add more cloud providers when the VSCode extension is executed. +properties: + provider: + description: | + Object describing the external cloud provider + type: object + properties: + name: + description: Complete name of cloud provider + type: string + logo: + description: Reference to image of cloud provider + type: string + color: + description: Color preference for cloud provider + type: string + instances: + type: array + item: + type: object + properties: + name: + description: Name of instance + type: string + gpu: + description: | + GPU model. Currently supported models: [P100, P4000, RTX2070, RTX2080Ti, T4, V100, A100, RTX3090, A40, A4000, RTX4000] + type: string + ngpus: + description: Number of GPUs + type: integer + cost: + description: Cost per hour + type: float + +#Example +#google: +# name: "Google Cloud Platform" +# logo: "https://blog.hubspot.com/hubfs/image8-2.jpg" +# color: "#ea4335" +# instances: +# - name: "a2-highgpu-1g" +# gpu: "a100" +# ngpus: 1 +# cost: 3.67 +# - name: "n1-standard-1" +# gpu: "t4" +# ngpus: 1 +# cost: 0.4 \ No newline at end of file diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 9c9b9aa..7836547 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -106,7 +106,6 @@ function App() { } else if (event.data["message_type"] === "analysis") { processAnalysisState(event.data); } else if (event.data["message_type"] === "loaded_additional_providers") { - console.log("received loaded file",event.data["additionalProviders"]); setExternalData(event.data["additionalProviders"]); } else if (event.data["message_type"] === "text_change") { setTextChanged(true); diff --git a/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js b/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js index c75d621..4633d49 100644 --- a/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js +++ b/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js @@ -1,7 +1,7 @@ -import { enableFetchMocks } from "jest-fetch-mock"; -enableFetchMocks(); import { render, screen } from "@testing-library/react"; import DeploymentTab from "./DeploymentTab"; +import { enableFetchMocks } from "jest-fetch-mock"; +enableFetchMocks(); const { ResizeObserver } = window; const numIterations = 10000; diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js index 908c60e..91e2c6b 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js @@ -1,7 +1,7 @@ -import { enableFetchMocks } from 'jest-fetch-mock' -enableFetchMocks() import { render, screen } from "@testing-library/react"; import ProviderPanel from "./ProviderPanel"; +import { enableFetchMocks } from 'jest-fetch-mock' +enableFetchMocks() const { ResizeObserver } = window; const numIterations = 10000; diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 5ffa791..2a29af6 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -7,78 +7,90 @@ import { deploymentScatterGraphColorSize, } from "../data/properties"; -let instanceID = 0; -let instanceArray = []; -let cloudProviders = {}; +class ResponseBuffer{ + constructor(){ + this.instanceId = 0; + this.instanceArray = []; + this.cloudProviders = {}; + } +} export const loadYamlFile = async (habitatData, additionalProviders) => { + const buffer = new ResponseBuffer(); + const response = await fetch("data/providers.yaml"); if (!response.ok) { console.log("Could not read local file"); } const localDataText = await response.text(); const localDataYaml = load(localDataText); - CloudProviderReader(localDataYaml, habitatData); - CloudProviderReader(additionalProviders, habitatData); - return { cloudProviders, instanceArray }; + CloudProviderReader(localDataYaml, habitatData,buffer); + CloudProviderReader(additionalProviders, habitatData,buffer); + return { cloudProviders: buffer.cloudProviders, instanceArray: buffer.instanceArray }; }; -const CloudProviderReader = (data, habitatData) => { +const CloudProviderReader = (data, habitatData,storageBuffer) => { if (data && habitatData) { for (const [key, value] of Object.entries(data)) { + if (!value) { + continue; + } const cloudProviderName = key.toLocaleLowerCase(); - if (!(cloudProviderName in cloudProviders)) { - cloudProviders[cloudProviderName] = { + if (!(cloudProviderName in storageBuffer.cloudProviders)) { + storageBuffer.cloudProviders[cloudProviderName] = { name: value.name, logo: value.logo, color: value.color, }; } - for (const instanceData of value.instances) { - if ( - instanceData.ngpus && - Number.isInteger(instanceData.ngpus) && - instanceData.ngpus !== 0 && - instanceData.cost && - typeof instanceData.cost === "number" - ) { - const instanceAlreadyInArr = instanceArray.find( - (item) => - item.info.provider === cloudProviderName && - item.info.gpu === instanceData.gpu && - item.info.instance === instanceData.name && - item.info.ngpus === instanceData.ngpus - ); - if (instanceAlreadyInArr) { - instanceAlreadyInArr.info.cost = instanceData.cost; - instanceAlreadyInArr.y = - (instanceData.cost / 3.6e6) * instanceAlreadyInArr.x; - } else { - const found_in_habitat = habitatData.find( - (item) => item[0].toLowerCase() === instanceData.gpu.toLowerCase() - ); - const found_in_gpuPropertyList = gpuPropertyList.find( + if (value.instances) { + for (const instanceData of value.instances) { + if ( + instanceData.ngpus && + Number.isInteger(instanceData.ngpus) && + instanceData.ngpus !== 0 && + instanceData.cost && + typeof instanceData.cost === "number" + ) { + const instanceAlreadyInArr = storageBuffer.instanceArray.find( (item) => - item.name.toLocaleLowerCase() === - instanceData.gpu.toLocaleLowerCase() + item.info.provider === cloudProviderName && + item.info.gpu === instanceData.gpu && + item.info.instance === instanceData.name && + item.info.ngpus === instanceData.ngpus ); - if (found_in_habitat && found_in_gpuPropertyList) { - instanceArray.push({ - id: instanceID, - x: found_in_habitat[1], // msec - y: (instanceData.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration - info: { - instance: instanceData.name.toLocaleLowerCase(), - gpu: instanceData.gpu.toLocaleLowerCase(), - ngpus: instanceData.ngpus, - cost: instanceData.cost, - provider: cloudProviderName.toLocaleLowerCase(), - }, - vmem: found_in_gpuPropertyList.vmem, - fill: value.color, - z: deploymentScatterGraphColorSize.NORMALSIZE, - }); - instanceID += 1; + if (instanceAlreadyInArr) { + instanceAlreadyInArr.info.cost = instanceData.cost; + instanceAlreadyInArr.y = + (instanceData.cost / 3.6e6) * instanceAlreadyInArr.x; + } else { + const found_in_habitat = habitatData.find( + (item) => + item[0].toLowerCase() === instanceData.gpu?.toLowerCase() + ); + const found_in_gpuPropertyList = gpuPropertyList.find( + (item) => + item.name.toLocaleLowerCase() === + instanceData.gpu?.toLocaleLowerCase() + ); + if (found_in_habitat && found_in_gpuPropertyList) { + storageBuffer.instanceArray.push({ + id: storageBuffer.instanceId, + x: found_in_habitat[1], // msec + y: (instanceData.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration + info: { + instance: instanceData.name?.toLocaleLowerCase(), + gpu: instanceData.gpu?.toLocaleLowerCase(), + ngpus: instanceData.ngpus, + cost: instanceData.cost, + provider: cloudProviderName.toLocaleLowerCase(), + }, + vmem: found_in_gpuPropertyList.vmem, + fill: value.color, + z: deploymentScatterGraphColorSize.NORMALSIZE, + }); + storageBuffer.instanceId += 1; + } } } } diff --git a/skyline-vscode/react-ui/src/utils/parsers.test.js b/skyline-vscode/react-ui/src/utils/parsers.test.js index cdbaa82..83eb13f 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.test.js +++ b/skyline-vscode/react-ui/src/utils/parsers.test.js @@ -1,7 +1,8 @@ +import { loadYamlFile } from "./parsers"; import { enableFetchMocks } from "jest-fetch-mock"; enableFetchMocks(); -const data = [ +const habitatData = [ ["source", 22.029312], ["P100", 14.069682], ["P4000", 127.268085], // 27.268085 @@ -16,7 +17,6 @@ const data = [ ["RTX4000", 20.2342], ]; -import { loadYamlFile } from "./parsers"; const yamlTestData = ` --- @@ -38,13 +38,19 @@ const dataFromVSCodeExtension = { instances: [{ name: "p3.2xlarge", gpu: "v100", ngpus: 1, cost: 3.06 }], }, }; +const incorrectDataFromVSCodeExtension = { + coreweave: { + name: "coreweave", + instances: [{ name: "p3.2xlarge", gpu: "v100", ngpus: 2.3, cost: '3.06' }], + }, +} test("Parse yaml files and return list of cloud providers and instances", async () => { jest.spyOn(global, "fetch").mockResolvedValue({ ok: true, text: jest.fn().mockResolvedValue(yamlTestData), }); - const resp = await loadYamlFile(data, dataFromVSCodeExtension); + const resp = await loadYamlFile(habitatData, dataFromVSCodeExtension); expect(resp.cloudProviders).toStrictEqual({ google: { name: "Google Cloud Platform", @@ -89,3 +95,39 @@ test("Parse yaml files and return list of cloud providers and instances", async z: 200, }); }); + +test("External yaml file is not in correct format", async()=>{ + jest.spyOn(global, "fetch").mockResolvedValue({ + ok: true, + text: jest.fn().mockResolvedValue(yamlTestData), + }); + const resp = await loadYamlFile(habitatData, incorrectDataFromVSCodeExtension); + expect(resp.cloudProviders).toStrictEqual({ + google: { + name: "Google Cloud Platform", + logo: "resources/google.png", + color: "#ea4335", + }, + coreweave: { + name: "coreweave", + logo: undefined, + color: undefined, + }, + }); + expect(resp.instanceArray.length).toEqual(1); + expect(resp.instanceArray).toContainEqual({ + id: 0, + x: 10.068596, + y: 0.000010264374255555554, + info: { + instance: "a2-highgpu-1g", + gpu: "a100", + ngpus: 1, + cost: 3.67, + provider: "google", + }, + vmem: 40, + fill: "#ea4335", + z: 200, + }); +}) \ No newline at end of file From c2a7157635315d2f34f95fecab910593be313acd Mon Sep 17 00:00:00 2001 From: John Calderon Date: Mon, 20 Mar 2023 15:45:35 -0400 Subject: [PATCH 04/16] changed URL providers to S3 --- skyline-vscode/package.json | 1 - skyline-vscode/react-ui/src/App.js | 2 +- .../react-ui/src/sections/ProviderPanel.js | 13 ++++++++++++- skyline-vscode/react-ui/src/utils/parsers.js | 6 ++++-- skyline-vscode/src/skyline_session.ts | 16 +++++++--------- skyline-vscode/src/utils.ts | 15 --------------- 6 files changed, 24 insertions(+), 29 deletions(-) diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index a65567c..789f410 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -67,7 +67,6 @@ }, "dependencies": { "google-protobuf": "^3.18.0", - "js-yaml": "^4.1.0", "ts-protoc-gen": "^0.15.0" } } diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 7836547..043c778 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -53,7 +53,7 @@ function restartProfiling() { }); } -const sendMock = false; +const sendMock = true; function App() { const [analysisState, setAnalysisState] = useState(); diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.js index 9754cea..8dd2cb8 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.js @@ -13,6 +13,7 @@ import Col from "react-bootstrap/Col"; import Image from "react-bootstrap/Image"; import Table from "react-bootstrap/Table"; import Form from "react-bootstrap/Form"; +import Spinner from "react-bootstrap/Spinner"; import { deploymentScatterGraphColorSize @@ -328,7 +329,17 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { ) : ( -
Loading
+
+ + + + + Loading instances + + + + +
)} ); diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 2a29af6..34433d6 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -17,10 +17,12 @@ class ResponseBuffer{ export const loadYamlFile = async (habitatData, additionalProviders) => { const buffer = new ResponseBuffer(); - - const response = await fetch("data/providers.yaml"); + //data/providers.yaml + const response = await fetch("https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/providers.yaml", + { cache: "no-store" }); if (!response.ok) { console.log("Could not read local file"); + return { cloudProviders: null, instanceArray: null }; } const localDataText = await response.text(); const localDataYaml = load(localDataText); diff --git a/skyline-vscode/src/skyline_session.ts b/skyline-vscode/src/skyline_session.ts index 006981b..2c6f310 100644 --- a/skyline-vscode/src/skyline_session.ts +++ b/skyline-vscode/src/skyline_session.ts @@ -1,4 +1,3 @@ - import * as vscode from 'vscode'; import * as pb from './protobuf/innpv_pb'; import * as path from 'path'; @@ -6,7 +5,7 @@ const fs = require('fs'); import {Socket} from 'net'; import { simpleDecoration } from './decorations'; -import { energy_component_type_mapping,yaml_loader } from './utils'; +import { energy_component_type_mapping } from './utils'; const crypto = require('crypto'); const resolve = require('path').resolve; @@ -83,7 +82,7 @@ export class SkylineSession { this.webviewPanel.onDidDispose(this.disconnect.bind(this)); this.webviewPanel.webview.html = this._getHtmlForWebview(); this.connect(); - this.load_extension_yaml_file(); + this.load_external_providers(); vscode.workspace.onDidChangeTextDocument(this.on_text_change.bind(this)); this.restart_profiling = this.restart_profiling.bind(this); @@ -164,14 +163,13 @@ export class SkylineSession { this.webviewPanel.webview.postMessage(errorEvent); } - load_extension_yaml_file() { - const extensionFile = yaml_loader(this.providers); - let loadedFile = { + load_external_providers() { + let otherUrls = { "message_type": "loaded_additional_providers", - "additionalProviders": extensionFile, + "additionalProviders": this.providers, }; - console.log("sending loaded file"); - this.webviewPanel.webview.postMessage(loadedFile); + console.log("sending list of urls"); + this.webviewPanel.webview.postMessage(otherUrls); } webview_handle_message(msg: any) { diff --git a/skyline-vscode/src/utils.ts b/skyline-vscode/src/utils.ts index f850756..d5bedbc 100644 --- a/skyline-vscode/src/utils.ts +++ b/skyline-vscode/src/utils.ts @@ -1,6 +1,3 @@ -const yaml = require('js-yaml'); -const fs = require('fs'); - import {EnergyConsumptionComponentType} from './protobuf/innpv_pb'; export const energy_component_type_mapping = (code: number):string => { let result:string="INVALID"; @@ -17,15 +14,3 @@ export const energy_component_type_mapping = (code: number):string => { } return result; } - -export const yaml_loader = (pathToFile:String):Object | null => { - let yamlFile:Object | null = null; - try { - const doc = yaml.load(fs.readFileSync(pathToFile, 'utf8')); - yamlFile = doc; - } catch (e) { - console.log("Could not read the file",e); - } - - return yamlFile; -} \ No newline at end of file From 4d2c3c9d43761e9b169e39b9cc0355538273aab4 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Tue, 21 Mar 2023 14:32:46 -0400 Subject: [PATCH 05/16] get data from multiple urls --- skyline-vscode/react-ui/src/App.js | 7 ++- skyline-vscode/react-ui/src/utils/parsers.js | 54 +++++++++++++++----- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 043c778..8164248 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -53,7 +53,7 @@ function restartProfiling() { }); } -const sendMock = true; +const sendMock = false; function App() { const [analysisState, setAnalysisState] = useState(); @@ -64,7 +64,10 @@ function App() { const [errorText, setErrorText] = useState(); const [connectionStatus, setConnectionStatus] = useState(false); const [numIterations, setNumIterations] = useState(100000); - const [externalData, setExternalData] = useState(null); + const [externalData, setExternalData] = useState([ + "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/dummy1.yaml", + "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/dummy2.yaml" + ]); App.vscodeApi = vscodeApi; diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 34433d6..3425123 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -7,8 +7,8 @@ import { deploymentScatterGraphColorSize, } from "../data/properties"; -class ResponseBuffer{ - constructor(){ +class ResponseBuffer { + constructor() { this.instanceId = 0; this.instanceArray = []; this.cloudProviders = {}; @@ -18,20 +18,48 @@ class ResponseBuffer{ export const loadYamlFile = async (habitatData, additionalProviders) => { const buffer = new ResponseBuffer(); //data/providers.yaml - const response = await fetch("https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/providers.yaml", - { cache: "no-store" }); - if (!response.ok) { - console.log("Could not read local file"); - return { cloudProviders: null, instanceArray: null }; + let urlList = [ + "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/providers.yaml", + ]; + urlList = urlList.concat(additionalProviders); + const listOfPromises = urlList.map((url)=>fetch(url,{ cache: "no-store" })) + try { + const responses = await Promise.all(listOfPromises); + for(let resp of responses){ + if(resp.ok){ + const localDataText = await resp.text(); + const localDataYaml = load(localDataText); + CloudProviderReader(localDataYaml, habitatData, buffer); + }else{ + console.log("error reading from url: ", resp.url); + } + } + return { + cloudProviders: buffer.cloudProviders, + instanceArray: buffer.instanceArray, + }; + } catch(error){ + console.log("There was an error", error); } - const localDataText = await response.text(); - const localDataYaml = load(localDataText); - CloudProviderReader(localDataYaml, habitatData,buffer); - CloudProviderReader(additionalProviders, habitatData,buffer); - return { cloudProviders: buffer.cloudProviders, instanceArray: buffer.instanceArray }; + + + + // const response = await fetch( + // "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/providers.yaml", + // { cache: "no-store" } + // ); + // if (!response.ok) { + // console.log("Could not read local file"); + // return { cloudProviders: null, instanceArray: null }; + // } + // const localDataText = await response.text(); + // const localDataYaml = load(localDataText); + // CloudProviderReader(localDataYaml, habitatData, buffer); + // CloudProviderReader(additionalProviders, habitatData, buffer); + }; -const CloudProviderReader = (data, habitatData,storageBuffer) => { +const CloudProviderReader = (data, habitatData, storageBuffer) => { if (data && habitatData) { for (const [key, value] of Object.entries(data)) { if (!value) { From e62f7763fab2cbfecd59669c45182de8df985fb2 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Tue, 21 Mar 2023 14:33:49 -0400 Subject: [PATCH 06/16] changed extension setting description --- skyline-vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index 789f410..c692a24 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -38,7 +38,7 @@ "skyline.providers": { "type": "string", "default": "", - "description": "location of additional cloud providers" + "description": "additional urls separated by commas" } } } From 37dc1d820f8017cb1b9cb7bb1ba10bd4ae0c6a62 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Wed, 22 Mar 2023 13:18:41 -0400 Subject: [PATCH 07/16] changed to centML S3 --- skyline-vscode/react-ui/src/App.js | 6 +++--- skyline-vscode/react-ui/src/utils/parsers.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 8164248..23a0ce4 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -53,7 +53,7 @@ function restartProfiling() { }); } -const sendMock = false; +const sendMock = true; function App() { const [analysisState, setAnalysisState] = useState(); @@ -65,8 +65,8 @@ function App() { const [connectionStatus, setConnectionStatus] = useState(false); const [numIterations, setNumIterations] = useState(100000); const [externalData, setExternalData] = useState([ - "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/dummy1.yaml", - "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/dummy2.yaml" + "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/dummy1.yaml", + "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/dummy2.yaml" ]); App.vscodeApi = vscodeApi; diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 3425123..73ea777 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -19,7 +19,7 @@ export const loadYamlFile = async (habitatData, additionalProviders) => { const buffer = new ResponseBuffer(); //data/providers.yaml let urlList = [ - "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/providers.yaml", + "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.yaml", ]; urlList = urlList.concat(additionalProviders); const listOfPromises = urlList.map((url)=>fetch(url,{ cache: "no-store" })) From 5d990c2bc2b513e1eb065df02266dc506b913ff9 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 23 Mar 2023 12:06:40 -0400 Subject: [PATCH 08/16] local tests --- skyline-vscode/react-ui/src/App.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 23a0ce4..c1ea717 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -64,10 +64,7 @@ function App() { const [errorText, setErrorText] = useState(); const [connectionStatus, setConnectionStatus] = useState(false); const [numIterations, setNumIterations] = useState(100000); - const [externalData, setExternalData] = useState([ - "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/dummy1.yaml", - "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/dummy2.yaml" - ]); + const [externalData, setExternalData] = useState([]); App.vscodeApi = vscodeApi; @@ -109,7 +106,8 @@ function App() { } else if (event.data["message_type"] === "analysis") { processAnalysisState(event.data); } else if (event.data["message_type"] === "loaded_additional_providers") { - setExternalData(event.data["additionalProviders"]); + console.log(event.data["additionalProviders"].split(",")) + setExternalData(event.data["additionalProviders"].split(",")); } else if (event.data["message_type"] === "text_change") { setTextChanged(true); } else if (event.data["message_type"] === "error") { From d59ab65b065847ff9bf4a6937a4edaa9d50667dd Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 23 Mar 2023 12:07:26 -0400 Subject: [PATCH 09/16] local changes to skylinesession --- skyline-vscode/package.json | 6 +++--- skyline-vscode/src/extension.ts | 2 +- skyline-vscode/src/skyline_session.ts | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index d034bb7..091036e 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -24,17 +24,17 @@ "properties": { "skyline.address": { "type": "string", - "default": "localhost", + "default": "localhost-2", "description": "Specifies the address of the profiler." }, "skyline.port": { "type": "number", - "default": 60120, + "default": 60150, "description": "Specifies the port of the profiler." }, "skyline.providers": { "type": "string", - "default": "", + "default": "another", "description": "additional urls separated by commas" }, "skyline.isTelemetryEnabled": { diff --git a/skyline-vscode/src/extension.ts b/skyline-vscode/src/extension.ts index c5d440c..be5ed1f 100644 --- a/skyline-vscode/src/extension.ts +++ b/skyline-vscode/src/extension.ts @@ -51,7 +51,7 @@ export function activate(context: vscode.ExtensionContext) { retainContextWhenHidden: true } ); - + console.log("EXTENSION.TS FILE",vsconfig); let sess_options: SkylineSessionOptions = { context: context, projectRoot: uri[0].fsPath, diff --git a/skyline-vscode/src/skyline_session.ts b/skyline-vscode/src/skyline_session.ts index a1ecba3..b6214e2 100644 --- a/skyline-vscode/src/skyline_session.ts +++ b/skyline-vscode/src/skyline_session.ts @@ -173,6 +173,8 @@ export class SkylineSession { } load_external_providers() { + console.log("PROVIDERS: ",this.providers); + console.log("PORT: ",this.port); let otherUrls = { "message_type": "loaded_additional_providers", "additionalProviders": this.providers, From 906de838124f06fb0f49935406eee376eaf9945b Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 23 Mar 2023 16:03:07 -0400 Subject: [PATCH 10/16] installation of json parser --- skyline-vscode/react-ui/package-lock.json | 102 +++++++++++++++++-- skyline-vscode/react-ui/package.json | 1 + skyline-vscode/react-ui/src/utils/parsers.js | 3 + 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/skyline-vscode/react-ui/package-lock.json b/skyline-vscode/react-ui/package-lock.json index e9987f2..736f6fc 100644 --- a/skyline-vscode/react-ui/package-lock.json +++ b/skyline-vscode/react-ui/package-lock.json @@ -1534,6 +1534,17 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1563,6 +1574,11 @@ "argparse": "^2.0.1" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -3614,13 +3630,13 @@ } }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "requires": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" } }, @@ -3867,6 +3883,11 @@ "schema-utils": "^2.6.5" }, "dependencies": { + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -3883,6 +3904,19 @@ "@types/json-schema": "^7.0.5", "ajv": "^6.12.4", "ajv-keywords": "^3.5.2" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + } } }, "semver": { @@ -5473,6 +5507,17 @@ "text-table": "^0.2.0" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -5511,6 +5556,11 @@ "argparse": "^2.0.1" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -6226,6 +6276,11 @@ "universalify": "^2.0.0" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "schema-utils": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz", @@ -6234,6 +6289,19 @@ "@types/json-schema": "^7.0.4", "ajv": "^6.12.2", "ajv-keywords": "^3.4.1" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + } } }, "semver": { @@ -8957,9 +9025,9 @@ "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -11345,6 +11413,24 @@ "@types/json-schema": "^7.0.8", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + } } }, "select-hose": { diff --git a/skyline-vscode/react-ui/package.json b/skyline-vscode/react-ui/package.json index 6ef3bcf..a7ad86a 100644 --- a/skyline-vscode/react-ui/package.json +++ b/skyline-vscode/react-ui/package.json @@ -7,6 +7,7 @@ "@fortawesome/fontawesome-svg-core": "^1.2.36", "@fortawesome/free-solid-svg-icons": "^5.15.4", "@fortawesome/react-fontawesome": "^0.1.17", + "ajv": "^8.12.0", "bootstrap": "^5.1.3", "buffer": "^6.0.3", "js-yaml": "^4.1.0", diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 73ea777..1c2a0fc 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -2,11 +2,14 @@ * Yaml parser */ import { load } from "js-yaml"; +import Ajv from "ajv/dist/jtd"; import { gpuPropertyList, deploymentScatterGraphColorSize, } from "../data/properties"; +const ajv = new Ajv() // options can be passed, e.g. {allErrors: true} + class ResponseBuffer { constructor() { this.instanceId = 0; From f85468dac8ba39c0bd7a8e284f563ce033d83e41 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Fri, 24 Mar 2023 14:29:02 -0400 Subject: [PATCH 11/16] changed to json schema --- .../src/schema/CloudProvidersSchema.js | 46 ++++++++++ .../react-ui/src/sections/ProviderPanel.js | 3 +- skyline-vscode/react-ui/src/utils/parsers.js | 83 ++++++++++++++----- 3 files changed, 108 insertions(+), 24 deletions(-) create mode 100644 skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js diff --git a/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js b/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js new file mode 100644 index 0000000..a3045e8 --- /dev/null +++ b/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js @@ -0,0 +1,46 @@ +export const cloudProviderSchema = { + title:"SCHEMA DEFINITION FOR CLOUD PROVIDERS", + description:"Array of cloud providers, (there needs to be at least 1 cloud provider)", + type: "array", + items: { + type: "object", + description:"each cloud provider needs to include it's name, logo, color, and instace specifications", + required:["name","logo","color","instances"], + properties: { + name: { type: "string", description: "name of cloud provider. eg: gcp,aws,azure,coreweaver" }, + logo: { type: "string", description: "url reference to the logo" }, + color: { type: "string", description: "color (CSS format)" }, + instances: { + type: "array", + description: "list of instances (there needs to be at least 1 instance)", + items: { + type: "object", + required:["name","gpu","ngpus","cost"], + properties: { + name: { type: "string", description:"name of the instance" }, + gpu: { + enum: [ + "p100", + "p4000", + "rtx2070", + "rtx2080ti", + "t4", + "v100", + "a100", + "rtx3090", + "a40", + "a4000", + "rtx4000", + ], + description:"only GPUs supported by our product DeepView.Predict" + }, + ngpus: { type: "integer", minimum: 1, description:"number of gpus in the instance" }, + cost: { type: "number", exclusiveMinimum: 0, description: "cost per hour" }, + }, + }, + minItems: 1, + }, + }, + }, + minItems: 1, +}; diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.js index 022db42..26ace4b 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.js @@ -25,7 +25,7 @@ import { currencyFormat, } from "../utils/utils"; -import { loadYamlFile } from "../utils/parsers"; +import { loadYamlFile,loadJsonFiles } from "../utils/parsers"; const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { const [providerPanelSettings, setProviderPanelSettings] = useState({ @@ -147,6 +147,7 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { useEffect(() => { async function fetchData() { + await loadJsonFiles(habitatData, additionalProviders); const data = await loadYamlFile(habitatData, additionalProviders); setProviderPanelSettings((prevState) => ({ ...prevState, diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 1c2a0fc..d872954 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -2,13 +2,30 @@ * Yaml parser */ import { load } from "js-yaml"; -import Ajv from "ajv/dist/jtd"; +import Ajv from "ajv"; import { gpuPropertyList, deploymentScatterGraphColorSize, } from "../data/properties"; +import { cloudProviderSchema } from "../schema/CloudProvidersSchema"; -const ajv = new Ajv() // options can be passed, e.g. {allErrors: true} +const ajv = new Ajv({ allErrors: true }); // to report all validation errors (rather than failing on the first errors) +const validate = ajv.compile(cloudProviderSchema); + +const data = [ + { + name: "gcp", + logo: "", + color: "#ea4335", + instances: [{ name: "a2-highgpu-1g", gpu: "a100", ngpus: 1, cost: 0.05 }], + }, +]; + +const valid = validate(data); +if (!valid) { + console.log("Error"); + validate.errors.forEach((err) => console.log(err.message)); +} class ResponseBuffer { constructor() { @@ -25,15 +42,17 @@ export const loadYamlFile = async (habitatData, additionalProviders) => { "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.yaml", ]; urlList = urlList.concat(additionalProviders); - const listOfPromises = urlList.map((url)=>fetch(url,{ cache: "no-store" })) + const listOfPromises = urlList.map((url) => + fetch(url, { cache: "no-store" }) + ); try { const responses = await Promise.all(listOfPromises); - for(let resp of responses){ - if(resp.ok){ + for (let resp of responses) { + if (resp.ok) { const localDataText = await resp.text(); const localDataYaml = load(localDataText); CloudProviderReader(localDataYaml, habitatData, buffer); - }else{ + } else { console.log("error reading from url: ", resp.url); } } @@ -41,25 +60,9 @@ export const loadYamlFile = async (habitatData, additionalProviders) => { cloudProviders: buffer.cloudProviders, instanceArray: buffer.instanceArray, }; - } catch(error){ + } catch (error) { console.log("There was an error", error); } - - - - // const response = await fetch( - // "https://sandbox-public-data.s3.us-east-2.amazonaws.com/centml/providers.yaml", - // { cache: "no-store" } - // ); - // if (!response.ok) { - // console.log("Could not read local file"); - // return { cloudProviders: null, instanceArray: null }; - // } - // const localDataText = await response.text(); - // const localDataYaml = load(localDataText); - // CloudProviderReader(localDataYaml, habitatData, buffer); - // CloudProviderReader(additionalProviders, habitatData, buffer); - }; const CloudProviderReader = (data, habitatData, storageBuffer) => { @@ -131,3 +134,37 @@ const CloudProviderReader = (data, habitatData, storageBuffer) => { } } }; + +export const loadJsonFiles = async (habitatData, additionalProviders) => { + const buffer = new ResponseBuffer(); + let urlList = [ + "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json", + ]; + urlList = urlList.concat(additionalProviders); + const listOfPromises = urlList.map((url) => + fetch(url, { cache: "no-store" }) + ); + try { + const responses = await Promise.all(listOfPromises); + for (let resp of responses) { + if (resp.ok) { + const respJsonData = await resp.json(); + const valid = validate(respJsonData); + if (valid) { + console.log(respJsonData); + } else { + console.log("Error"); + validate.errors.forEach((err) => console.log(err.message)); + } + } else { + console.log("error reading from url: ", resp.url); + } + } + return { + cloudProviders: buffer.cloudProviders, + instanceArray: buffer.instanceArray, + }; + } catch (error) { + console.log("There was an error", error); + } +}; From 00355870b252fb04763ea7ec289f255d347255a7 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Mon, 27 Mar 2023 16:03:04 -0400 Subject: [PATCH 12/16] finished reading json from external urls and show incorrect data --- skyline-vscode/react-ui/package-lock.json | 32 --- skyline-vscode/react-ui/package.json | 2 - .../react-ui/src/components/Modals.js | 48 +++++ .../src/schema/CloudProvidersSchema.js | 38 +++- .../react-ui/src/sections/ProviderPanel.js | 87 +++++--- skyline-vscode/react-ui/src/utils/parsers.js | 199 ++++++------------ 6 files changed, 197 insertions(+), 209 deletions(-) create mode 100644 skyline-vscode/react-ui/src/components/Modals.js diff --git a/skyline-vscode/react-ui/package-lock.json b/skyline-vscode/react-ui/package-lock.json index 736f6fc..2afb952 100644 --- a/skyline-vscode/react-ui/package-lock.json +++ b/skyline-vscode/react-ui/package-lock.json @@ -3711,11 +3711,6 @@ "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" }, - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, "aria-query": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.0.2.tgz", @@ -4084,11 +4079,6 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, "batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -4229,15 +4219,6 @@ "node-int64": "^0.4.0" } }, - "buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -6818,11 +6799,6 @@ "harmony-reflect": "^1.4.6" } }, - "ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" - }, "ignore": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", @@ -8967,14 +8943,6 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "requires": { - "argparse": "^2.0.1" - } - }, "jsdom": { "version": "16.7.0", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", diff --git a/skyline-vscode/react-ui/package.json b/skyline-vscode/react-ui/package.json index a7ad86a..705f570 100644 --- a/skyline-vscode/react-ui/package.json +++ b/skyline-vscode/react-ui/package.json @@ -9,8 +9,6 @@ "@fortawesome/react-fontawesome": "^0.1.17", "ajv": "^8.12.0", "bootstrap": "^5.1.3", - "buffer": "^6.0.3", - "js-yaml": "^4.1.0", "less": "^4.1.2", "react": "^17.0.2", "react-bootstrap": "^2.1.2", diff --git a/skyline-vscode/react-ui/src/components/Modals.js b/skyline-vscode/react-ui/src/components/Modals.js new file mode 100644 index 0000000..c26439f --- /dev/null +++ b/skyline-vscode/react-ui/src/components/Modals.js @@ -0,0 +1,48 @@ +import React, { useState } from "react"; +import Button from "react-bootstrap/Button"; +import Modal from "react-bootstrap/Modal"; + +export const ProviderPanelModal = ({ errors }) => { + const [show, setShow] = useState(false); + + const handleClose = () => setShow(false); + const handleShow = () => setShow(true); + + return ( + <> +
+ There were some errors fetching data from the urls. Please see the + List of Errors +
+ + + + List of Errors + + + {errors?.map((err, idx) => { + return ( +
+
{`Error message : ${err.msg}`}
+ {err.code &&

{`Error code: ${err.code}`}

} + {err.invalidFields?.map((incorrectField, id) => { + return ( +
+
{incorrectField.field}
+

{incorrectField.err}

+
+ ); + })} +
+ ); + })} + + + + + + + ); +}; diff --git a/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js b/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js index a3045e8..773c86c 100644 --- a/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js +++ b/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js @@ -1,23 +1,29 @@ export const cloudProviderSchema = { - title:"SCHEMA DEFINITION FOR CLOUD PROVIDERS", - description:"Array of cloud providers, (there needs to be at least 1 cloud provider)", + title: "SCHEMA DEFINITION FOR CLOUD PROVIDERS", + description: + "Array of cloud providers, (there needs to be at least 1 cloud provider)", type: "array", items: { type: "object", - description:"each cloud provider needs to include it's name, logo, color, and instace specifications", - required:["name","logo","color","instances"], + description: + "each cloud provider needs to include it's name, logo, color, and instace specifications", + required: ["name", "logo", "color", "instances"], properties: { - name: { type: "string", description: "name of cloud provider. eg: gcp,aws,azure,coreweaver" }, + name: { + type: "string", + description: "name of cloud provider. eg: gcp,aws,azure,coreweaver", + }, logo: { type: "string", description: "url reference to the logo" }, color: { type: "string", description: "color (CSS format)" }, instances: { type: "array", - description: "list of instances (there needs to be at least 1 instance)", + description: + "list of instances (there needs to be at least 1 instance)", items: { type: "object", - required:["name","gpu","ngpus","cost"], + required: ["name", "gpu", "ngpus", "cost"], properties: { - name: { type: "string", description:"name of the instance" }, + name: { type: "string", description: "name of the instance" }, gpu: { enum: [ "p100", @@ -32,11 +38,21 @@ export const cloudProviderSchema = { "a4000", "rtx4000", ], - description:"only GPUs supported by our product DeepView.Predict" + description: + "only GPUs supported by our product DeepView.Predict", + }, + ngpus: { + type: "integer", + minimum: 1, + description: "number of gpus in the instance", + }, + cost: { + type: "number", + exclusiveMinimum: 0, + description: "cost per hour", }, - ngpus: { type: "integer", minimum: 1, description:"number of gpus in the instance" }, - cost: { type: "number", exclusiveMinimum: 0, description: "cost per hour" }, }, + }, minItems: 1, }, diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.js index 26ace4b..3bdc20f 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.js @@ -14,26 +14,28 @@ import Image from "react-bootstrap/Image"; import Table from "react-bootstrap/Table"; import Form from "react-bootstrap/Form"; import Spinner from "react-bootstrap/Spinner"; -import Alert from 'react-bootstrap/Alert'; +import Alert from "react-bootstrap/Alert"; + +import { deploymentScatterGraphColorSize } from "../data/properties"; + +import { ProviderPanelModal } from "../components/Modals"; -import { - deploymentScatterGraphColorSize -} from "../data/properties"; import { calculate_training_time, numberFormat, currencyFormat, } from "../utils/utils"; -import { loadYamlFile,loadJsonFiles } from "../utils/parsers"; +import { loadJsonFiles } from "../utils/parsers"; -const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { +const ProviderPanel = ({ numIterations, habitatData, additionalProviders }) => { const [providerPanelSettings, setProviderPanelSettings] = useState({ plotData: null, initialData: null, cloudProviders: null, nearest: null, clicked: null, + fetchErrors: null, maxNumGpu: 0, provider: "all", gpu: "all", @@ -41,7 +43,9 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { estimated_time: 0, }); - const habitatIsDemo = habitatData.find(item=>item[0]==="demo" && item[1] === 1) + const habitatIsDemo = habitatData.find( + (item) => item[0] === "demo" && item[1] === 1 + ); const MAX_GPU = [1, 2, 4, 0]; // 0 is all @@ -106,7 +110,9 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { const recalculateCost = (provider) => { if (provider == null) return; - const originalData = providerPanelSettings.initialData.find((item) => item.id === provider.id); + const originalData = providerPanelSettings.initialData.find( + (item) => item.id === provider.id + ); let totalHr = calculate_training_time(numIterations, originalData); // NEED YUBO FEEDBACK let totalCost = provider.info.cost * totalHr; @@ -147,13 +153,13 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { useEffect(() => { async function fetchData() { - await loadJsonFiles(habitatData, additionalProviders); - const data = await loadYamlFile(habitatData, additionalProviders); + const jsondata = await loadJsonFiles(habitatData, additionalProviders); setProviderPanelSettings((prevState) => ({ ...prevState, - plotData: data.instanceArray, - initialData: data.instanceArray, - cloudProviders: data.cloudProviders, + plotData: jsondata.instanceArray, + initialData: jsondata.instanceArray, + cloudProviders: jsondata.cloudProviders, + fetchErrors: jsondata.errors, })); } fetchData(); @@ -161,16 +167,20 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { }, []); return ( <> - {providerPanelSettings.plotData && providerPanelSettings.cloudProviders ? ( + {providerPanelSettings.plotData && + providerPanelSettings.cloudProviders ? (
Providers {habitatIsDemo && ( + + + Currently showing a demo data because local GPU is not + supported by DeepView.Predict + + + )} - Currently showing a demo data because local GPU is not supported by Habitat - - )} - @@ -183,15 +193,15 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { } > - {Object.keys(providerPanelSettings.cloudProviders).map( - (provider, index) => { - return ( - - ); - } - )} + {Object.keys( + providerPanelSettings.cloudProviders + ).map((provider, index) => { + return ( + + ); + })}
@@ -259,6 +269,13 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { /> + {providerPanelSettings.fetchErrors && ( + + + + )}
@@ -340,14 +357,14 @@ const ProviderPanel = ({ numIterations, habitatData,additionalProviders }) => { ) : (
- - - - Loading instances - - - - + + + + Loading instances + + + +
)} diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index d872954..80162d1 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -1,7 +1,3 @@ -/** - * Yaml parser - */ -import { load } from "js-yaml"; import Ajv from "ajv"; import { gpuPropertyList, @@ -12,159 +8,104 @@ import { cloudProviderSchema } from "../schema/CloudProvidersSchema"; const ajv = new Ajv({ allErrors: true }); // to report all validation errors (rather than failing on the first errors) const validate = ajv.compile(cloudProviderSchema); -const data = [ - { - name: "gcp", - logo: "", - color: "#ea4335", - instances: [{ name: "a2-highgpu-1g", gpu: "a100", ngpus: 1, cost: 0.05 }], - }, -]; +// const data = [ +// { +// name: "gcp", +// logo: "", +// color: "#ea4335", +// instances: [{ name: "a2-highgpu-1g", gpu: "a100", ngpus: 1, cost: 0.05 }], +// }, +// ]; -const valid = validate(data); -if (!valid) { - console.log("Error"); - validate.errors.forEach((err) => console.log(err.message)); -} +// const valid = validate(data); +// if (!valid) { +// console.log("Error"); +// validate.errors.forEach((err) => console.log(err.message)); +// } class ResponseBuffer { constructor() { this.instanceId = 0; this.instanceArray = []; this.cloudProviders = {}; + this.errors = []; } } - -export const loadYamlFile = async (habitatData, additionalProviders) => { +export const loadJsonFiles = async (habitatData, additionalProviders) => { const buffer = new ResponseBuffer(); - //data/providers.yaml let urlList = [ - "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.yaml", + "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json", ]; urlList = urlList.concat(additionalProviders); const listOfPromises = urlList.map((url) => fetch(url, { cache: "no-store" }) ); - try { - const responses = await Promise.all(listOfPromises); - for (let resp of responses) { - if (resp.ok) { - const localDataText = await resp.text(); - const localDataYaml = load(localDataText); - CloudProviderReader(localDataYaml, habitatData, buffer); - } else { - console.log("error reading from url: ", resp.url); - } - } - return { - cloudProviders: buffer.cloudProviders, - instanceArray: buffer.instanceArray, - }; - } catch (error) { - console.log("There was an error", error); - } -}; - -const CloudProviderReader = (data, habitatData, storageBuffer) => { - if (data && habitatData) { - for (const [key, value] of Object.entries(data)) { - if (!value) { - continue; - } - const cloudProviderName = key.toLocaleLowerCase(); - if (!(cloudProviderName in storageBuffer.cloudProviders)) { - storageBuffer.cloudProviders[cloudProviderName] = { - name: value.name, - logo: value.logo, - color: value.color, - }; - } - if (value.instances) { - for (const instanceData of value.instances) { - if ( - instanceData.ngpus && - Number.isInteger(instanceData.ngpus) && - instanceData.ngpus !== 0 && - instanceData.cost && - typeof instanceData.cost === "number" - ) { - const instanceAlreadyInArr = storageBuffer.instanceArray.find( - (item) => - item.info.provider === cloudProviderName && - item.info.gpu === instanceData.gpu && - item.info.instance === instanceData.name && - item.info.ngpus === instanceData.ngpus - ); - if (instanceAlreadyInArr) { - instanceAlreadyInArr.info.cost = instanceData.cost; - instanceAlreadyInArr.y = - (instanceData.cost / 3.6e6) * instanceAlreadyInArr.x; - } else { + const responses = await Promise.all(listOfPromises); + for (let resp of responses) { + if (resp.ok) { + try { + const respJsonData = await resp.json(); + const valid = validate(respJsonData); + if (valid) { + for (const cloudProvider of respJsonData) { + buffer.cloudProviders[cloudProvider.name.toLocaleLowerCase()] = { + name: cloudProvider.name, + logo: cloudProvider.logo, + color: cloudProvider.color, + }; + for (const instanceData of cloudProvider.instances) { const found_in_habitat = habitatData.find( (item) => - item[0].toLowerCase() === instanceData.gpu?.toLowerCase() + item[0].toLowerCase() === instanceData.gpu.toLowerCase() ); const found_in_gpuPropertyList = gpuPropertyList.find( (item) => item.name.toLocaleLowerCase() === - instanceData.gpu?.toLocaleLowerCase() + instanceData.gpu.toLocaleLowerCase() ); - if (found_in_habitat && found_in_gpuPropertyList) { - storageBuffer.instanceArray.push({ - id: storageBuffer.instanceId, - x: found_in_habitat[1], // msec - y: (instanceData.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration - info: { - instance: instanceData.name?.toLocaleLowerCase(), - gpu: instanceData.gpu?.toLocaleLowerCase(), - ngpus: instanceData.ngpus, - cost: instanceData.cost, - provider: cloudProviderName.toLocaleLowerCase(), - }, - vmem: found_in_gpuPropertyList.vmem, - fill: value.color, - z: deploymentScatterGraphColorSize.NORMALSIZE, - }); - storageBuffer.instanceId += 1; - } + buffer.instanceArray.push({ + id: buffer.instanceId, + x: found_in_habitat[1], // msec + y: (instanceData.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration + info: { + instance: instanceData.name.toLocaleLowerCase(), + gpu: instanceData.gpu.toLocaleLowerCase(), + ngpus: instanceData.ngpus, + cost: instanceData.cost, + provider: cloudProvider.name.toLocaleLowerCase(), + }, + vmem: found_in_gpuPropertyList.vmem, + fill: cloudProvider.color, + z: deploymentScatterGraphColorSize.NORMALSIZE, + }); + buffer.instanceId += 1; } } - } - } - } - } -}; - -export const loadJsonFiles = async (habitatData, additionalProviders) => { - const buffer = new ResponseBuffer(); - let urlList = [ - "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json", - ]; - urlList = urlList.concat(additionalProviders); - const listOfPromises = urlList.map((url) => - fetch(url, { cache: "no-store" }) - ); - try { - const responses = await Promise.all(listOfPromises); - for (let resp of responses) { - if (resp.ok) { - const respJsonData = await resp.json(); - const valid = validate(respJsonData); - if (valid) { - console.log(respJsonData); } else { - console.log("Error"); - validate.errors.forEach((err) => console.log(err.message)); + buffer.errors.push({ + msg: `invalid data format from url: ${resp.url}`, + invalidFields: validate.errors.map((err) => ({ + field: err.dataPath, + err: err.message, + })), + }); } - } else { - console.log("error reading from url: ", resp.url); + } catch (error) { + buffer.errors.push({ + msg: `error reading from url: ${resp.url}`, + code: "incorrect json data", + }); } + } else { + buffer.errors.push({ + msg: `error reading from url: ${resp.url}`, + code: resp.status, + }); } - return { - cloudProviders: buffer.cloudProviders, - instanceArray: buffer.instanceArray, - }; - } catch (error) { - console.log("There was an error", error); } + return { + cloudProviders: Object.keys(buffer.cloudProviders).length > 1 ? buffer.cloudProviders: null, + instanceArray: buffer.instanceArray.length > 0 ? buffer.instanceArray:null, + errors: buffer.errors.length > 0 ? buffer.errors:null, + }; }; From d36ae3b94c8935bba65d38355069ce3fd4bab5af Mon Sep 17 00:00:00 2001 From: John Calderon Date: Tue, 28 Mar 2023 16:30:43 -0400 Subject: [PATCH 13/16] changed unit testing and tested with build script --- README.md | 29 ++++ schemas/CloudProvidersSchema.yaml | 52 ------- skyline-vscode/package.json | 6 +- .../react-ui/public/data/providers.yaml | 146 ------------------ skyline-vscode/react-ui/src/App.js | 18 +-- .../src/schema/CloudProvidersSchema.js | 4 +- .../src/sections/DeploymentTab.test.js | 33 ++-- .../react-ui/src/sections/Habitat.js | 5 +- .../react-ui/src/sections/Habitat.test.js | 4 +- .../src/sections/ProviderPanel.test.js | 57 ++++--- skyline-vscode/react-ui/src/utils/parsers.js | 26 +--- .../react-ui/src/utils/parsers.test.js | 131 ++++++++-------- skyline-vscode/src/extension.ts | 1 - skyline-vscode/src/skyline_session.ts | 19 +-- 14 files changed, 182 insertions(+), 349 deletions(-) delete mode 100644 schemas/CloudProvidersSchema.yaml delete mode 100644 skyline-vscode/react-ui/public/data/providers.yaml diff --git a/README.md b/README.md index 900b8b7..e2dc957 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,35 @@ To install, either: Once you have the vsix file, run `code --install-extension vscode*.vsix` to install the extension. **Note: the file [build_vsix_dev.sh] is only to be used for development** +**Adding cloud instances to the Deployment Tab:** You can include information about the instances that you use thrugh the extension settings. There you will find an option named **providers** that accepts a list of urls separated by commas. Each url must be a JSON file that follows the schema speficied here : [schema](skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js).
+Addittionally, you need to add the necessary access so the extension can read the file.
+You can use an AWS S3 bucket to store your files. Change CORS settings in Permissions tab. + +**CORS requirements**: +``` +[ + { + "AllowedHeaders": [], + "AllowedMethods": [ + "GET" + ], + "AllowedOrigins": [ + "http://*", + "https://*", + "vscode-webview://*" + ], + "ExposeHeaders": [] + } +] +``` +and allow public access to the file + +This is our [file](https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json) that you can use as an example. + + + + + ### Backend Installation This plugin requires Skyline (the installation instruction for which can be found [here](https://github.com/CentML/skyline)) and (optionally) Habitat (used to extrapolate GPU runtimes, instructions found [here](https://github.com/CentML/habitat)). **Skyline must be launched before running this extension.** diff --git a/schemas/CloudProvidersSchema.yaml b/schemas/CloudProvidersSchema.yaml deleted file mode 100644 index 874a87e..0000000 --- a/schemas/CloudProvidersSchema.yaml +++ /dev/null @@ -1,52 +0,0 @@ -title: - Schema definition for cloud providers -description: | - This schema is intended to be used as a template to add more cloud providers when the VSCode extension is executed. -properties: - provider: - description: | - Object describing the external cloud provider - type: object - properties: - name: - description: Complete name of cloud provider - type: string - logo: - description: Reference to image of cloud provider - type: string - color: - description: Color preference for cloud provider - type: string - instances: - type: array - item: - type: object - properties: - name: - description: Name of instance - type: string - gpu: - description: | - GPU model. Currently supported models: [P100, P4000, RTX2070, RTX2080Ti, T4, V100, A100, RTX3090, A40, A4000, RTX4000] - type: string - ngpus: - description: Number of GPUs - type: integer - cost: - description: Cost per hour - type: float - -#Example -#google: -# name: "Google Cloud Platform" -# logo: "https://blog.hubspot.com/hubfs/image8-2.jpg" -# color: "#ea4335" -# instances: -# - name: "a2-highgpu-1g" -# gpu: "a100" -# ngpus: 1 -# cost: 3.67 -# - name: "n1-standard-1" -# gpu: "t4" -# ngpus: 1 -# cost: 0.4 \ No newline at end of file diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index b2df632..88901fb 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -28,17 +28,17 @@ "properties": { "skyline.address": { "type": "string", - "default": "localhost-2", + "default": "localhost", "description": "Specifies the address of the profiler." }, "skyline.port": { "type": "number", - "default": 60150, + "default": 60120, "description": "Specifies the port of the profiler." }, "skyline.providers": { "type": "string", - "default": "another", + "default": "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/dummy1.json", "description": "additional urls separated by commas" }, "skyline.isTelemetryEnabled": { diff --git a/skyline-vscode/react-ui/public/data/providers.yaml b/skyline-vscode/react-ui/public/data/providers.yaml deleted file mode 100644 index 5ee09c8..0000000 --- a/skyline-vscode/react-ui/public/data/providers.yaml +++ /dev/null @@ -1,146 +0,0 @@ - -#GPU Instances -#costs are 1 GPU * hr -#For Google we need to add the cost for the instance itself, a100 run on specific A2 Series -#while t4, p4, v100, and p100 run on General purposes N1 Series - -#References: -#FOR AWS: -#https://aws.amazon.com/ec2/instance-types/ -#https://aws.amazon.com/ec2/pricing/on-demand/ - -#FOR GOOGLE: -#https://cloud.google.com/compute/gpus-pricing -#https://cloud.google.com/products/calculator/ - -#FOR AZURE: -#https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-gpu -#https://azure.microsoft.com/en-ca/pricing/details/virtual-machines/windows/#pricing -# ---- - google: - name: "Google Cloud Platform" - logo: "resources/google.png" - color: "#ea4335" - instances: - - name: "a2-highgpu-1g" - gpu: "a100" - ngpus: 1 - cost: 3.67 - - name: "n1-standard-1" - gpu: "t4" - ngpus: 1 - cost: 0.4 - - name: "n1-standard-1" - gpu: "p4" - ngpus: 1 - cost: 0.65 - - name: "n1-standard-1" - gpu: "v100" - ngpus: 1 - cost: 2.53 - - name: "n1-standard-1" - gpu: "p100" - ngpus: 1 - cost: 1.51 - - name: "a2-highgpu-2g" - gpu: "a100" - ngpus: 2 - cost: 7.35 - - name: "n1-standard-1" - gpu: "t4" - ngpus: 2 - cost: 0.75 - - name: "n1-standard-1" - gpu: "p4" - ngpus: 2 - cost: 1.25 - - name: "n1-standard-1" - gpu: "v100" - ngpus: 2 - cost: 5.01 - - name: "n1-standard-1" - gpu: "p100" - ngpus: 2 - cost: 2.97 - - name: "a2-highgpu-4g" - gpu: "a100" - ngpus: 4 - cost: 14.7 - - name: "n1-standard-1" - gpu: "t4" - ngpus: 4 - cost: 1.45 - - name: "n1-standard-1" - gpu: "p4" - ngpus: 4 - cost: 2.45 - - name: "n1-standard-1" - gpu: "v100" - ngpus: 4 - cost: 9.97 - - name: "n1-standard-1" - gpu: "p100" - ngpus: 4 - cost: 5.89 - aws: - name: "Amazon Web Services" - logo: "resources/aws.png" - color: "#ff9900" - instances: - - name: "p3.2xlarge" - gpu: "v100" - ngpus: 1 - cost: 3.06 - - name: "g4dn.xlarge" - gpu: "t4" - ngpus: 1 - cost: 0.526 - - name: "g4dn.12xlarge" - gpu: "t4" - ngpus: 4 - cost: 3.912 - - name: "p3.8xlarge" - gpu: "v100" - ngpus: 4 - cost: 12.24 - azure: - name: "Microsoft Azure" - logo: "resources/azure.png" - color: "#008AD7" - instances: - - name: "NC6s v2" - gpu: "p100" - ngpus: 1 - cost: 2.07 - - name: "NC6s v3" - gpu: "v100" - ngpus: 1 - cost: 3.06 - - name: "NC4as T4 v3" - gpu: "t4" - ngpus: 1 - cost: 0.526 - - name: "NC12s v2" - gpu: "p100" - ngpus: 2 - cost: 4.14 - - name: "NC12s v3" - gpu: "v100" - ngpus: 2 - cost: 6.12 - - name: "NC24rs v2" - gpu: "p100" - ngpus: 4 - cost: 9.108 - - name: "NC24rs v3" - gpu: "v100" - ngpus: 4 - cost: 13.46 - - name: "NC64as T4 v3" - gpu: "t4" - ngpus: 4 - cost: 4.352 - - - \ No newline at end of file diff --git a/skyline-vscode/react-ui/src/App.js b/skyline-vscode/react-ui/src/App.js index 3961734..a4ebe08 100644 --- a/skyline-vscode/react-ui/src/App.js +++ b/skyline-vscode/react-ui/src/App.js @@ -53,7 +53,7 @@ function restartProfiling() { }); } -const sendMock = true; +const sendMock = false; function App() { const [analysisState, setAnalysisState] = useState(); @@ -64,7 +64,6 @@ function App() { const [errorText, setErrorText] = useState(); const [connectionStatus, setConnectionStatus] = useState(false); const [numIterations, setNumIterations] = useState(100000); - const [externalData, setExternalData] = useState([]); App.vscodeApi = vscodeApi; @@ -95,7 +94,6 @@ function App() { ), fine: computePercentage(fine, state.breakdown.iteration_run_time_ms), }); - ReactTooltip.rebuild(); } }; @@ -104,14 +102,14 @@ function App() { if (event.data["message_type"] === "connection") { setConnectionStatus(event.data["status"]); } else if (event.data["message_type"] === "analysis") { - if(event.data.habitat[0][0]==="unavailable" && event.data.habitat[0][1]===-1.0){ - event.data.habitat = profiling_data.habitat - event.data.habitat.push(["demo",1]) + if ( + event.data.habitat[0][0] === "unavailable" && + event.data.habitat[0][1] === -1.0 + ) { + event.data.habitat = profiling_data.habitat; + event.data.habitat.push(["demo", 1]); } processAnalysisState(event.data); - } else if (event.data["message_type"] === "loaded_additional_providers") { - console.log(event.data["additionalProviders"].split(",")) - setExternalData(event.data["additionalProviders"].split(",")); } else if (event.data["message_type"] === "text_change") { setTextChanged(true); } else if (event.data["message_type"] === "error") { @@ -229,7 +227,7 @@ function App() { diff --git a/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js b/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js index 773c86c..a7defa1 100644 --- a/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js +++ b/skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js @@ -1,12 +1,12 @@ export const cloudProviderSchema = { title: "SCHEMA DEFINITION FOR CLOUD PROVIDERS", description: - "Array of cloud providers, (there needs to be at least 1 cloud provider)", + "list of cloud providers, (there needs to be at least 1 cloud provider)", type: "array", items: { type: "object", description: - "each cloud provider needs to include it's name, logo, color, and instace specifications", + "each cloud provider needs to include it's name, logo, color, and instance specifications", required: ["name", "logo", "color", "instances"], properties: { name: { diff --git a/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js b/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js index 4633d49..87253bc 100644 --- a/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js +++ b/skyline-vscode/react-ui/src/sections/DeploymentTab.test.js @@ -5,7 +5,7 @@ enableFetchMocks(); const { ResizeObserver } = window; const numIterations = 10000; -const data = [ +const habitatData = [ ["source", 22.029312], ["P100", 14.069682], ["P4000", 127.268085], @@ -20,18 +20,21 @@ const data = [ ["RTX4000", 20.2342], ]; -const yamlTestData = ` ---- - google: - name: "Google Cloud Platform" - logo: "resources/google.png" - color: "#ea4335" - instances: - - name: "a2-highgpu-1g" - gpu: "a100" - ngpus: 1 - cost: 3.67 -` +const correctData = [ + { + name:'google', + logo:'resources/google.png', + color:'#ea4335', + instances:[ + { + name:'a2-highgpu-1g', + gpu: 'a100', + ngpus:1, + cost:36.67 + } + ] + } +] beforeEach(() => { delete window.ResizeObserver; @@ -79,11 +82,11 @@ test("Shows deployment target when there is habitat data", async () => { // Mock fetch response jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, - text: jest.fn().mockResolvedValue(yamlTestData) + json: jest.fn().mockResolvedValue(correctData) }) const { container } = render( - + ); // ASSERT diff --git a/skyline-vscode/react-ui/src/sections/Habitat.js b/skyline-vscode/react-ui/src/sections/Habitat.js index cda7d6f..bcac6b2 100644 --- a/skyline-vscode/react-ui/src/sections/Habitat.js +++ b/skyline-vscode/react-ui/src/sections/Habitat.js @@ -18,7 +18,7 @@ export default function Habitat({ habitatData }) {
-
The local GPU is not supported by Habitat
+
The local GPU is not supported by DeepView.Predict
@@ -37,8 +37,7 @@ export default function Habitat({ habitatData }) { - Loading Habitat - predictions. + Loading DeepView.Predict data diff --git a/skyline-vscode/react-ui/src/sections/Habitat.test.js b/skyline-vscode/react-ui/src/sections/Habitat.test.js index 9e3e53c..49fd53d 100644 --- a/skyline-vscode/react-ui/src/sections/Habitat.test.js +++ b/skyline-vscode/react-ui/src/sections/Habitat.test.js @@ -66,7 +66,7 @@ test("Shows loading spinner when there is no habitat data", () => { render(); // ASSERT - expect(screen.getByText(/loading habitat predictions/i)).toBeTruthy(); + expect(screen.getByText(/loading deepview.predict data/i)).toBeTruthy(); }); test("Shows graph when habitat data is present", () => { @@ -82,5 +82,5 @@ test("no habitat data received from backend", ()=>{ // ARRANGE const { container } = render(); // ASSERT - expect(screen.getByText(/the local gpu is not supported by habitat/i)).toBeTruthy(); + expect(screen.getByText(/the local gpu is not supported by deepview.predict/i)).toBeTruthy(); }) diff --git a/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js b/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js index 8e51d61..213b2d4 100644 --- a/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js +++ b/skyline-vscode/react-ui/src/sections/ProviderPanel.test.js @@ -6,7 +6,7 @@ enableFetchMocks() const { ResizeObserver } = window; const numIterations = 10000; -const data = [ +const habitatData = [ ["source", 22.029312], ["P100", 14.069682], ["P4000", 127.268085], // 27.268085 @@ -21,18 +21,34 @@ const data = [ ["RTX4000", 20.2342], ]; -const yamlTestData = ` ---- - google: - name: "Google Cloud Platform" - logo: "resources/google.png" - color: "#ea4335" - instances: - - name: "a2-highgpu-1g" - gpu: "a100" - ngpus: 1 - cost: 3.67 -` +const correctData = [ + { + name: "gcp", + logo: "resources/google.png", + color: "#ea4335", + instances: [ + { + name: "a2-highgpu-1g", + gpu: "a100", + ngpus: 1, + cost: 3.67, + }, + ], + }, + { + name: "aws", + logo: "resources/aws.png", + color: "#ff9900", + instances: [ + { + name: "p3.2xlarge", + gpu: "v100", + ngpus: 1, + cost: 3.06, + }, + ], + }, +]; const noHabitatData = [ ["source", 22.029312], @@ -85,11 +101,11 @@ test("Shows a scatter chart", async() => { // Mock fetch response jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, - text: jest.fn().mockResolvedValue(yamlTestData) + json: jest.fn().mockResolvedValue(correctData) }) const { container } = render( - + ); // ASSERT @@ -100,9 +116,14 @@ test("Shows a scatter chart", async() => { expect(await screen.findByText(/select a configuration/i)).toBeTruthy(); }); -test("no habitat data received from backend", ()=>{ +test("no habitat data received from backend", async()=>{ // ARRANGE - const { container } = render(); + // Mock fetch response + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue(correctData) + }) + const { container } = render(); // ASSERT - expect(screen.getByText(/currently showing a demo data because local gpu is not supported by habitat/i)).toBeTruthy(); + expect(await screen.findByText(/Currently showing a demo data because local GPU is not supported by DeepView.Predict/i)).toBeTruthy(); }) \ No newline at end of file diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 80162d1..3e9ac66 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -8,21 +8,6 @@ import { cloudProviderSchema } from "../schema/CloudProvidersSchema"; const ajv = new Ajv({ allErrors: true }); // to report all validation errors (rather than failing on the first errors) const validate = ajv.compile(cloudProviderSchema); -// const data = [ -// { -// name: "gcp", -// logo: "", -// color: "#ea4335", -// instances: [{ name: "a2-highgpu-1g", gpu: "a100", ngpus: 1, cost: 0.05 }], -// }, -// ]; - -// const valid = validate(data); -// if (!valid) { -// console.log("Error"); -// validate.errors.forEach((err) => console.log(err.message)); -// } - class ResponseBuffer { constructor() { this.instanceId = 0; @@ -36,7 +21,8 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { let urlList = [ "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json", ]; - urlList = urlList.concat(additionalProviders); + const additionalList = additionalProviders ? additionalProviders.split(","):[]; + urlList = urlList.concat(additionalList); const listOfPromises = urlList.map((url) => fetch(url, { cache: "no-store" }) ); @@ -85,7 +71,7 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { buffer.errors.push({ msg: `invalid data format from url: ${resp.url}`, invalidFields: validate.errors.map((err) => ({ - field: err.dataPath, + field: err.instancePath, err: err.message, })), }); @@ -93,18 +79,18 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { } catch (error) { buffer.errors.push({ msg: `error reading from url: ${resp.url}`, - code: "incorrect json data", + code: `status: ${resp.statusText} | code: ${resp.status}`, }); } } else { buffer.errors.push({ msg: `error reading from url: ${resp.url}`, - code: resp.status, + code: `status: ${resp.statusText} | code: ${resp.status}`, }); } } return { - cloudProviders: Object.keys(buffer.cloudProviders).length > 1 ? buffer.cloudProviders: null, + cloudProviders: Object.keys(buffer.cloudProviders).length > 0 ? buffer.cloudProviders: null, instanceArray: buffer.instanceArray.length > 0 ? buffer.instanceArray:null, errors: buffer.errors.length > 0 ? buffer.errors:null, }; diff --git a/skyline-vscode/react-ui/src/utils/parsers.test.js b/skyline-vscode/react-ui/src/utils/parsers.test.js index 83eb13f..2b3dffa 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.test.js +++ b/skyline-vscode/react-ui/src/utils/parsers.test.js @@ -1,4 +1,4 @@ -import { loadYamlFile } from "./parsers"; +import { loadJsonFiles } from "./parsers"; import { enableFetchMocks } from "jest-fetch-mock"; enableFetchMocks(); @@ -17,48 +17,77 @@ const habitatData = [ ["RTX4000", 20.2342], ]; - -const yamlTestData = ` ---- - google: - name: "Google Cloud Platform" - logo: "resources/google.png" - color: "#ea4335" - instances: - - name: "a2-highgpu-1g" - gpu: "a100" - ngpus: 1 - cost: 3.67 -`; -const dataFromVSCodeExtension = { - aws: { - name: "Amazon Web Services", +const correctData = [ + { + name: "gcp", + logo: "resources/google.png", + color: "#ea4335", + instances: [ + { + name: "a2-highgpu-1g", + gpu: "a100", + ngpus: 1, + cost: 3.67, + }, + ], + }, + { + name: "aws", logo: "resources/aws.png", color: "#ff9900", - instances: [{ name: "p3.2xlarge", gpu: "v100", ngpus: 1, cost: 3.06 }], + instances: [ + { + name: "p3.2xlarge", + gpu: "v100", + ngpus: 1, + cost: 3.06, + }, + ], }, -}; -const incorrectDataFromVSCodeExtension = { - coreweave: { - name: "coreweave", - instances: [{ name: "p3.2xlarge", gpu: "v100", ngpus: 2.3, cost: '3.06' }], +]; + +const incorrectData = [ + { + name: "gcp", + logo: "resources/google.png", + color: "#ea4335", + instances: [ + { + name: "a2-highgpu-1g", + gpu: "a100", + ngpus: 1, + cost: 3.67, + }, + ], }, -} + { + name: "aws", + logo: "resources/aws.png", + instances: [ + { + name: "p3.2xlarge", + gpu: "rtx3050", + ngpus: 0, + cost: -3.06, + }, + ], + }, +]; -test("Parse yaml files and return list of cloud providers and instances", async () => { +test("Validate JSON file (URL) and return list of cloud providers and instances", async () => { jest.spyOn(global, "fetch").mockResolvedValue({ ok: true, - text: jest.fn().mockResolvedValue(yamlTestData), + json: jest.fn().mockResolvedValue(correctData), }); - const resp = await loadYamlFile(habitatData, dataFromVSCodeExtension); + const resp = await loadJsonFiles(habitatData, ""); expect(resp.cloudProviders).toStrictEqual({ - google: { - name: "Google Cloud Platform", + gcp: { + name: "gcp", logo: "resources/google.png", color: "#ea4335", }, aws: { - name: "Amazon Web Services", + name: "aws", logo: "resources/aws.png", color: "#ff9900", }, @@ -73,7 +102,7 @@ test("Parse yaml files and return list of cloud providers and instances", async gpu: "a100", ngpus: 1, cost: 3.67, - provider: "google", + provider: "gcp", }, vmem: 40, fill: "#ea4335", @@ -96,38 +125,14 @@ test("Parse yaml files and return list of cloud providers and instances", async }); }); -test("External yaml file is not in correct format", async()=>{ +test("JSON file (URL) is not in the correct format", async () => { jest.spyOn(global, "fetch").mockResolvedValue({ ok: true, - text: jest.fn().mockResolvedValue(yamlTestData), - }); - const resp = await loadYamlFile(habitatData, incorrectDataFromVSCodeExtension); - expect(resp.cloudProviders).toStrictEqual({ - google: { - name: "Google Cloud Platform", - logo: "resources/google.png", - color: "#ea4335", - }, - coreweave: { - name: "coreweave", - logo: undefined, - color: undefined, - }, - }); - expect(resp.instanceArray.length).toEqual(1); - expect(resp.instanceArray).toContainEqual({ - id: 0, - x: 10.068596, - y: 0.000010264374255555554, - info: { - instance: "a2-highgpu-1g", - gpu: "a100", - ngpus: 1, - cost: 3.67, - provider: "google", - }, - vmem: 40, - fill: "#ea4335", - z: 200, + json: jest.fn().mockResolvedValue(incorrectData), }); -}) \ No newline at end of file + const resp = await loadJsonFiles(habitatData, ""); + expect(resp).toBeDefined(); + expect(resp.cloudProviders).toBeNull() + expect(resp.instanceArray).toBeNull() + expect(resp.errors[0].msg).toMatch(/invalid data format from url/) +}); diff --git a/skyline-vscode/src/extension.ts b/skyline-vscode/src/extension.ts index be5ed1f..6c61f66 100644 --- a/skyline-vscode/src/extension.ts +++ b/skyline-vscode/src/extension.ts @@ -51,7 +51,6 @@ export function activate(context: vscode.ExtensionContext) { retainContextWhenHidden: true } ); - console.log("EXTENSION.TS FILE",vsconfig); let sess_options: SkylineSessionOptions = { context: context, projectRoot: uri[0].fsPath, diff --git a/skyline-vscode/src/skyline_session.ts b/skyline-vscode/src/skyline_session.ts index b6214e2..c272233 100644 --- a/skyline-vscode/src/skyline_session.ts +++ b/skyline-vscode/src/skyline_session.ts @@ -6,6 +6,7 @@ const fs = require('fs'); import {Socket} from 'net'; import { simpleDecoration } from './decorations'; import { energy_component_type_mapping, getObjectKeyNameFromValue } from './utils'; +import { stringify } from 'querystring'; const crypto = require('crypto'); const resolve = require('path').resolve; @@ -91,7 +92,6 @@ export class SkylineSession { this.webviewPanel.onDidDispose(this.disconnect.bind(this)); this.webviewPanel.webview.html = this._getHtmlForWebview(); this.connect(); - this.load_external_providers(); vscode.workspace.onDidChangeTextDocument(this.on_text_change.bind(this)); this.restart_profiling = this.restart_profiling.bind(this); @@ -172,17 +172,6 @@ export class SkylineSession { this.webviewPanel.webview.postMessage(errorEvent); } - load_external_providers() { - console.log("PROVIDERS: ",this.providers); - console.log("PORT: ",this.port); - let otherUrls = { - "message_type": "loaded_additional_providers", - "additionalProviders": this.providers, - }; - console.log("sending list of urls"); - this.webviewPanel.webview.postMessage(otherUrls); - } - webview_handle_message(msg: any) { console.log("webview_handle_message"); console.log(msg); @@ -393,7 +382,7 @@ export class SkylineSession { Skyline - + @@ -421,7 +410,9 @@ export class SkylineSession { "breakdown": {}, "habitat": [] as Array<[string, number]>, - "energy": {} + "additionalProviders": this.providers, + "energy": {}, + }; if (this.msg_throughput) { From 5c7d0b3877a0d7b7c0fe6fd83439cc0c476cec44 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Wed, 29 Mar 2023 09:36:11 -0400 Subject: [PATCH 14/16] changed default providers settings to empty string --- skyline-vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index 88901fb..8357492 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -38,7 +38,7 @@ }, "skyline.providers": { "type": "string", - "default": "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/dummy1.json", + "default": "", "description": "additional urls separated by commas" }, "skyline.isTelemetryEnabled": { From 5697d6e604c73d4ad1cd368a1893b91b536e9cc0 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Wed, 29 Mar 2023 11:53:14 -0400 Subject: [PATCH 15/16] PR review --- README.md | 4 +- .../react-ui/src/data/properties.js | 2 + skyline-vscode/react-ui/src/utils/parsers.js | 51 ++--- skyline-vscode/react-ui/src/utils/utils.js | 209 ++++++++++-------- 4 files changed, 145 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index e2dc957..e05ddab 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ To install, either: Once you have the vsix file, run `code --install-extension vscode*.vsix` to install the extension. **Note: the file [build_vsix_dev.sh] is only to be used for development** -**Adding cloud instances to the Deployment Tab:** You can include information about the instances that you use thrugh the extension settings. There you will find an option named **providers** that accepts a list of urls separated by commas. Each url must be a JSON file that follows the schema speficied here : [schema](skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js).
-Addittionally, you need to add the necessary access so the extension can read the file.
+**Adding cloud instances to the Deployment Tab:** You can include information about the instances that you use thrugh the extension settings. There you will find an option named **providers** that accepts a list of urls separated by commas. Each url must be a JSON file that follows the schema specified here : [schema](skyline-vscode/react-ui/src/schema/CloudProvidersSchema.js).
+Additionally, you need to add the necessary access so the extension can read the file.
You can use an AWS S3 bucket to store your files. Change CORS settings in Permissions tab. **CORS requirements**: diff --git a/skyline-vscode/react-ui/src/data/properties.js b/skyline-vscode/react-ui/src/data/properties.js index 8325bb4..5e9d5fa 100644 --- a/skyline-vscode/react-ui/src/data/properties.js +++ b/skyline-vscode/react-ui/src/data/properties.js @@ -21,3 +21,5 @@ export const deploymentScatterGraphColorSize = { HIGHLIGHTSIZE: 280, }; +export const CENTML_CLOUD_PROVIDERS_URL = "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json"; + diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index 3e9ac66..f22b27a 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -1,25 +1,24 @@ import Ajv from "ajv"; import { gpuPropertyList, + CENTML_CLOUD_PROVIDERS_URL, deploymentScatterGraphColorSize, } from "../data/properties"; +import {fetchingURLErrors} from '../utils/utils'; import { cloudProviderSchema } from "../schema/CloudProvidersSchema"; const ajv = new Ajv({ allErrors: true }); // to report all validation errors (rather than failing on the first errors) const validate = ajv.compile(cloudProviderSchema); -class ResponseBuffer { - constructor() { - this.instanceId = 0; - this.instanceArray = []; - this.cloudProviders = {}; - this.errors = []; - } -} export const loadJsonFiles = async (habitatData, additionalProviders) => { - const buffer = new ResponseBuffer(); + let instanceId = 0; + const instanceArray = []; + const cloudProviders = {}; + const errors = []; + + // const buffer = new cloudProviderAndInstancesBuilder(); let urlList = [ - "https://deepview-explorer-public.s3.amazonaws.com/vscode-cloud-providers/providers.json", + CENTML_CLOUD_PROVIDERS_URL, ]; const additionalList = additionalProviders ? additionalProviders.split(","):[]; urlList = urlList.concat(additionalList); @@ -34,7 +33,7 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { const valid = validate(respJsonData); if (valid) { for (const cloudProvider of respJsonData) { - buffer.cloudProviders[cloudProvider.name.toLocaleLowerCase()] = { + cloudProviders[cloudProvider.name.toLocaleLowerCase()] = { name: cloudProvider.name, logo: cloudProvider.logo, color: cloudProvider.color, @@ -49,8 +48,8 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { item.name.toLocaleLowerCase() === instanceData.gpu.toLocaleLowerCase() ); - buffer.instanceArray.push({ - id: buffer.instanceId, + instanceArray.push({ + id: instanceId, x: found_in_habitat[1], // msec y: (instanceData.cost / 3.6e6) * found_in_habitat[1], // cost per msec * habitatData = cost per 1 iteration info: { @@ -64,34 +63,22 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { fill: cloudProvider.color, z: deploymentScatterGraphColorSize.NORMALSIZE, }); - buffer.instanceId += 1; + instanceId += 1; } } } else { - buffer.errors.push({ - msg: `invalid data format from url: ${resp.url}`, - invalidFields: validate.errors.map((err) => ({ - field: err.instancePath, - err: err.message, - })), - }); + errors.push(fetchingURLErrors("schemaValidationErrors",resp,validate)); } } catch (error) { - buffer.errors.push({ - msg: `error reading from url: ${resp.url}`, - code: `status: ${resp.statusText} | code: ${resp.status}`, - }); + errors.push(fetchingURLErrors("noJsonResponseFromUrl",resp,null)); } } else { - buffer.errors.push({ - msg: `error reading from url: ${resp.url}`, - code: `status: ${resp.statusText} | code: ${resp.status}`, - }); + errors.push(fetchingURLErrors(null,resp,null)); } } return { - cloudProviders: Object.keys(buffer.cloudProviders).length > 0 ? buffer.cloudProviders: null, - instanceArray: buffer.instanceArray.length > 0 ? buffer.instanceArray:null, - errors: buffer.errors.length > 0 ? buffer.errors:null, + cloudProviders: Object.keys(cloudProviders).length > 0 ? cloudProviders: null, + instanceArray: instanceArray.length > 0 ? instanceArray:null, + errors: errors.length > 0 ? errors:null, }; }; diff --git a/skyline-vscode/react-ui/src/utils/utils.js b/skyline-vscode/react-ui/src/utils/utils.js index 1fae669..89d63f2 100644 --- a/skyline-vscode/react-ui/src/utils/utils.js +++ b/skyline-vscode/react-ui/src/utils/utils.js @@ -1,79 +1,72 @@ -const BYTE_UNITS = [ - 'B', - 'KB', - 'MB', - 'GB', -]; +const BYTE_UNITS = ["B", "KB", "MB", "GB"]; -const ENERGY_UNITS = [ - 'J', - 'KJ', - 'MJ', - 'GJ', - 'TJ', - 'PJ', - 'EJ' -] +const ENERGY_UNITS = ["J", "KJ", "MJ", "GJ", "TJ", "PJ", "EJ"]; const GENERIC_UNITS = [ - '', - 'Thousands', - 'Millions', + "", + "Thousands", + "Millions", "Billions", - 'Trillion', - 'Quadrillion' -] + "Trillion", + "Quadrillion", +]; const formatTimeUnits = [ - [365,'day'], - [24, 'hour'], - [60, 'minute'], - [60, 'second'], - [1000,'msec'] -] + [365, "day"], + [24, "hour"], + [60, "minute"], + [60, "second"], + [1000, "msec"], +]; // reference : https://www.epa.gov/energy/greenhouse-gas-equivalencies-calculator const ENERGY_CONVERSION_UNITS = { kwh: 2.77778e-7, // 1J = 2.77778e-7 kwh carbon: 4.33e-4, // Electricity consumed (kilowatt-hours) : 7.09 × 10-4 metric tons CO2/kWh - miles: 1/(4.03e-4), // Miles driven by the average gasoline-powered passenger vehicle : 4.03 x 10-4 metric tons CO2E/mile - household: 1/7.94, // Home energy use : 7.94 metric tons CO2 per home per year. - phone: 1/(8.22e-6) // 8.22 x 10-6 metric tons CO2/smartphone charged -} + miles: 1 / 4.03e-4, // Miles driven by the average gasoline-powered passenger vehicle : 4.03 x 10-4 metric tons CO2E/mile + household: 1 / 7.94, // Home energy use : 7.94 metric tons CO2 per home per year. + phone: 1 / 8.22e-6, // 8.22 x 10-6 metric tons CO2/smartphone charged +}; -export function unitScale(quantity, unit){ +export function unitScale(quantity, unit) { let idx = 0; - while(quantity > 1000){ - quantity /=1000; - idx +=1; + while (quantity > 1000) { + quantity /= 1000; + idx += 1; } - switch(unit){ - case 'energy': - return {val:parseFloat(Number(quantity).toFixed(2)),scale:ENERGY_UNITS[idx],scale_index:idx} - case 'generic': - return {val:parseFloat(Number(quantity).toFixed(2)),scale:GENERIC_UNITS[idx],scale_index:idx} + switch (unit) { + case "energy": + return { + val: parseFloat(Number(quantity).toFixed(2)), + scale: ENERGY_UNITS[idx], + scale_index: idx, + }; + case "generic": + return { + val: parseFloat(Number(quantity).toFixed(2)), + scale: GENERIC_UNITS[idx], + scale_index: idx, + }; default: - return null + return null; } - } -export function homeEnergyUsedFormat(quantity){ - if (quantity >= 1){ - return [parseFloat(Number(quantity).toFixed(2)),'year'] +export function homeEnergyUsedFormat(quantity) { + if (quantity >= 1) { + return [parseFloat(Number(quantity).toFixed(2)), "year"]; } let idx = 0; let converter = []; - while(quantity < 1){ + while (quantity < 1) { converter = formatTimeUnits[idx]; quantity *= converter[0]; - idx+=1; + idx += 1; } - return [parseFloat(Number(quantity).toFixed(2)),converter[1]]; + return [parseFloat(Number(quantity).toFixed(2)), converter[1]]; } - // export function processFileReference(fileReferenceProto) { // return { // filePath: path.join(...(fileReferenceProto.getFilePath().getComponentsList())), @@ -82,8 +75,8 @@ export function homeEnergyUsedFormat(quantity){ // }; export function toPercentage(numerator, denominator) { - return numerator / denominator * 100; -}; + return (numerator / denominator) * 100; +} export function toReadableByteSize(sizeBytes) { let index = 0; @@ -99,10 +92,10 @@ export function toReadableByteSize(sizeBytes) { } return `${size.toFixed(1)} ${BYTE_UNITS[index]}`; -}; +} -export function scalePercentages({scaleSelector, shouldScale, applyFactor}) { - return function(list, scaleFactor) { +export function scalePercentages({ scaleSelector, shouldScale, applyFactor }) { + return function (list, scaleFactor) { let total = 0; const adjusted = []; @@ -119,18 +112,19 @@ export function scalePercentages({scaleSelector, shouldScale, applyFactor}) { } return adjusted.map(([newValue, element]) => - applyFactor(element, toPercentage(newValue, total))); + applyFactor(element, toPercentage(newValue, total)) + ); }; -}; +} export function getTraceByLevel(tree) { console.log("getTraceByLevel"); - var tree_size = function(idx) { + var tree_size = function (idx) { let total = 1; let num_children = tree[idx]["num_children"]; for (let i = 0; i < num_children; i++) { - tree[idx+total]["depth"] = 1 + tree[idx]["depth"]; - tree[idx+total]["parent"] = tree[idx]; + tree[idx + total]["depth"] = 1 + tree[idx]["depth"]; + tree[idx + total]["parent"] = tree[idx]; total += tree_size(idx + total); } return total; @@ -139,69 +133,87 @@ export function getTraceByLevel(tree) { tree[0]["depth"] = 0; tree_size(0); - let coarseDecomposition = tree.filter(node => { return node["depth"] === 1; }); - for (let fineLevel = 1; ; fineLevel ++) { - let fineDecomposition = tree.filter(node => { return node["depth"] === fineLevel; }); + let coarseDecomposition = tree.filter((node) => { + return node["depth"] === 1; + }); + for (let fineLevel = 1; ; fineLevel++) { + let fineDecomposition = tree.filter((node) => { + return node["depth"] === fineLevel; + }); console.log(`fineLevel: ${fineLevel}, length: ${fineDecomposition.length}`); - if (fineDecomposition.length === 0) return { coarse: coarseDecomposition, fine: coarseDecomposition }; - if (fineDecomposition.length >= 7) return { coarse: coarseDecomposition, fine: fineDecomposition }; + if (fineDecomposition.length === 0) + return { coarse: coarseDecomposition, fine: coarseDecomposition }; + if (fineDecomposition.length >= 7) + return { coarse: coarseDecomposition, fine: fineDecomposition }; } } export function computePercentage(operations, total_time) { let tracked_total_time = 0; for (let elem in operations) { - tracked_total_time += operations[elem]["forward_ms"] + operations[elem]["backward_ms"]; + tracked_total_time += + operations[elem]["forward_ms"] + operations[elem]["backward_ms"]; } total_time = Math.max(total_time, tracked_total_time); for (let elem in operations) { - operations[elem]["percentage"] = 100 * (operations[elem]["forward_ms"] + operations[elem]["backward_ms"]) / total_time; + operations[elem]["percentage"] = + (100 * + (operations[elem]["forward_ms"] + operations[elem]["backward_ms"])) / + total_time; } if (tracked_total_time < total_time) { operations.push({ name: "untracked", - percentage: 100 * (total_time - tracked_total_time) / total_time, + percentage: (100 * (total_time - tracked_total_time)) / total_time, num_children: 0, forward_ms: 0, backward_ms: 0, size_bytes: 0, - total_time: total_time - tracked_total_time + total_time: total_time - tracked_total_time, }); } return operations; } -export function energy_data(currentTotal){ - const kwh = currentTotal * ENERGY_CONVERSION_UNITS['kwh']; - const carbon_emission_tons = kwh*ENERGY_CONVERSION_UNITS['carbon']; - const carbon_unit = carbon_emission_tons < 1 ? - `${parseFloat(Number(carbon_emission_tons*1000).toFixed(2))} kg`: - `${parseFloat(Number(carbon_emission_tons).toFixed(2))} Metric Tons`; - const miles = unitScale(carbon_emission_tons * ENERGY_CONVERSION_UNITS['miles'],'generic'); - const household = homeEnergyUsedFormat(carbon_emission_tons * ENERGY_CONVERSION_UNITS['household']); - const phone = unitScale(carbon_emission_tons * ENERGY_CONVERSION_UNITS['phone'],'generic'); +export function energy_data(currentTotal) { + const kwh = currentTotal * ENERGY_CONVERSION_UNITS["kwh"]; + const carbon_emission_tons = kwh * ENERGY_CONVERSION_UNITS["carbon"]; + const carbon_unit = + carbon_emission_tons < 1 + ? `${parseFloat(Number(carbon_emission_tons * 1000).toFixed(2))} kg` + : `${parseFloat(Number(carbon_emission_tons).toFixed(2))} Metric Tons`; + const miles = unitScale( + carbon_emission_tons * ENERGY_CONVERSION_UNITS["miles"], + "generic" + ); + const household = homeEnergyUsedFormat( + carbon_emission_tons * ENERGY_CONVERSION_UNITS["household"] + ); + const phone = unitScale( + carbon_emission_tons * ENERGY_CONVERSION_UNITS["phone"], + "generic" + ); return { kwh: parseFloat(Number(kwh).toFixed(2)), carbon: carbon_unit, miles: `${miles.val} ${miles.scale}`, household: household, - phone: `${phone.val} ${phone.scale}` - } - + phone: `${phone.val} ${phone.scale}`, + }; } -export function calculate_training_time(numIterations, instance){ +export function calculate_training_time(numIterations, instance) { // instance.x is the time for 1 iterations in msec // 3.6e6 to convert total training time from msec to hours, divided by the total number of GPUS // output is in hours - return numIterations * instance.x / 3.6e6 / instance.info.ngpus; + return (numIterations * instance.x) / 3.6e6 / instance.info.ngpus; } -export function numberFormat(num){ +export function numberFormat(num) { const formatter = new Intl.NumberFormat("en-US", { notation: "compact", compactDisplay: "long", @@ -209,12 +221,35 @@ export function numberFormat(num){ return formatter.format(num); } -export function currencyFormat(cost){ +export function currencyFormat(cost) { const scientificFormater = new Intl.NumberFormat("en-US", { - style: 'currency', - currency: 'USD', + style: "currency", + currency: "USD", notation: "compact", compactDisplay: "short", }); return scientificFormater.format(cost); -} \ No newline at end of file +} + +export function fetchingURLErrors(type, response, validationErrors) { + switch (type) { + case "schemaValidationErrors": + return { + msg: `invalid data format from url: ${response?.url}`, + invalidFields: validationErrors?.errors.map((err) => ({ + field: err.instancePath, + err: err.message, + })), + }; + case "noJsonResponseFromUrl": + return { + msg: `no json data from url: ${response?.url}`, + code: `status: ${response?.statusText} | code: ${response?.status}`, + }; + default: // url is not accesible + return { + msg: `error reading from url: ${response?.url}`, + code: `status: ${response?.statusText} | code: ${response?.status}`, + }; + } +} From 234958890516a7b86f87be64341dda4a6c4b292d Mon Sep 17 00:00:00 2001 From: John Calderon Date: Wed, 29 Mar 2023 12:28:37 -0400 Subject: [PATCH 16/16] changed utils.js file --- skyline-vscode/react-ui/src/utils/parsers.js | 14 ++++++++++---- skyline-vscode/react-ui/src/utils/utils.js | 17 +++++------------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/skyline-vscode/react-ui/src/utils/parsers.js b/skyline-vscode/react-ui/src/utils/parsers.js index f22b27a..035d48a 100644 --- a/skyline-vscode/react-ui/src/utils/parsers.js +++ b/skyline-vscode/react-ui/src/utils/parsers.js @@ -4,7 +4,7 @@ import { CENTML_CLOUD_PROVIDERS_URL, deploymentScatterGraphColorSize, } from "../data/properties"; -import {fetchingURLErrors} from '../utils/utils'; +import {getErrMsgFromInvalidURL} from '../utils/utils'; import { cloudProviderSchema } from "../schema/CloudProvidersSchema"; const ajv = new Ajv({ allErrors: true }); // to report all validation errors (rather than failing on the first errors) @@ -67,13 +67,19 @@ export const loadJsonFiles = async (habitatData, additionalProviders) => { } } } else { - errors.push(fetchingURLErrors("schemaValidationErrors",resp,validate)); + errors.push({ + msg: `invalid data format from url: ${resp.url}`, + invalidFields: validate.errors.map((err) => ({ + field: err.instancePath, + err: err.message, + })), + }); } } catch (error) { - errors.push(fetchingURLErrors("noJsonResponseFromUrl",resp,null)); + errors.push(getErrMsgFromInvalidURL("noJsonResponseFromUrl",resp)); } } else { - errors.push(fetchingURLErrors(null,resp,null)); + errors.push(getErrMsgFromInvalidURL("invalidUrl",resp)); } } return { diff --git a/skyline-vscode/react-ui/src/utils/utils.js b/skyline-vscode/react-ui/src/utils/utils.js index 89d63f2..a64443e 100644 --- a/skyline-vscode/react-ui/src/utils/utils.js +++ b/skyline-vscode/react-ui/src/utils/utils.js @@ -231,25 +231,18 @@ export function currencyFormat(cost) { return scientificFormater.format(cost); } -export function fetchingURLErrors(type, response, validationErrors) { +export function getErrMsgFromInvalidURL(type, response) { + const code = `status: ${response?.statusText} | code: ${response?.status}`; switch (type) { - case "schemaValidationErrors": - return { - msg: `invalid data format from url: ${response?.url}`, - invalidFields: validationErrors?.errors.map((err) => ({ - field: err.instancePath, - err: err.message, - })), - }; case "noJsonResponseFromUrl": return { msg: `no json data from url: ${response?.url}`, - code: `status: ${response?.statusText} | code: ${response?.status}`, + code }; default: // url is not accesible return { - msg: `error reading from url: ${response?.url}`, - code: `status: ${response?.statusText} | code: ${response?.status}`, + msg: `url is not accesible: ${response?.url}`, + code }; } }