From efdca68f7ee0403c353ceec284ec31d91793f845 Mon Sep 17 00:00:00 2001 From: Qinzheng Sun Date: Mon, 1 Apr 2019 17:27:36 +0800 Subject: [PATCH] [VS Code] version 0.2.0 (#2442) * add installation guide for VS code extension (#2223) * add installation guide for VS code extension * [VS Code] view container (#2301) * add a dashboard in grafana to list all tasks in node (#2197) * Fix format in issue templates (#2233) Fix format in issue templates: - remove trailing spaces - change chinese colon into english * Fix auto retries when out of memory. (#1108) * Distinguish cgroup OOM from dmesg. * Remove cgroup OOM detection Make all OOM cause exiting by 5 * Exit 55 when OOM * Refine homepage for new users (#2155) Updated first level bullets, to add more content for administrators and users, who is first time touch OpenPAI, or computing platform. * Fix yarn container failed when docker container exited quickly. (#2256) * REST server: remove expires in JWT payload of unit test (#2263) * Deploy: add explicit config field in webportal plugin (#2251) * Deploy: add explicit config field in webportal plugin * Fix json.dumps * t * fix * Update PLUGINS.md * Update webportal.md * alert on unhealthy gpu (#2209) * Pylon: fix double start query in yarn redirect (#2258) * Pylon: fix double start query in yarn redirect * Hide debug info in docker-compose.yaml * adapt user transfer script to new config (#2266) * Webportal: add pai-version attribute to (#2245) * Webportal: add pai-version attribute to * Use preprocess to apply window.PAI_VERSION * set version in layout.html * Fix ib drivers bug (#2269) * FIx ib installation script bug (#2271) * [BUG] Fix hadoop ai build path (#2262) * fix hadoop ai build bugs * refine * Web portal submit job: support init json from sessionStorage. (#2253) * YARN and HDFS log persistence (#2244) * rm log persist * change log dir to host * persist nm log to host * resolve conflict * persist namenode log * persist data node log * add comments * move log path to common pai storage * use twisted in yarn-exporter (#2273) * [Job Debugging] Basic Implement Of Job Debugging. (#2272) * Refine document for new user to submit job (#2278) 1. add new guidance to submit job for beginners. 2. refine homepage to connect with new guidance. 3. reorganize content of troubleshooting for next refactoring. 4. fix links in md files. * [Drivers] Fix the issue when installing IB drivers. (#2275) * fix can not report zombie process using gpu error (#2279) * fix external process error * add debug log * fix short ID and long ID do not match * use time based atomic ref to exchange info between threads * add test case for AtomicRef * fix bug in file remove (#2288) * fix hadoop build error (#2296) * export vc/node related metrics from yarn (#2289) * 720 * open hdfs explorer in view container enable tslint rule "ordered-imports" * add tslint rule for indent * add home button to hdfs explorer's navigation; adjust octicon's color * fix lint error * [VS Code] Add job list (#2160) * add job list view to pai extension * [VS Code] joblist fix (#2185) * eager load recent jobs when job submitted * avoid eager getChildren, and let vscode treeview.reveal do it implicitly * fix lint error * [VS Code] default to generate jsonc job config file (#2368) * 720 * open hdfs explorer in view container enable tslint rule "ordered-imports" * add tslint rule for indent * add home button to hdfs explorer's navigation; adjust octicon's color * fix lint error * [VS Code] Add job list (#2160) * add job list view to pai extension * [VS Code] joblist fix (#2185) * eager load recent jobs when job submitted * avoid eager getChildren, and let vscode treeview.reveal do it implicitly * default to generate jsonc job config file * [VS Code] Refine error messages; Fix Cluster Explorer's bug * [VS Code] changelog and readme (#2429) * 539 * 712 * 452 * [VS Code] v0.11 compatible issue (#2457) * 536 * 600 * [VS Code] fix cluster explorer's right-click menu (#2463) --- contrib/pai_vscode/CHANGELOG.md | 11 +- contrib/pai_vscode/README.md | 21 +- contrib/pai_vscode/VSCodeExt.md | 25 +- contrib/pai_vscode/assets/job-list.png | Bin 0 -> 9747 bytes contrib/pai_vscode/i18n/common.json | 24 +- contrib/pai_vscode/icons/ellipsis.svg | 24 ++ contrib/pai_vscode/icons/error.svg | 1 + contrib/pai_vscode/icons/history.svg | 31 ++ contrib/pai_vscode/icons/latest.svg | 1 + contrib/pai_vscode/icons/loading.svg | 31 ++ contrib/pai_vscode/icons/loading_dark.svg | 31 ++ contrib/pai_vscode/icons/octicon/file.svg | 2 +- .../pai_vscode/icons/octicon/file_dark.svg | 2 +- contrib/pai_vscode/icons/octicon/home.svg | 1 + .../pai_vscode/icons/octicon/home_dark.svg | 1 + contrib/pai_vscode/icons/octicon/terminal.svg | 2 +- .../icons/octicon/terminal_dark.svg | 2 +- contrib/pai_vscode/icons/ok.svg | 1 + contrib/pai_vscode/icons/pai_container.png | Bin 0 -> 1837 bytes contrib/pai_vscode/icons/queue.svg | 1 + contrib/pai_vscode/icons/run.svg | 1 + contrib/pai_vscode/icons/stop.svg | 1 + contrib/pai_vscode/package.json | 152 +++++++- contrib/pai_vscode/package.nls.json | 24 +- contrib/pai_vscode/package.nls.zh-cn.json | 24 +- .../schemas/pai_job_config.schema.json | 2 +- contrib/pai_vscode/src/common/constants.ts | 39 +- contrib/pai_vscode/src/common/singleton.ts | 2 +- .../pai_vscode/src/common/treeViewHelper.ts | 47 +++ contrib/pai_vscode/src/extension.ts | 1 + contrib/pai_vscode/src/pai/clusterManager.ts | 64 ++-- .../src/pai/configurationTreeDataProvider.ts | 186 ++++----- .../src/pai/container/hdfsTreeView.ts | 218 +++++++++++ .../src/pai/container/jobListTreeView.ts | 360 ++++++++++++++++++ contrib/pai_vscode/src/pai/hdfs.ts | 268 +++++++------ contrib/pai_vscode/src/pai/paiInterface.ts | 13 + contrib/pai_vscode/src/pai/paiJobManager.ts | 73 ++-- contrib/pai_vscode/src/pai/paiUri.ts | 70 ++++ contrib/pai_vscode/src/pai/paiWebpages.ts | 30 +- .../pai_vscode/src/pai/recentJobManager.ts | 84 ++++ contrib/pai_vscode/src/root.ts | 10 +- .../src/test/clusterConfiguration.test.ts | 1 + contrib/pai_vscode/tslint.json | 13 +- 43 files changed, 1557 insertions(+), 338 deletions(-) create mode 100644 contrib/pai_vscode/assets/job-list.png create mode 100644 contrib/pai_vscode/icons/ellipsis.svg create mode 100644 contrib/pai_vscode/icons/error.svg create mode 100644 contrib/pai_vscode/icons/history.svg create mode 100644 contrib/pai_vscode/icons/latest.svg create mode 100644 contrib/pai_vscode/icons/loading.svg create mode 100644 contrib/pai_vscode/icons/loading_dark.svg create mode 100644 contrib/pai_vscode/icons/octicon/home.svg create mode 100644 contrib/pai_vscode/icons/octicon/home_dark.svg create mode 100644 contrib/pai_vscode/icons/ok.svg create mode 100644 contrib/pai_vscode/icons/pai_container.png create mode 100644 contrib/pai_vscode/icons/queue.svg create mode 100644 contrib/pai_vscode/icons/run.svg create mode 100644 contrib/pai_vscode/icons/stop.svg create mode 100644 contrib/pai_vscode/src/common/treeViewHelper.ts create mode 100644 contrib/pai_vscode/src/pai/container/hdfsTreeView.ts create mode 100644 contrib/pai_vscode/src/pai/container/jobListTreeView.ts create mode 100644 contrib/pai_vscode/src/pai/paiUri.ts create mode 100644 contrib/pai_vscode/src/pai/recentJobManager.ts diff --git a/contrib/pai_vscode/CHANGELOG.md b/contrib/pai_vscode/CHANGELOG.md index 7a1d5e20e1..2c90cb1dc7 100644 --- a/contrib/pai_vscode/CHANGELOG.md +++ b/contrib/pai_vscode/CHANGELOG.md @@ -4,8 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.0.1] - 2018-01-09 +## [0.1.0] - 2019-01 ### Added - Open job pages and dashboard pages in VS Code - Submit job to PAI cluster from VS Code - Open PAI's hdfs as a workspace folder + +## [0.2.0] - 2019-03 +### Added +- Generate jsonc job config by default +- Add a PAI view container (sidebar), includes + - Job list view + - Auto refresh enabled + - HDFS explorer + - You can choose where hdfs explorer will be shown (view container or workspace folder) diff --git a/contrib/pai_vscode/README.md b/contrib/pai_vscode/README.md index 7ae088bc48..f224328a4d 100644 --- a/contrib/pai_vscode/README.md +++ b/contrib/pai_vscode/README.md @@ -41,6 +41,21 @@ The extension has a useful feature called "Simulate Job Running". It enables the ![](https://raw.githubusercontent.com/Microsoft/pai/master/contrib/pai_vscode/assets/simulate-job.gif) +## Sidebar +HDFS Explorer and Job List view will be shown in the extension's view container (sidebar). + +### HDFS Explorer +1. You are able to connect to an OpenPAI cluster's HDFS by double click its corresponding tree node. +2. The HDFS directory structure will be expended in the tree view. +3. Right click the folder or file node to perform file system operations. + +### Job List +1. Browse latest jobs in the OpenPAI cluster in job list tree view. +2. Browse recent submitted jobs by local (extension) in job list tree view. +3. Job list will auto refresh every 10 seconds. +4. Double click job node to browse Job details in external browser. + +![](https://raw.githubusercontent.com/Microsoft/pai/master/contrib/pai_vscode/assets/job-list.png) ## Commands ### Command Pallete @@ -61,7 +76,7 @@ The extension has a useful feature called "Simulate Job Running". It enables the |Submit Job...|Submit job to selected cluster| |Simulate Job Running...|Generate Dockerfile to simulate PAI job running| |Edit Configuration...|Edit cluster configuration| -|Open HDFS...|Open selected cluster's HDFS as a VS Code workspace folder| +|Open HDFS...|Open HDFS explorer of selected cluster| ## Settings |ID|Description| @@ -70,6 +85,10 @@ The extension has a useful feature called "Simulate Job Running". It enables the |pai.job.upload.exclude|Glob pattern for excluding files and folders| |pai.job.upload.include|Glob pattern for including files and folders| |pai.job.generateJobName.enabled|Controls whether the extension will add a random suffix to your job name when submitting job| +|pai.job.jobList.recentJobsLength|Controls the number of recently submitted jobs to keep in history for each PAI cluster| +|pai.job.jobList.allJobsPageSize|Controls the page size of list when listing jobs for each PAI cluster| +|pai.job.jobList.refreshInterval|Controls the refresh interval of job list (in seconds)| +|pai.hdfs.location|Location where hdfs explorer will be shown| ## Requirements PAI Cluster Version >= 0.8.0 diff --git a/contrib/pai_vscode/VSCodeExt.md b/contrib/pai_vscode/VSCodeExt.md index 2f05d80622..eabbe9267f 100644 --- a/contrib/pai_vscode/VSCodeExt.md +++ b/contrib/pai_vscode/VSCodeExt.md @@ -1,23 +1,22 @@ # OpenPAI VS Code Client -OpenPAI VS Code Client can submit AI jobs, simulate job running locally, manage HDFS files, and etc. It's an extension of Visual Studio Code. - -Visual Studio Code is a popular, free, lightweight but powerful source code editor which runs on your desktop and is available for Windows, macOS and Linux. - ## Installation -1. Download and install [Visual Studio Code](https://code.visualstudio.com/) by several clicks. +Visual Studio Code is a lightweight but powerful source code editor which runs on your desktop and is available for Windows, macOS and Linux. + +More information please refer to the [Visual Studio Code Official Site](https://code.visualstudio.com/). -2. Install **OpenPAI Client**. +OpenPAI Client is a VS Code extension to connect PAI clusters, submit AI jobs, and manage files on HDFS, etc. You need to install the extension in VS code before using it. - 1. Launch VS Code. - 2. Click the "Extensions" icon in Activity Bar or press **Ctrl+Shift+X** to bring up the Extensions view. - 3. Input **openpai** in the text box, the OpenPAI VS Code Client will appear in the result list. - 4. Click the **Install** button. The extension will be installed. - 5. After a successful installation, you will see an introduction page. Follow the instructions there and try the PAI client. +To install the OpenPAI Client: +1. Launch VS Code. +2. Click the "Extensions" icon in Activity Bar or press **Ctrl+Shift+X** to bring up the Extensions view. +3. Input **openpai** in the text box, the OpenPAI VS Code Client will appear in the result list. +4. Click the **Install** button. The extension will be installed. +5. After a successful installation, you will see an introduction page. Follow the instructions there and try the PAI client. - ![Extension](./assets/ext-install-1.png) +![Extension](./assets/ext-install-1.png) -## Next step +## Next steps Learn how to [use OpenPAI VS Code Client](./README.md) diff --git a/contrib/pai_vscode/assets/job-list.png b/contrib/pai_vscode/assets/job-list.png new file mode 100644 index 0000000000000000000000000000000000000000..19c7d43c9cceffe056a6200861796b2e8ad60673 GIT binary patch literal 9747 zcmeHtXIN9)+HKetwt}EkrHTkBMHEGl76Iu^={0PcNEf7&1PfB6L{va(=%EJ@g0!Fz zL_mrlgdPdf2{Aw*1Onf}v(LTfJokIPd;Z*i_eV%FGv{2BIp24TcZ{{-?i%ZyI>~nu z1OlDX)4gK~0x_upuN!|J2U_Y194&!2rbnha8lb8^fhFL}QFnDCbr9$sf@RO?81VhX zLtUFkAP`&o;ftx$uf!Du5-ZWWqi!AwA!Wm=Erwy+8|RlCdYZ`srie zdonM*c$miA$GA_i*N#xsZS1 zL1Bb&CS7giai?a{0!+ynx@ zh=4$_SD)IWW1eB}ru&`7F@qR9z@M=vG|+kFadE6D;@ol2fP^{}^inXE9aM9K!gRa! z=Ro`MXr?2~&qYL&jOWvUzPXMUDA^HfVKdcCN7`RH>4>?M{cgSHO8Yhwo2L`4$*!uR zLcwu5ear7~`jPu-Cbl42eAeu|8fcOCfq4imSF~*)uf44(4I)&94fnOK_jL%JPj_!) zY#-<-ZjB~gEivBSJV!B7UXkhwbnz66oO1|?QeJClikCX*pRC&3xYC%bYnI?V8)zic z`?}kzWhD9+>Fe&+Q{OQE2<@`zB!8DS83R9=f#yLdtZZe~=6XETV8dD!)R1>?+9}0s z;?|=oRui=yR}tlYQG^Pgl9F~zJK}7XwD6DwPGZT2fK^R*e@ipe$OvBweNq^;cUj+YV^Cem`U=fT3;U9UYAc#zBM#CYIT+S_tP5v=Y=n6XF;vruU6%gK+Qp0(%2 z2ntoT?ni3`H_oFn1$gZrI*D|dtL88y)nr@lYeOM&luMFg=Mnw&JJzmP<1XE6(PUxDy0!iK{R+hh{#^YFpqe z;b|eQak8n97UK1Lp{1efxOM##UuHMTJ3bTNW1N$B7# zLMR<+T&T{}H0Rb>zs}HPm0&Vi&o8(86 zU)f3uQJBoNp(*!rd>>ReG~HwsQ2g+roxgMO^XI3>SXd}VNcjC;@+N6Jlb`yE0~v>l z*MED5E0g2J-JC(MxsIJz$^GTx;$HYiGj3^toR0QssW2dQ{<1!6P3hX(IWX5HHLg_2 zR_%oCoDxvEm>{TJMKdPMBEUk&*-Lf(?(~PNFtmYmlTa#BogYoA4yP^?Z3*)DuQFA; zdXF=V-#+vVU43VP%$kSTPEw49XMrdfSokL2gcw$h$6$B6hYX9htg(Z;@$FO z6?hfEo(Wq=iANs8pZrt^T>QrU$$@*Vd83Nh)59_Sf?KMsO9IJpoK+=%O``5FBNZ0M zbBx`BLh5Nkj&4yX0(rU0R}#zj&-@Pt`g}>k#F1ZqdIkc~FaBFQ+aY=+XVZ=+ofA${ z^D>s}f3zI-{`C@Q;Gjz%8iU~D{VWrCCE=hx(FwO2*7deMiI1#5IbICgDLl5 z2;HIJvHZq0`~{PsOWTK{{wT_}_qc%;ywkp>#O~|MWI?Ct0|)*!xRccsv1xJLpgmp| zdp0MWE-^KDur%)%W0ZOGxr`l#%S^^@nq3H9qRT5;t^~C~3>V1x=b#r37_071%VX|5 zOZ9{>9fRfVOJE55<9+>BZi2^9F^`Fl@mvS1B1SZ!iCRi9=1ke7Y(849p~ubG*KHff{JUN(Cx{{c>89%-J-m4c&2FXUUqkcFM}@RJQyHtmq$NX z;5`@;EAjNtuYk9YXss-+99!()G@uyesL6Owet73g5!|LmgzKZ@6K59D_I$oUgI=yU z3FijgK)8+RYkSkdyBSfYHzmy+Acik2Z1q#iOnhu0%~=|>xm<)Z$_4VU2*O> zD(W`$;rVZ9T*zk|_vl`9^PpIZHcsJZIU*`)XGIECPvS-X%pKBv9aY?@#uG1_RIV*8 z-n+ZvlR>{%P+VrEQ6Z-Kgmzwmo=7Y1RV>f^#dn7Aa*3OgeW0raXOEekXViYc;EgFK z^!l`2f(3iyP29>HCgSzt`s-0j=4s2kuX6ORLX3Y;LF!G^R&-u|)k5T4nC8{CuPw>k z8y8VS6pCC+hjKX@+{m#ud}Sj~>3FBsKj?f~_@_1xkza*0(lVGC=BOBm`*6XB_H{*a zJt7Pl+aIq*ED3%uqUI*=eC@Jhf#U0Q;bD(y#6=6#`i;bmgb(}X0>o!W;#9Puju4LH z$c>g4o1cRX!1UGfHd>RTZv)NyjT@@*ow@W>!De`e1XuYARFWUrEjhAG#s6_e_#>p| zG$B_H|6CKXbVYS`aVgCL7ouebR_19intgL2uQ9uhCm!{>gyj9M04IFAi4f#s zt)r8iCV2z|VH--@PS%F}F#aEb6T zLa(z>+HyKbr!>Lba@U|>*EoS5MYH`_ipsWBd3P(t>a)MgXRfDB`+e?Bg{Ymr-pux2 zNlC8hT&3(a>CfuVb>rLzHy9ava~?sY$FZ`4W-P;bFi~n)Ehlx1rEg&030Y9Qf8C;D zi`j0gB+C_5T6B5;slk;s9xL`(^f9Ra`cL9-PI`xTT}2| zYkgN zXk==g5S{fqvy|BgvoKm^rOB~M++w(39aEv$F|kk#Jve8nNh-RyeA$k92HbiJ-$!~U zEIGXnLq@bS$zt`)in-WlYeNDHox80u&RW(@!9E{;wXw8Eo!~rqt6W~%yJt(h%$BxsBz z)ih$`Z;7T}hG1&oQ|4+y7MD$~VFb{x`?rH^AaztueJh8I@nEXYPUEmbT<`KWMbgpW zxW>-7LIqiJ#pJRV^;4&vck^mbIFVE=2e*?VY!~tQt6r-c5xGLfccsqyRLI$`ono$~ z8G~mNsv83IJqNwap^i`+Tqxt76*Riv^)vI7j&(1C3 z!g;E?lOWib3r9NK%g2=jc9 zi@dTygAQ~0Ne#Q@>r;EX6VA$)>9m^C=(VeOhwsap0*+o|1?%tw{SHZ!U9WVIPXucH z&X`gm<~2DD<5LHlO68w>%uPruHd+yHo{lA>R`9$9O?`d6PW@|ysCy~wx@LK`WAEK# z+>sGiY;KBAf2w^wnv|~s`z5ZW>CuUVnqEs}HEp34ZP|-T!NXgnH9-UJT=grv#QHpxceZD8Z*SsswR?ZgRrnt5 z%2*hMG~FU$n5B4ab@ZL+;X+F{oiehzgB68&Q4>kzAsMsZmnzimcm4uZO=6fn`h^? zsFvr{oiTb`O^|LP4GPP!!V*6BKBei0E@X$CY#^2D0^szP7Q8l@F3n%SZZBLzxVG1q zn|*!y8Y}66T7MPga?5n~T`Eg2Dp*6)ZNWam2w&sO<>kguFSKSzVBF?*zU7TmP~9)y zhkMHG2Ad;CI>|JDo8B}ZaCuKEtkoGhfl0#35HR%OAdFl5HuJk`(pePC@I7&?Ei87{ zOQzGBz$5E8QhBOqMf@3|xC~|M06;yP7i8B#=R^+Q!naKsU!)Xb85&Sm2ug~))i zm3L+i9fjSW7mp-%xGQJDax>_8&0t+OuKo zV;{!7tdVcaN$h`g>@;?#%GZU<^lxiX%ivUX*!s2TO1tLl@9r{ow=^l;+d(H=``Wf` z7z3D*sb$61iY%R=H*qF!hPgZOw#IpHX&W3EqXb(jjxyedfz)AQmNCt=DqkQv)BKL& ztlAQ6-oM~KyO8zBc$5}EpO1Dnbz{mUKc#tAyDklCJmj0&LBGFb6jmw+8mL!?23;+` z&?3Jw!tH3!lVN2m$pfOR{hvkMZ%OzMX7Umdbm@$?OO-xh0^ao=-I~vIaE$!cJopCn z*4zd(8gd@XbItG5Q;h(!#l?EYxKz8B1mZ+ClJROiljt70PW#`?|*LnXWBTN&}yyYF;f3V(zjr^v?{JFqON((j7YR|;yjk9=zn z*ER9z9%?>XDwnnO$ExsY$JdLEQxOxG=Y6g8p=!4upJoc!)&3NYq^XAn(Q>aWc|bgb zTMpo(0i%THefSq;V(A2?9yw88nr{Fep;PlcpNPO%c@f7n_Y*~ zwt*wQf`yGQ)CL}S-%!mo?7#G$W@eoEoiygY-|zs`@Qjvm2frLR)E33lV-opp6F2`1 zZ1=`!*aIi3mN?n`QQFZoL0CodEQP619cpwTtRo~~g@ZtVHi}S+E*6J%m~#pckLVuG zf-0x?LG;6whPmnSbKB!M{~}c51u-?{phOcVhD(c4U@ZIjQ-}Xl?kJo(r&_c<6XUe< zrQqZZG*}i4>uN~OYDP0$uN7X5tjy9J&UIHVEcEp{s}tmra=u<*xf zkAPZD|%+ zoqn&pw{9gHWq~+Q{OY(zy1GeA5}ISmMLeT5USHdcbLvbIbn3KIZu@F*KPE9dvkfpc zxT+fFGp+_Le}C9^LLw^tx_k677M`e}m67MXkUa}Uih-ly$T1L%3x@~nkzHTf{&uUd zB^n|)9Xgw+5y0!PS@$d6aZk>BV(b2=R2j4-uKsEQpS zMxE1iFH6K%1p}#-KNQb@(Sw@xD=Vk3?W{#k3G>Le{u!VIP9`6T6gLF6O%(1lTFgY??(B;-#>c7jn3Hq#MKp6rAkk2+wGs3jt>c zwKmt=wn!7N!*b%$>6|&?%cUCko%d*znKAdsaj`~gs54;v%zWneV+BKYU2BXHt|2`& zb}aWH0Z#QQIOb}^7j@`RKUl$gvbOXJB~j7Cd$i8WN~9U^G|sP1Q+J2#lQ{_e@U4xj zVe3DCEss=oDXG57zUXR(#3#@$nkQZ$r9-h5!(!3|1FN9QJ-l-K=67ALACG5m^Agf| zzp`d$D)0`L>p-{ea8HfZg&`7;o{B?1GIKwx9PB;%-rWRP(NEDO-7ED~_7L69^>8V* zyfcN}*4Xaxnqo+>WK;76MI$o?Nb$$5mmMJ^t2Q$!R*vp=&g@yGJi{Y-fiH0_DiSEto{W;AL05CJL8y9)p-wn9FGQ?Kg*g+q~{T-3oE2 zOcBD~yh1wyuaaCb6npIF9M^(^Bv#QfAZ6f&(QwfXr+2{_2_O-cfQE*9vajqsYb8vH z@0%L?!3WR}TXqE9k;~q$Zun6Z@l0*pf=(EGdVS>_rQ_9XEpH7IC8!gzW*WNWvo!uA#w@c6bwSx==DCjck&t%GU?qn>_KR;mv)0@Iwz1694RPo1c z=v49~F*zr%nV80p4m_CqJc4&*nQh%fbkSL5FCbi975Lsb8bI~&m#(FjfE~E8GQD1# zD7o$p4rwXohaaEP8fSGvE+>KaX+(E#7TznYULzGV^^G9L8Jrb3T&_o%+{_8Dn<@IW zxNPAxEyEI~#Bw|G=)Y;B-?Sr1l{~rSg=R%(tB41}w@fq^ z91p@qZPN+x_Q_up^;>GQChtm>Kz|V)(`+B>`Qh8am+sV4*ws>8HwgP$ql{6}FYJGT ze*?nDA-nnuQLP%#l(QXW3EjoZH%<%Rj<^jq z%2&pqO_VFW0`W%hly6Aw$MlUsPP7Tq9w3>^7o{V#rw1o2qs|m)^DJ@X6&WP(fGL|p zZQV_4i!Z=)wR#=joh^`e9#BBH{sX462lS!{IG#I!1{wrbnA}WMZU~Aa`>9R6q?vLI zH9MD`VEda|hAd$2vSO;)%O@Ef>k&T04})e7$sEY+#?Po~KI&L$W!yh|@^ej~vCXJW zwY{s}f??)Q=BW#FDA`*I0gp5}G9x#PRE^zT_B>?&WXEChwHVH@f zF)3C4sq{THI^hNFxR8L-!8)R?DLa#IUnzdxCu(6<1rq!>Gw%Me(Ra{{HVClOVJE+W zr0r?5v!I3KbKerEU!G~D+dBS*QLfvss&ZX42@;c!%#do9PC?h&wjKR)0*C)&nbDQ8 zzG}t4=X&a|10c+zy3iAvC)GAOpRe9P$8cR@Ut$87$&uw-X?fMeQt|V(QT+f7ATWhm z`;Uzt96D*g9e{r+uKr12vFMIH#!O61`F!6guzg(_jV~T>ulL~}AG@9djn)G-)_U*G zMtjnfUo>|=RM~!iAmQQiR67e0Sfr+U?rWtrplUKeqR!M=L`ob{UM?hPr03rNlWF2y&#M*Z^-?kH^k$E9S)NkFMukRQp4u*a)vFf! zijn2Dz_WrTy?pWP#^EJ8w<>nUH(zGY6rox>Os1?0kwJui)MspqELR#nc-`O>9>8o! z^&jBzOqbO!!2!YBp6^Du>`eS1=X_89wElZH*{m9s-g4nbt^S(q_bxdyxTq&flHnc= z<2MI`&6{|vWU>8N;W8_#oics0!co!%Sp;Ll_Lp_y-F&`_`jtrKRB8m8Xm%q_DO=RukGh?iXqT%O#$#8>M1}u|_fz?x?deQV?W} zj9qgI07HrMISG zZNeqg)19bpDI?|xn;dc)B7Vq-uP$~%(#visqdt&OjNpB-@$NFz4i51tg~&=yT*)x$ z>Pow(3Ku->xC@l3`9s6+$>5@wV;EI5EWhFOH0&VTWIkNqq{L7a)d+#^e;j_|T)^F~ z9o8Ms?i`srqA7q5YxOSR;WVsMA-G50-j(w-cxQ=C(B8H1;kRj?**&*#VtApxl&q7* ziz@nQ?qi1wOSMcipSsH*M}CAnfVQ8 zMWNiCRsu5 z-p*fV=ns=R&TgCiy@B7XOd*7Zx|cao^K({4;V&*@`%Kq4Z<4<3Ds$mN6Td|u>5t;ePvR7ADPQn9cTnsPxW5Kjj z%Cx|b!;fClj!gU_buizrQ>iO*^DkU0uD)aibOvX}LHOYBG)EJ%iK^Uv%GyC1a zVQLJs1+}4CF)SOG@7icXNyjL4e==B&z&gs*H7C8X6LXZ;bt*4M4Sm0qk^vx#)%FiR z{fR7WdbJ`Kty*|n^bCL4*~FyK@h5lAY)c7!s>=+$c&WBx(;RxTgKoAu{Q}Jmdje!# z$6{l)ukB(|CAR7U07W&(67>TujiIGV-Z$VNawRPn_J0ArU#aqsvFt=I5qf}}pUsiQ zT>$Am*^Pkcho0R7uFk(d>V@{$hUFC7R&Ck+C1uq=WD`DwmLn3R@w*XjcBt@rcjaFz z(#x>MQ>({-^aEBzchC=xz5@&g21=zrBFT3&dO=noq=sj|mCM@bdTQNrScVlC_&s8} zMw|=fGsz#fa1sztFo);$6&C$wVEKN#k@1aGd8dg{Kf6oJz{Ed0>_*BKcHlc76sHRB z#eb-*!K74$8EpH|L-Q1cGu!It7KT%osrfae{IG+D9jH`6_@2^@3s~#?yTXSM)!1z} z;)$T$pfcCW7GSu%=6W8~Iu2u!JNU6k|D3xpcqSc=3-iwykMFSh9s8*o{`M0qf`?`f?73 zzYyBP7kcsL>&!TGPNZVBAHYv{3KXYnMj|f0_~1zSvuX+_s-EXogd&<<8bLY(NmIGV z0%%~F==zE|o#pRC$VQ-TKV*(Na@s)-v&4kQT&4Rhv4# z^%RiW3NnF=D`Lu;`M_{sU`kfRr=8z1G!XlXi6Wo)ZxOURU2}h8bmWxhA(1*-&B%BS zZ>;A2l<^`(^kj_JHRh=So`o(1m?!){!`W?QW`Dh0StnXkm1yR0v#ny-Z}9{Wo7q)B z+>VQ?88MhhJtZ<7wmWisB&#|fS4uS5JxjtINhrn&`$Mvdx11>!6FmMoMJaihvP#}V zV-Pfw35FSL5GhCPg8wOq5@SuQ!KPsYoPYxYj1A67%VIO;j81(%4{%ZqrU#Qe zc=0bg%X0*jEe&46POtD5^d8&R6s_^3WMn9k9wlil-)kgCjD>6P0Y^cUy=vgy?B?T_ zGd@^TR~ndje^0#(WB5nigJQ=vOuP?5u?0ee1Kw<+SxAr_HB~;ZvGPg^Wr7pfR@x|C2EK=VK86 zh_w~eDEink!ygRV6T;zj;*k4Jx`Cv9MwzNip{=s`l=%UOegr@$R`TMXCGq{007^$-3^=%?yu6lkisScRy zu3jrey{P4uvgGo}kn}NnQ~cH(rcoftsGb7HwI7VCrjU`1&Zg`6waHm?=|6}lm~~^$ zv9sVhEmBz$tk>;vh@2>8i}W#}js+M>s~O2(cx|78x*;puQ~#~N*}Jf8&Kmjx zy=8e-*lWWd1_Hcnx?=D^#8Rv^lqXFo$lQB*{#jbG&JF~s!Ld%fH)ef5IL(qaY7N!c z;gRCv?7X1%fcJm2SO1p+`1^sI|9zwIzmp07ztjJN&D=DGBH0gwSBRCjQDAlp;6W~s No|f^QDviIM{uhpnpyvPp literal 0 HcmV?d00001 diff --git a/contrib/pai_vscode/i18n/common.json b/contrib/pai_vscode/i18n/common.json index f7ad857859..057d4202b4 100644 --- a/contrib/pai_vscode/i18n/common.json +++ b/contrib/pai_vscode/i18n/common.json @@ -23,10 +23,18 @@ "treeview.node.edit": "Edit Configuration...", "treeview.node.openhdfs": "Open HDFS...", "treeview.node.openPortal": "Open Web Portal...", - "treeview.node.listjob": "List Jobs...", + "treeview.node.listjob": "List Jobs Externally...", "treeview.node.create-config": "Create Job Config...", "treeview.node.submitjob": "Submit Job...", "treeview.node.simulate": "Simulate Job Running...", + "treeview.hdfs.select-cluster.label": "Double click to connect to a PAI cluster's HDFS...", + "treeview.joblist.recent": "Recent Submitted Jobs from VS Code", + "treeview.joblist.all": "All Jobs", + "treeview.joblist.view": "View Job Detail", + "treeview.joblist.more": "View More...", + "treeview.joblist.error": "Failed to load job list: {0}", + "container.hdfs.mkdir.prompt": "Please enter a folder name", + "container.hdfs.mkdir.cancelled": "Cancelled creating new folder", "hdfs.workspace.title": "HDFS Explorer - {0}", "hdfs.progress": "Transfering file - {0}% ({1} bytes / {2} bytes)", "hdfs.downloading": "Downloading {0}", @@ -47,6 +55,7 @@ "job.prepare.status": "PAI: Preparing for job submission", "job.prepare.cluster.cancelled": "No cluster selected, job submission cancelled.", "job.prepare.config.prompt": "Please select a PAI job config json file", + "job.prepare.config.invalid": "Invalid job config json file, job submission cancelled.", "job.prepare.config.cancelled": "No job config selected, job submission cancelled.", "job.prepare.upload.prompt": "Enable auto uploading of code?", "job.prepare.upload.yes.detail": "The extension will upload your project files to PAI job config's code dir automatically.", @@ -57,6 +66,7 @@ "job.upload.status": "PAI: Uploading code", "job.upload.progress": "PAI: Uploading code - {0} / {1}", "job.upload.error": "Error occurred while uploading code: {0}", + "job.upload.invalid-code-dir": "Auto uploading doesn't support code dir with url scheme hdfs:// or webhdfs://. Please use environment variable $PAI_DEFAULT_FS_URI instead.", "job.request.status": "PAI: Submitting job", "job.submission.error": "Error occurred while submitting job: {0}", "job.submission.success": "Successfully submitted job.", @@ -93,10 +103,18 @@ "treeview.node.edit": "编辑配置...", "treeview.node.openhdfs": "打开 HDFS...", "treeview.node.openPortal": "打开 OpenPAI 门户...", - "treeview.node.listjob": "打开任务列表...", + "treeview.node.listjob": "在浏览器里打开任务列表...", "treeview.node.create-config": "创建任务配置文件...", "treeview.node.submitjob": "提交任务...", "treeview.node.simulate": "模拟任务执行...", + "treeview.hdfs.select-cluster.label": "双击以连接到 PAI 集群的 HDFS...", + "treeview.joblist.recent": "近期从 VS Code 提交的任务", + "treeview.joblist.all": "所有任务", + "treeview.joblist.view": "查看任务详情", + "treeview.joblist.more": "显示更多...", + "treeview.joblist.error": "载入任务列表时发生错误：{0}", + "container.hdfs.mkdir.prompt": "请输入文件夹名", + "container.hdfs.mkdir.cancelled": "新建文件夹操作已取消", "hdfs.workspace.title": "HDFS 浏览器 - {0}", "hdfs.progress": "正在传输 - {0}% ({1} 字节 / {2} 字节)", "hdfs.downloading": "正在下载 {0}", @@ -118,6 +136,7 @@ "job.prepare.status": "PAI: 正在准备提交任务", "job.prepare.cluster.cancelled": "未选择集群，任务提交已被取消。", "job.prepare.config.prompt": "请选择一个 PAI 任务配置 JSON", + "job.prepare.config.invalid": "任务配置文件不合法，任务提交已被取消。", "job.prepare.config.cancelled": "未选择任务配置文件，任务提交已被取消。", "job.prepare.upload.prompt": "是否启用代码自动上传功能？", "job.prepare.upload.yes.detail": "插件将会自动上传你的项目文件至 PAI 任务配置中的 code dir", @@ -128,6 +147,7 @@ "job.upload.status": "PAI: 正在上传代码", "job.upload.progress": "PAI: 代码上传 - {0} / {1}", "job.upload.error": "代码上传时发生错误：{0}", + "job.upload.invalid-code-dir": "自动上传不支持 hdfs:// 及 webhdfs:// 形式的 code dir, 请使用环境变量 $PAI_DEFAULT_FS_URI", "job.submission.name-exist": "提交失败，已存在同名任务，是否启用自动生成任务名称后缀功能?", "job.submission.name-exist.enable": "启用并重新提交", "job.submission.error": "提交任务时发生错误：{0}", diff --git a/contrib/pai_vscode/icons/ellipsis.svg b/contrib/pai_vscode/icons/ellipsis.svg new file mode 100644 index 0000000000..9a76b9fab8 --- /dev/null +++ b/contrib/pai_vscode/icons/ellipsis.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/error.svg b/contrib/pai_vscode/icons/error.svg new file mode 100644 index 0000000000..8e08d84186 --- /dev/null +++ b/contrib/pai_vscode/icons/error.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/history.svg b/contrib/pai_vscode/icons/history.svg new file mode 100644 index 0000000000..9ef41c37cb --- /dev/null +++ b/contrib/pai_vscode/icons/history.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/latest.svg b/contrib/pai_vscode/icons/latest.svg new file mode 100644 index 0000000000..3d882c1878 --- /dev/null +++ b/contrib/pai_vscode/icons/latest.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/loading.svg b/contrib/pai_vscode/icons/loading.svg new file mode 100644 index 0000000000..e762f06d5e --- /dev/null +++ b/contrib/pai_vscode/icons/loading.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/loading_dark.svg b/contrib/pai_vscode/icons/loading_dark.svg new file mode 100644 index 0000000000..7dc1ebd8cf --- /dev/null +++ b/contrib/pai_vscode/icons/loading_dark.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/octicon/file.svg b/contrib/pai_vscode/icons/octicon/file.svg index 1bb5df6e03..f4efbc238b 100644 --- a/contrib/pai_vscode/icons/octicon/file.svg +++ b/contrib/pai_vscode/icons/octicon/file.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/file_dark.svg b/contrib/pai_vscode/icons/octicon/file_dark.svg index 4dc8aa60dd..6abbe9f171 100644 --- a/contrib/pai_vscode/icons/octicon/file_dark.svg +++ b/contrib/pai_vscode/icons/octicon/file_dark.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/home.svg b/contrib/pai_vscode/icons/octicon/home.svg new file mode 100644 index 0000000000..8799a322c1 --- /dev/null +++ b/contrib/pai_vscode/icons/octicon/home.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/home_dark.svg b/contrib/pai_vscode/icons/octicon/home_dark.svg new file mode 100644 index 0000000000..1d033a9d85 --- /dev/null +++ b/contrib/pai_vscode/icons/octicon/home_dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/terminal.svg b/contrib/pai_vscode/icons/octicon/terminal.svg index d1aaf41897..b6df312df4 100644 --- a/contrib/pai_vscode/icons/octicon/terminal.svg +++ b/contrib/pai_vscode/icons/octicon/terminal.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/terminal_dark.svg b/contrib/pai_vscode/icons/octicon/terminal_dark.svg index 5fa29ec4ad..fb45d72a62 100644 --- a/contrib/pai_vscode/icons/octicon/terminal_dark.svg +++ b/contrib/pai_vscode/icons/octicon/terminal_dark.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/ok.svg b/contrib/pai_vscode/icons/ok.svg new file mode 100644 index 0000000000..3efeb56727 --- /dev/null +++ b/contrib/pai_vscode/icons/ok.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/pai_container.png b/contrib/pai_vscode/icons/pai_container.png new file mode 100644 index 0000000000000000000000000000000000000000..b071bcc917cd7e77f004f83de17dcbb97cdde351 GIT binary patch literal 1837 zcmV+|2h#Y7P) zaB^>EX>4U6ba`-PAZ2)IW&i+q+U-|SlH?`~{pS=p0zycFm)@|>Om9?d{cUI}V`znE`d^m@y0?*!c%@zLn?@A;|wQ|0Rg-2%p~ zsrF{Cw=nPV>(=-(1ODk;ir>TT!SC;@!FnNBhf7&ZoN;YuafW8xd2mj+8MrCsJd&%Q z&kL@%yp%H1n)$TzC?rz^fe)i^iS!}{11C1Y*EGdx_zU`W zo&8#5WQThMd}rcx>QsQurb>}$(56L!*2$q`Evp#XQoQi>tg)3|{DQiklXfwH9^8Pr z!xfk-;D72u*Sh^$VMAPg-8VQ`RdpXqx8K6CEv@a#-^9i6#B=v_;LWoI6?b}OR_kqmCH*i))pv33<$C! zT1Y8dDMnPeGypW9dEr_n5Fn8fP_k5J1vU+Sgtalp=o(w3Jf4H#fRC`U0ULxRz^YZ? zKUN!jXb4Lc5fzgrT}D#Ys-mi~$iVD@5o63UdDbaYr_DC|91i#K>f-9=&AX2%!2wPQ4T}~%#-asRN~dx^1`J_jsaqqx!G&2T-FMwQ?bK#@Q1!JT*IlE)jHiMh<2`tW*Nxb`Us> zcQDJ1YApx3gIVqcQ6U9N%E2Zmk3!Ko?p4k0^+^2C%u=V$FE1y9w9q4`mawRQ( zhua6J#klepUkIHjJT}dY=);&%8h$YL-T7+!ThP09)7&(6^9tYR!}pPjE$VBM8Lufh zyo9??1$@$xD=ZHmUgun%E3j>Qi7!K{FhV}np_4bLJ?&X|l~!#bww42YvxtBrSxjDG z(`ca{;WMvB=~)?eJ2Q-Zc=lS(P0i`KZMGV)r|vD%Z~kY%N|yjk0BLB z(aIX4iM!9X0*3avp{--0ZbTOkF=q||F(Kpx!MjK!26kTsk%)lvj(rkRnU54L7ZKk+ zqmnBEHQOw5X^DE{Rr_si)3?-pYFjht3@%f53tOe`+;&?VDJcVJpLO(94i|tK&Jv(V z<2g!2iBM35^O&^27Vw@j7qK>8nFG`n!f-8YVrXCR6rIUb=IBHnObUAQ7T_faLZz}p zr!Xp&-E^I;Kum=r!KZdF;M+BzL&VzP6%_1db*u$Ohjmbe9usTFKQZV7B|kbZNNHC{ROvehra{95VB4@bWBCCB4h!Kju^UH=tB4murz_{ z6ABDayPOTHv|guXd<+yMG)E9|+lss+4F&@bBO_2OB#m%Hxz2)wQ3FOPHC1(dw=8O> zQbbL%5s0Xq(D9QYQM|!7L-(k-$=Xnp!Vf2cx+A_FU?W>0>!g@1;M1|e4REYk*$GIs zGpzw1RDFu#{XqqNt88Xa{t^WAoJetgMw)?-s716j0sATUf0iiIFJju?g8nB#1FkD$ zDhdz=BZMoelO{v({p^cKD`=uC3g2i2HQJ;T6uB-&MN(5oA+;3|6y41_$~07UAXMaw z==gpCT@b#&D{3S?pExM(kO{$bdTB7uclc@3OXsWYCj>3ttMG1wz#x>W%|>iwzlG9& zX6uaTHI&D?JDoZ!BTPUGv)cv%BJ!r3TCX6L;JXIIVy%L3YTi??$aADhTPM;S`8{pZ z)d}(!DmSJ%30YdK{%DF!U(d0>1^qXI3^jL0T>TA}KY(~t6V8?Z000SaNLh0L04^f{ z04^f|c%?sf00007bV*G`2jc<^2MPxb)M581uqq+R2g>6biku7%dSX14(P|CT3J(K;|on5RrFD zZ|kU#7#7*P`7N|EAi!|Q&t*SCilqhX8W=kAlG#g`# bF?-wreZy%BUP^}Z00000NkvXXu0mjfo8C@@ literal 0 HcmV?d00001 diff --git a/contrib/pai_vscode/icons/queue.svg b/contrib/pai_vscode/icons/queue.svg new file mode 100644 index 0000000000..a0577b519e --- /dev/null +++ b/contrib/pai_vscode/icons/queue.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/run.svg b/contrib/pai_vscode/icons/run.svg new file mode 100644 index 0000000000..a37ceb2579 --- /dev/null +++ b/contrib/pai_vscode/icons/run.svg @@ -0,0 +1 @@ +ProgressBar_16x \ No newline at end of file diff --git a/contrib/pai_vscode/icons/stop.svg b/contrib/pai_vscode/icons/stop.svg new file mode 100644 index 0000000000..6b6668df9c --- /dev/null +++ b/contrib/pai_vscode/icons/stop.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/package.json b/contrib/pai_vscode/package.json index 5467611c62..99348c0f93 100644 --- a/contrib/pai_vscode/package.json +++ b/contrib/pai_vscode/package.json @@ -2,7 +2,7 @@ "name": "pai-vscode", "displayName": "OpenPAI VS Code Client", "description": "Interact with Open Platform for AI (OpenPAI) from inside your editor", - "version": "0.1.0", + "version": "0.2.0", "publisher": "OpenPAIVSCodeClient", "preview": true, "icon": "assets/pai_logo.png", @@ -85,9 +85,23 @@ "title": "%paiext.hdfs.download%", "category": "PAI" }, + { + "command": "paiext.container.joblist.refresh", + "title": "%paiext.common.refresh%", + "icon": { + "light": "icons/refresh_light.svg", + "dark": "icons/refresh_dark.svg" + }, + "category": "PAI" + }, + { + "command": "paiext.container.joblist.more", + "title": "%paiext.cluster.job.more%", + "category": "PAI" + }, { "command": "paiext.cluster.refresh", - "title": "%paiext.cluster.refresh%", + "title": "%paiext.common.refresh%", "icon": { "light": "icons/refresh_light.svg", "dark": "icons/refresh_dark.svg" @@ -102,14 +116,58 @@ "dark": "icons/add_dark.svg" }, "category": "PAI" + }, + { + "command": "paiext.container.hdfs.refresh", + "title": "%paiext.common.refresh%", + "icon": { + "light": "icons/refresh_light.svg", + "dark": "icons/refresh_dark.svg" + }, + "category": "PAI" + }, + { + "command": "paiext.container.hdfs.back", + "title": "%paiext.container.hdfs.back%", + "icon": { + "light": "icons/octicon/home.svg", + "dark": "icons/octicon/home_dark.svg" + } + }, + { + "command": "paiext.container.hdfs.delete", + "title": "%paiext.container.hdfs.delete%" + }, + { + "command": "paiext.container.hdfs.mkdir", + "title": "%paiext.container.hdfs.mkdir%" } ], + "viewsContainers": { + "activitybar": [ + { + "id": "PAIContainer", + "title": "%container.title%", + "icon": "icons/pai_container.png" + } + ] + }, "views": { "explorer": [ { "id": "PAIExplorer", "name": "%explorer.paiClusterExplorer%" } + ], + "PAIContainer": [ + { + "id": "PAIContainerHDFS", + "name": "%container.hdfs.title%" + }, + { + "id": "PAIContainerJobList", + "name": "%container.joblist.title%" + } ] }, "menus": { @@ -131,7 +189,11 @@ "when": "false" }, { - "command": "paiext.hdfs.open", + "command": "paiext.container.joblist.refresh", + "when": "false" + }, + { + "command": "paiext.container.joblist.more", "when": "false" }, { @@ -145,6 +207,22 @@ { "command": "paiext.hdfs.download", "when": "false" + }, + { + "command": "paiext.container.hdfs.refresh", + "when": "false" + }, + { + "command": "paiext.container.hdfs.back", + "when": "false" + }, + { + "command": "paiext.container.hdfs.delete", + "when": "false" + }, + { + "command": "paiext.container.hdfs.mkdir", + "when": "false" } ], "view/title": [ @@ -157,6 +235,21 @@ "command": "paiext.cluster.add", "when": "view == PAIExplorer", "group": "navigation" + }, + { + "command": "paiext.container.hdfs.refresh", + "when": "view == PAIContainerHDFS", + "group": "navigation" + }, + { + "command": "paiext.container.hdfs.back", + "when": "view == PAIContainerHDFS", + "group": "navigation" + }, + { + "command": "paiext.container.joblist.refresh", + "when": "view == PAIContainerJobList", + "group": "navigation" } ], "explorer/context": [ @@ -207,6 +300,35 @@ { "command": "paiext.cluster.delete", "when": "view == PAIExplorer && viewItem == PAIConfiguration" + }, + { + "command": "paiext.container.hdfs.mkdir", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsFile", + "group": "1@1" + }, + { + "command": "paiext.hdfs.download", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsRoot", + "group": "2@1" + }, + { + "command": "paiext.hdfs.upload.files", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsFile", + "group": "2@2" + }, + { + "command": "paiext.hdfs.upload.folders", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsFile", + "group": "2@3" + }, + { + "command": "paiext.container.hdfs.delete", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsRoot", + "group": "3@1" + }, + { + "command": "paiext.cluster.job.list", + "when": "view == PAIContainerJobList && viewItem && viewItem == PAIJobListCluster" } ] }, @@ -244,6 +366,30 @@ "type": "boolean", "description": "%config.job.generateJobName.enabled%", "default": null + }, + "pai.job.jobList.recentJobsLength": { + "type": "number", + "description": "%config.job.jobList.recentJobsLength%", + "default": 5 + }, + "pai.job.jobList.allJobsPageSize": { + "type": "number", + "description": "%config.job.jobList.allJobsPageSize%", + "default": 20 + }, + "pai.job.jobList.refreshInterval": { + "type": "number", + "description": "%config.job.jobList.refreshInterval%", + "default": 10 + }, + "pai.hdfs.location": { + "type": "string", + "enum": [ + "sidebar", + "explorer" + ], + "default": "sidebar", + "description": "%config.hdfs.location%" } } } diff --git a/contrib/pai_vscode/package.nls.json b/contrib/pai_vscode/package.nls.json index 73cc7308c5..eabb2210c0 100644 --- a/contrib/pai_vscode/package.nls.json +++ b/contrib/pai_vscode/package.nls.json @@ -1,21 +1,33 @@ { "explorer.paiClusterExplorer": "PAI Cluster Explorer", + "container.title": "Open Platform for AI", + "container.hdfs.title": "HDFS Explorer", + "container.joblist.title": "PAI Job List", + "paiext.common.refresh": "Refresh", "paiext.cluster.add": "Add PAI Cluster", "paiext.cluster.edit": "Edit PAI Cluster Configuration", "paiext.cluster.delete": "Delete PAI Cluster Configuration", - "paiext.cluster.refresh": "Refresh", "paiext.hdfs.open": "Open HDFS", - "paiext.hdfs.upload.files": "Upload Files...", - "paiext.hdfs.upload.folders": "Upload Folders...", - "paiext.hdfs.download": "Download...", + "paiext.hdfs.upload.files": "Upload Files", + "paiext.hdfs.upload.folders": "Upload Folders", + "paiext.hdfs.download": "Download", + "paiext.container.hdfs.back": "Go back to cluster selection", + "paiext.container.hdfs.delete": "Delete", + "paiext.container.hdfs.mkdir": "New Folder", "paiext.cluster.dashboard.open": "Open Dashboard", - "paiext.cluster.job.list": "Open Job List", + "paiext.cluster.job.list": "Open Job List Externally", "paiext.cluster.job.submit": "Submit Job to PAI Cluster", "paiext.cluster.job.create-config": "Create PAI Job Config JSON", "paiext.cluster.job.simulate": "Simulate PAI Job Running", + "paiext.cluster.job.view": "View Job Detail", + "paiext.cluster.job.more": "View More...", "config.title": "OpenPAI VS Code Client Settings", "config.job.upload.enabled": "Controls whether the extension will upload your project files to PAI job config's code dir automatically", "config.job.upload.exclude": "Glob pattern for excluding files and folders", "config.job.upload.include": "Glob pattern for including files and folders", - "config.job.generateJobName.enabled": "Controls whether the extension will add a random suffix to your job name when submitting job" + "config.job.generateJobName.enabled": "Controls whether the extension will add a random suffix to your job name when submitting job", + "config.job.jobList.recentJobsLength": "Controls the number of recently submitted jobs to keep in history for each PAI cluster", + "config.job.jobList.allJobsPageSize": "Controls the page size of list when listing jobs for each PAI cluster", + "config.job.jobList.refreshInterval": "Controls the refresh interval of job list (in seconds)", + "config.hdfs.location": "Location where hdfs explorer will be shown" } \ No newline at end of file diff --git a/contrib/pai_vscode/package.nls.zh-cn.json b/contrib/pai_vscode/package.nls.zh-cn.json index 31135fbb12..7a6ec1fb92 100644 --- a/contrib/pai_vscode/package.nls.zh-cn.json +++ b/contrib/pai_vscode/package.nls.zh-cn.json @@ -1,21 +1,33 @@ { "explorer.paiClusterExplorer": "PAI 集群浏览器", + "container.title": "AI 开发平台 (PAI)", + "container.hdfs.title": "HDFS 浏览器", + "container.joblist.title": "PAI 任务列表", + "paiext.common.refresh": "刷新", "paiext.cluster.add": "添加 PAI 集群", "paiext.cluster.edit": "编辑 PAI 集群配置", "paiext.cluster.delete": "删除 PAI 集群配置", - "paiext.cluster.refresh": "刷新", "paiext.hdfs.open": "打开 HDFS", - "paiext.hdfs.upload.files": "上传文件...", - "paiext.hdfs.upload.folders": "上传文件夹...", - "paiext.hdfs.download": "下载...", + "paiext.hdfs.upload.files": "上传文件", + "paiext.hdfs.upload.folders": "上传文件夹", + "paiext.hdfs.download": "下载", + "paiext.container.hdfs.back": "后退至集群选择", + "paiext.container.hdfs.delete": "删除", + "paiext.container.hdfs.mkdir": "新建文件夹", "paiext.cluster.dashboard.open": "打开仪表板", - "paiext.cluster.job.list": "打开任务列表", + "paiext.cluster.job.list": "在浏览器里打开任务列表", "paiext.cluster.job.submit": "在 PAI 集群上提交任务", "paiext.cluster.job.create-config": "创建 PAI 任务配置 JSON 文件", "paiext.cluster.job.simulate": "模拟 PAI 任务执行", + "paiext.cluster.job.view": "查看任务详情", + "paiext.cluster.job.more": "显示更多...", "config.title": "OpenPAI VS Code 客户端设置", "config.job.upload.enabled": "控制插件是否会自动将项目源代码上传至 PAI 任务配置文件的 CodeDir", "config.job.upload.exclude": "控制排除文件、文件夹的 Glob 模式", "config.job.upload.include": "控制包括文件、文件夹的 Glob 模式", - "config.job.generateJobName.enabled": "控制插件是否会在提交任务时，自动在任务名称后添加随机字符串，以避免重复" + "config.job.generateJobName.enabled": "控制插件是否会在提交任务时，自动在任务名称后添加随机字符串，以避免重复", + "config.job.jobList.recentJobsLength": "控制每个 PAI 集群保留最近提交任务的数量", + "config.job.jobList.allJobsPageSize": "控制 PAI 集群任务列表的分页大小", + "config.job.jobList.refreshInterval": "控制任务列表的刷新间隔（秒）", + "config.hdfs.location": "HDFS 浏览器显示的位置" } \ No newline at end of file diff --git a/contrib/pai_vscode/schemas/pai_job_config.schema.json b/contrib/pai_vscode/schemas/pai_job_config.schema.json index 0e6e2a996d..989c9d36e7 100644 --- a/contrib/pai_vscode/schemas/pai_job_config.schema.json +++ b/contrib/pai_vscode/schemas/pai_job_config.schema.json @@ -1,6 +1,6 @@ { "type": "object", - "description": "PAI job config", + "description": "PAI job config\nThis file can be submitted directly on PAI web portal.", "properties": { "jobName": { "type": "string", diff --git a/contrib/pai_vscode/src/common/constants.ts b/contrib/pai_vscode/src/common/constants.ts index c610763221..49d38dca0b 100644 --- a/contrib/pai_vscode/src/common/constants.ts +++ b/contrib/pai_vscode/src/common/constants.ts @@ -15,21 +15,46 @@ export const COMMAND_HDFS_UPLOAD_FOLDERS = 'paiext.hdfs.upload.folders'; export const COMMAND_HDFS_DOWNLOAD = 'paiext.hdfs.download'; export const COMMAND_OPEN_DASHBOARD = 'paiext.cluster.dashboard.open'; export const COMMAND_LIST_JOB = 'paiext.cluster.job.list'; +export const COMMAND_VIEW_JOB = 'paiext.cluster.job.view'; export const COMMAND_TREEVIEW_OPEN_PORTAL = 'paiext.treeview.openPortal'; export const COMMAND_TREEVIEW_DOUBLECLICK = 'paiext.treeview.doubleclick'; export const COMMAND_SUBMIT_JOB = 'paiext.cluster.job.submit'; export const COMMAND_SIMULATE_JOB = 'paiext.cluster.job.simulate'; export const COMMAND_CREATE_JOB_CONFIG = 'paiext.cluster.job.create-config'; +export const COMMAND_CONTAINER_HDFS_BACK = 'paiext.container.hdfs.back'; +export const COMMAND_CONTAINER_HDFS_REFRESH = 'paiext.container.hdfs.refresh'; +export const COMMAND_CONTAINER_HDFS_DELETE = 'paiext.container.hdfs.delete'; +export const COMMAND_CONTAINER_HDFS_MKDIR = 'paiext.container.hdfs.mkdir'; +export const COMMAND_CONTAINER_JOBLIST_REFRESH = 'paiext.container.joblist.refresh'; +export const COMMAND_CONTAINER_JOBLIST_MORE = 'paiext.container.joblist.more'; export const VIEW_CONFIGURATION_TREE = 'PAIExplorer'; export const CONTEXT_CONFIGURATION_ITEM = 'PAIConfiguration'; -export const CONTEXT_CONFIGURATION_ITEM_WEBPAGE = 'PAIWebpage'; +export const VIEW_CONTAINER_HDFS = 'PAIContainerHDFS'; +export const CONTEXT_HDFS_FILE = 'PAIHdfsFile'; +export const CONTEXT_HDFS_FOLDER = 'PAIHdfsFolder'; +export const CONTEXT_HDFS_ROOT = 'PAIHdfsRoot'; +export const CONTEXT_HDFS_SELECT_CLUSTER_ROOT = 'PAIHdfsSelectRoot'; +export const CONTEXT_HDFS_SELECT_CLUSTER = 'PAIHdfsSelect'; + +export const VIEW_CONTAINER_JOBLIST = 'PAIContainerJobList'; +export const CONTEXT_JOBLIST_CLUSTER = 'PAIJobListCluster'; + +export const SETTING_SECTION_HDFS = 'pai.hdfs'; +export const SETTING_HDFS_EXPLORER_LOCATION = 'location'; +export const ENUM_HDFS_EXPLORER_LOCATION = { + sidebar: 'sidebar', + explorer: 'explorer' +}; export const SETTING_SECTION_JOB = 'pai.job'; export const SETTING_JOB_UPLOAD_ENABLED = 'upload.enabled'; export const SETTING_JOB_UPLOAD_EXCLUDE = 'upload.exclude'; export const SETTING_JOB_UPLOAD_INCLUDE = 'upload.include'; export const SETTING_JOB_GENERATEJOBNAME_ENABLED = 'generateJobName.enabled'; +export const SETTING_JOB_JOBLIST_RECENTJOBSLENGTH = 'jobList.recentJobsLength'; +export const SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE = 'jobList.allJobsPageSize'; +export const SETTING_JOB_JOBLIST_REFERSHINTERVAL = 'jobList.refreshInterval'; export const ICON_PAI = { light: 'icons/PAI_light.png', @@ -57,6 +82,18 @@ export const ICON_CREATE_CONFIG = { light: 'icons/octicon/file.svg', dark: 'icons/octicon/file_dark.svg' }; +export const ICON_QUEUE = 'icons/queue.svg'; +export const ICON_STOP = 'icons/stop.svg'; +export const ICON_ERROR = 'icons/error.svg'; +export const ICON_RUN = 'icons/run.svg'; +export const ICON_OK = 'icons/ok.svg'; +export const ICON_HISTORY = 'icons/history.svg'; +export const ICON_LATEST = 'icons/latest.svg'; +export const ICON_ELLIPSIS = 'icons/ellipsis.svg'; +export const ICON_LOADING = { + light: 'icons/loading.svg', + dark: 'icons/loading_dark.svg' +}; export const OCTICON_CLOUDUPLOAD = '$(cloud-upload)'; diff --git a/contrib/pai_vscode/src/common/singleton.ts b/contrib/pai_vscode/src/common/singleton.ts index 2240d862c8..d3cc09b07c 100644 --- a/contrib/pai_vscode/src/common/singleton.ts +++ b/contrib/pai_vscode/src/common/singleton.ts @@ -6,7 +6,7 @@ import 'reflect-metadata'; // tslint:disable-line -import { Container, injectable } from 'inversify'; +import { injectable, Container } from 'inversify'; import * as vscode from 'vscode'; import { __ } from './i18n'; diff --git a/contrib/pai_vscode/src/common/treeViewHelper.ts b/contrib/pai_vscode/src/common/treeViewHelper.ts new file mode 100644 index 0000000000..525bf8d4f0 --- /dev/null +++ b/contrib/pai_vscode/src/common/treeViewHelper.ts @@ -0,0 +1,47 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ + +import { injectable } from 'inversify'; +import { isNil } from 'lodash'; +import { commands, workspace } from 'vscode'; + +import { COMMAND_TREEVIEW_DOUBLECLICK } from '../common/constants'; +import { __ } from '../common/i18n'; +import { Singleton } from '../common/singleton'; + +/** + * Supports double click commands for tree view + */ +@injectable() +export class TreeViewHelper extends Singleton { + + private lastClick?: { command: string, time: number }; + private readonly doubleClickInterval: number = 300; + + constructor() { + super(); + this.context.subscriptions.push( + commands.registerCommand(COMMAND_TREEVIEW_DOUBLECLICK, (command: string, ...args: string[]) => { + const mode: string | undefined = workspace.getConfiguration('workbench.list').get('openMode'); + if (mode === 'doubleClick') { + void commands.executeCommand(command, ...args); + } else { + // Single Click + if ( + !isNil(this.lastClick) && + this.lastClick.command === command && + Date.now() - this.lastClick.time < this.doubleClickInterval + ) { + this.lastClick = undefined; + void commands.executeCommand(command, ...args); + } else { + this.lastClick = { command, time: Date.now() }; + } + } + }) + ); + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/extension.ts b/contrib/pai_vscode/src/extension.ts index dcc3bc919c..138e3b37de 100644 --- a/contrib/pai_vscode/src/extension.ts +++ b/contrib/pai_vscode/src/extension.ts @@ -5,6 +5,7 @@ */ import * as vscode from 'vscode'; + import * as Singleton from './common/singleton'; import { allSingletonClasses } from './root'; diff --git a/contrib/pai_vscode/src/pai/clusterManager.ts b/contrib/pai_vscode/src/pai/clusterManager.ts index 21382072bc..38dcaa48e4 100644 --- a/contrib/pai_vscode/src/pai/clusterManager.ts +++ b/contrib/pai_vscode/src/pai/clusterManager.ts @@ -10,20 +10,28 @@ import * as request from 'request-promise-native'; import * as vscode from 'vscode'; import { - COMMAND_ADD_CLUSTER, COMMAND_DELETE_CLUSTER, COMMAND_EDIT_CLUSTER -} from '../common/constants'; + COMMAND_ADD_CLUSTER, COMMAND_DELETE_CLUSTER, COMMAND_EDIT_CLUSTER} from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ConfigurationNode, ConfigurationTreeDataProvider } from './configurationTreeDataProvider'; + +import { ClusterExplorerChildNode, ConfigurationTreeDataProvider, ITreeData } from './configurationTreeDataProvider'; import { IPAICluster } from './paiInterface'; import semverCompare = require('semver-compare'); // tslint:disable-line + export interface IConfiguration { readonly version: string; pais: IPAICluster[]; } +export type IClusterModification = { + index: number; + type: 'EDIT' | 'REMOVE'; +} | { + type: 'RESET'; +}; + export function getClusterIdentifier(conf: IPAICluster): string { return `${conf.username}@${conf.rest_server_uri}`; } @@ -32,10 +40,6 @@ export function getClusterName(conf: IPAICluster): string { return conf.name || getClusterIdentifier(conf); } -export function getClusterWebPortalUri(conf: IPAICluster): string { - return conf.web_portal_uri || conf.rest_server_uri.split(':')[0]; -} - /** * Manager class for cluster configurations */ @@ -57,6 +61,9 @@ export class ClusterManager extends Singleton { k8s_dashboard_uri: '127.0.0.1:9090' }; + private onDidChangeEmitter: vscode.EventEmitter = new vscode.EventEmitter(); + public onDidChange: vscode.Event = this.onDidChangeEmitter.event; // tslint:disable-line + private readonly EDIT: string = __('common.edit'); private readonly DISCARD: string = __('cluster.activate.fix.discard'); @@ -65,8 +72,20 @@ export class ClusterManager extends Singleton { public async onActivate(): Promise { this.context.subscriptions.push( vscode.commands.registerCommand(COMMAND_ADD_CLUSTER, () => this.add()), - vscode.commands.registerCommand(COMMAND_EDIT_CLUSTER, (node: ConfigurationNode) => this.edit(node.index)), - vscode.commands.registerCommand(COMMAND_DELETE_CLUSTER, (node: ConfigurationNode) => this.delete(node.index)) + vscode.commands.registerCommand(COMMAND_EDIT_CLUSTER, async (node: ClusterExplorerChildNode | ITreeData) => { + if (node instanceof ClusterExplorerChildNode) { + await this.edit(node.index); + } else { + await this.edit(node.clusterIndex); + } + }), + vscode.commands.registerCommand(COMMAND_DELETE_CLUSTER, async (node: ClusterExplorerChildNode | ITreeData) => { + if (node instanceof ClusterExplorerChildNode) { + await this.delete(node.index); + } else { + await this.delete(node.clusterIndex); + } + }) ); this.configuration = this.context.globalState.get(ClusterManager.CONF_KEY) || ClusterManager.default; try { @@ -126,7 +145,7 @@ export class ClusterManager extends Singleton { cluster.rest_server_uri = `${host}/rest-server`; cluster.k8s_dashboard_uri = `${host}/kubernetes-dashboard`; cluster.grafana_uri = `${host}/grafana`; - cluster.web_portal_uri = `${host}/`; + cluster.web_portal_uri = `${host}`; cluster.hdfs_uri = `hdfs://${host}:9000`; cluster.webhdfs_uri = `${host}/webhdfs/api/v1`; } catch { @@ -138,39 +157,40 @@ export class ClusterManager extends Singleton { cluster.hdfs_uri = `hdfs://${host}:9000`; cluster.k8s_dashboard_uri = `${host}:9090`; } - return this.edit(this.configuration!.pais.length, cluster); + return this.edit(this.allConfigurations.length, cluster); } public async edit(index: number, defaultConfiguration: IPAICluster = ClusterManager.paiDefault): Promise { - const original: IPAICluster = this.configuration!.pais[index] || defaultConfiguration; + const original: IPAICluster = this.allConfigurations[index] || defaultConfiguration; const editResult: IPAICluster | undefined = await Util.editJSON( original, `pai_cluster_${original.rest_server_uri}.json`, 'pai_cluster.schema.json' ); if (editResult) { - this.configuration!.pais[index] = editResult; - await (await getSingleton(ConfigurationTreeDataProvider)).refresh(index); + this.allConfigurations[index] = editResult; + this.onDidChangeEmitter.fire({ type: 'EDIT', index }); await this.save(); } } public async delete(index: number): Promise { - this.configuration!.pais.splice(index, 1); - await (await getSingleton(ConfigurationTreeDataProvider)).refresh(); + this.allConfigurations.splice(index, 1); + this.onDidChangeEmitter.fire({ type: 'REMOVE', index }); await this.save(); } - public save(): PromiseLike { - return this.context.globalState.update(ClusterManager.CONF_KEY, this.configuration!); + public async save(): Promise { + await this.context.globalState.update(ClusterManager.CONF_KEY, this.configuration!); + await (await getSingleton(ConfigurationTreeDataProvider)).refresh(); } public async pick(): Promise { - if (this.configuration!.pais.length === 1) { + if (this.allConfigurations.length === 1) { return 0; } - return await Util.pick(range(this.configuration!.pais.length), __('cluster.pick.prompt'), (index: number) => { - const conf: IPAICluster = this.allConfigurations![index]; + return await Util.pick(range(this.allConfigurations.length), __('cluster.pick.prompt'), (index: number) => { + const conf: IPAICluster = this.allConfigurations[index]; return { label: getClusterName(conf), detail: getClusterIdentifier(conf) @@ -195,7 +215,6 @@ export class ClusterManager extends Singleton { ); if (editResult) { this.configuration = editResult; - await (await getSingleton(ConfigurationTreeDataProvider)).refresh(); await this.save(); } break; @@ -205,5 +224,6 @@ export class ClusterManager extends Singleton { default: break; } + this.onDidChangeEmitter.fire({ type: 'RESET' }); } } \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts b/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts index 90b2c73f42..f9b8e2a87e 100644 --- a/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts +++ b/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts @@ -5,69 +5,43 @@ */ import { injectable } from 'inversify'; -import { isNil } from 'lodash'; import { - commands, Event, EventEmitter, TreeDataProvider, TreeItem, TreeItemCollapsibleState, window, - workspace + commands, window, + Event, EventEmitter, TreeDataProvider, TreeItem, TreeItemCollapsibleState } from 'vscode'; import { COMMAND_CREATE_JOB_CONFIG, COMMAND_EDIT_CLUSTER, COMMAND_LIST_JOB, COMMAND_OPEN_HDFS, COMMAND_REFRESH_CLUSTER, COMMAND_SIMULATE_JOB, COMMAND_SUBMIT_JOB, COMMAND_TREEVIEW_DOUBLECLICK, COMMAND_TREEVIEW_OPEN_PORTAL, - CONTEXT_CONFIGURATION_ITEM, CONTEXT_CONFIGURATION_ITEM_WEBPAGE, + CONTEXT_CONFIGURATION_ITEM, ICON_CREATE_CONFIG, ICON_DASHBOARD, ICON_EDIT, ICON_HDFS, ICON_LIST_JOB, ICON_PAI, ICON_SIMULATE_JOB, ICON_SUBMIT_JOB, VIEW_CONFIGURATION_TREE } from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ClusterManager, getClusterName } from './clusterManager'; + +import { getClusterName, ClusterManager } from './clusterManager'; import { IPAICluster } from './paiInterface'; interface IChildNodeDefinition { title: string; command: string; icon: string | { light: string, dark: string }; - type?: { new(...args: any[]): TreeNode }; condition?(conf: IPAICluster): boolean; } -/** - * General tree node recording its parent - */ -class TreeNode extends TreeItem { - constructor(title: string, public readonly parent?: TreeNode) { - super(title, parent ? TreeItemCollapsibleState.None : TreeItemCollapsibleState.Expanded); - } -} - -/** - * Tree node representing external link - */ -class TreeNodeWithLink extends TreeNode { - constructor(title: string, parent?: TreeNode) { - super(title, parent); - this.contextValue = CONTEXT_CONFIGURATION_ITEM_WEBPAGE; - } - - public get realCommand(): string | undefined { - return this.command && this.command.arguments && this.command.arguments[1]; - } -} - const childNodeDefinitions: IChildNodeDefinition[] = [ { title: 'treeview.node.openPortal', command: COMMAND_TREEVIEW_OPEN_PORTAL, - icon: ICON_DASHBOARD, - type: TreeNodeWithLink + icon: ICON_DASHBOARD }, { title: 'treeview.node.listjob', command: COMMAND_LIST_JOB, - icon: ICON_LIST_JOB, - type: TreeNodeWithLink + icon: ICON_LIST_JOB }, { title: 'treeview.node.create-config', @@ -97,43 +71,40 @@ const childNodeDefinitions: IChildNodeDefinition[] = [ } ]; +export interface ITreeData { + clusterIndex: number; + childDef?: IChildNodeDefinition; +} + /** * Root nodes representing cluster configuration */ -export class ConfigurationNode extends TreeNode { - public children: TreeNode[] = []; +export class ClusterExplorerRootNode extends TreeItem { + public readonly index: number; - public constructor(private _configuration: IPAICluster, public readonly index: number) { - super('...'); + public constructor(configuration: IPAICluster, index: number) { + super(getClusterName(configuration), TreeItemCollapsibleState.Expanded); this.iconPath = Util.resolvePath(ICON_PAI); - this.configuration = this._configuration; + this.index = index; this.contextValue = CONTEXT_CONFIGURATION_ITEM; } +} - public get configuration(): IPAICluster { - return this._configuration; - } - public set configuration(to: IPAICluster) { - this.label = getClusterName(to); - this._configuration = to; - this.initializeChildren(); - } - - private initializeChildren(): void { - this.children = []; - for (const def of childNodeDefinitions) { - if (def.condition && !def.condition(this.configuration)) { - continue; - } - const node: TreeNode = new (def.type || TreeNode)(__(def.title), this); - node.command = { - title: __(def.title), - command: COMMAND_TREEVIEW_DOUBLECLICK, - arguments: [this, def.command] - }; - node.iconPath = Util.resolvePath(def.icon); - this.children.push(node); - } +/** + * Child nodes representing operation + */ +export class ClusterExplorerChildNode extends TreeItem { + public readonly index: number; + + public constructor(clusterIndex: number, def: IChildNodeDefinition) { + super(__(def.title), TreeItemCollapsibleState.None); + this.iconPath = Util.resolvePath(def.icon); + this.index = clusterIndex; + this.command = { + title: __(def.title), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [def.command, this] + }; } } @@ -141,71 +112,54 @@ export class ConfigurationNode extends TreeNode { * Contributes to the tree view of cluster configurations */ @injectable() -export class ConfigurationTreeDataProvider extends Singleton implements TreeDataProvider { - private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); - public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line - - private configurationNodes: ConfigurationNode[] = []; - private lastClick?: { command: string, time: number }; - private readonly doubleClickInterval: number = 300; +export class ConfigurationTreeDataProvider extends Singleton implements TreeDataProvider { + private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); + public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line - constructor() { - super(); - this.context.subscriptions.push( - commands.registerCommand(COMMAND_REFRESH_CLUSTER, index => this.refresh(index)), - commands.registerCommand(COMMAND_TREEVIEW_DOUBLECLICK, (node: TreeNode, command: string) => { - const mode: string | undefined = workspace.getConfiguration('workbench.list').get('openMode'); - if (mode === 'doubleClick') { - void commands.executeCommand(command, node); - } else { - // Single Click - if ( - !isNil(this.lastClick) && - this.lastClick.command === command && - Date.now() - this.lastClick.time < this.doubleClickInterval - ) { - this.lastClick = undefined; - void commands.executeCommand(command, node); - } else { - this.lastClick = { command, time: Date.now() }; - } - } - }), - window.registerTreeDataProvider(VIEW_CONFIGURATION_TREE, this) - ); + public async refresh(): Promise { + this.onDidChangeTreeDataEmitter.fire(); } - public async refresh(index: number = -1): Promise { - const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; - if (index === -1 || !this.configurationNodes[index]) { - this.configurationNodes = allConfigurations.map((conf, i) => new ConfigurationNode(conf, i)); - this.onDidChangeTreeDataEmitter.fire(); + public async getTreeItem(data: ITreeData): Promise { + if (!data.childDef) { + const cluster: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations[data.clusterIndex]; + return new ClusterExplorerRootNode(cluster, data.clusterIndex); } else { - this.configurationNodes[index].configuration = allConfigurations[index]; - this.onDidChangeTreeDataEmitter.fire(this.configurationNodes[index]); + return new ClusterExplorerChildNode(data.clusterIndex, data.childDef); } } - public getTreeItem(element: TreeNode): TreeNode { - return element; - } - - public getChildren(element?: TreeNode): TreeNode[] | undefined { - if (!element) { - // Root nodes: configurations - return this.configurationNodes; - } - if (element instanceof ConfigurationNode) { - return element.children; + public async getChildren(data?: ITreeData): Promise { + if (!data) { + const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; + return allConfigurations.map((c, i) => ({ + clusterIndex: i + })); + } else if (!data.childDef) { + const cluster: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations[data.clusterIndex]; + return childNodeDefinitions.filter((def) => !def.condition || def.condition(cluster)).map(def => ({ + clusterIndex: data.clusterIndex, + childDef: def + })); + } else { + return undefined; } - return; } - public getParent(element: TreeNode): TreeNode | undefined { - return element.parent; + public getParent(data: ITreeData): ITreeData | undefined { + if (data.childDef) { + return { + clusterIndex: data.clusterIndex + }; + } else { + return undefined; + } } - public onActivate(): Promise { - return this.refresh(); + public async onActivate(): Promise { + this.context.subscriptions.push( + commands.registerCommand(COMMAND_REFRESH_CLUSTER, () => this.refresh()), + window.registerTreeDataProvider(VIEW_CONFIGURATION_TREE, this) + ); } } \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/container/hdfsTreeView.ts b/contrib/pai_vscode/src/pai/container/hdfsTreeView.ts new file mode 100644 index 0000000000..6639da7912 --- /dev/null +++ b/contrib/pai_vscode/src/pai/container/hdfsTreeView.ts @@ -0,0 +1,218 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ +/* tslint:disable:max-classes-per-file */ + +import { injectable } from 'inversify'; +import { + commands, window, Event, EventEmitter, FileType, + TreeDataProvider, TreeItem, TreeItemCollapsibleState, + TreeView, Uri +} from 'vscode'; + +import { + COMMAND_CONTAINER_HDFS_BACK, COMMAND_CONTAINER_HDFS_DELETE, COMMAND_CONTAINER_HDFS_MKDIR, COMMAND_CONTAINER_HDFS_REFRESH, + COMMAND_OPEN_HDFS, COMMAND_TREEVIEW_DOUBLECLICK, + CONTEXT_HDFS_FILE, CONTEXT_HDFS_FOLDER, CONTEXT_HDFS_ROOT, CONTEXT_HDFS_SELECT_CLUSTER, CONTEXT_HDFS_SELECT_CLUSTER_ROOT, + ICON_PAI, VIEW_CONTAINER_HDFS +} from '../../common/constants'; +import { __ } from '../../common/i18n'; +import { getSingleton, Singleton } from '../../common/singleton'; +import { Util } from '../../common/util'; + +import { getClusterName, ClusterManager } from '../clusterManager'; +import { HDFS, HDFSFileSystemProvider } from '../hdfs'; +import { IPAICluster } from '../paiInterface'; + +type IFileList = [string, FileType][]; + +/** + * Abstract tree node + */ +abstract class TreeNode extends TreeItem { + public parent?: TreeNode; +} + +/** + * File node + */ +class FileNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_FILE; + constructor(uri: Uri, parent: TreeNode) { + super(uri, TreeItemCollapsibleState.None); + this.parent = parent; + } +} + +/** + * Folder node + */ +class FolderNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_FOLDER; + constructor(uri: Uri, parent: TreeNode) { + super(uri, TreeItemCollapsibleState.Collapsed); + this.parent = parent; + } +} + +/** + * Root node + */ +class RootNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_ROOT; + constructor(uri: Uri) { + super(uri, TreeItemCollapsibleState.Expanded); + this.label = uri.toString(); + this.iconPath = Util.resolvePath(ICON_PAI); + } +} + +/** + * Cluster root node + */ +class SelectClusterRootNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_SELECT_CLUSTER_ROOT; + constructor() { + super(__('treeview.hdfs.select-cluster.label'), TreeItemCollapsibleState.Expanded); + } +} + +/** + * Cluster node (when no cluster is selected) + */ +class SelectClusterNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_SELECT_CLUSTER; + public readonly cluster: IPAICluster; + constructor(cluster: IPAICluster, parent: TreeNode) { + super(getClusterName(cluster)); + this.cluster = cluster; + this.parent = parent; + this.command = { + title: __('treeview.node.openhdfs'), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [COMMAND_OPEN_HDFS, this.cluster] + }; + this.iconPath = Util.resolvePath(ICON_PAI); + } +} + +/** + * Contributes to the tree view of cluster configurations + */ +@injectable() +export class HDFSTreeDataProvider extends Singleton implements TreeDataProvider { + public readonly view: TreeView; + public root: TreeNode; + + private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); + public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line + + private uri?: Uri; + + constructor() { + super(); + this.root = new SelectClusterRootNode(); + this.view = window.createTreeView(VIEW_CONTAINER_HDFS, { treeDataProvider: this }); + } + + public async onActivate(): Promise { + this.context.subscriptions.push( + commands.registerCommand(COMMAND_CONTAINER_HDFS_REFRESH, () => this.refresh()), + commands.registerCommand(COMMAND_CONTAINER_HDFS_BACK, () => this.reset()), + commands.registerCommand(COMMAND_CONTAINER_HDFS_DELETE, async (node: TreeItem) => { + await (await getSingleton(HDFS)).provider!.delete(node.resourceUri!, { recursive: true }); + }), + commands.registerCommand(COMMAND_CONTAINER_HDFS_MKDIR, async (node: TreeItem) => { + const res: string | undefined = await window.showInputBox({ + prompt: __('container.hdfs.mkdir.prompt') + }); + if (res === undefined) { + Util.warn('container.hdfs.mkdir.cancelled'); + } else { + await (await getSingleton(HDFS)).provider!.createDirectory(Util.uriPathAppend(node.resourceUri!, res)); + } + }) + ); + this.refresh(); + (await getSingleton(HDFS)).provider!.onDidChangeFile(() => this.refresh()); + } + + public reset(): void { + this.uri = undefined; + this.root = new SelectClusterRootNode(); + this.refresh(); + void this.view.reveal(this.root); + } + + public setUri(uri?: Uri): void { + if (uri === undefined) { + this.reset(); + } else { + this.uri = uri; + this.root = new RootNode(this.uri); + this.refresh(); + void this.view.reveal(this.root); + } + } + + public refresh(node?: TreeNode): void { + this.onDidChangeTreeDataEmitter.fire(node); + } + + public getTreeItem(element: TreeNode): TreeNode { + return element; + } + + public async getChildren(element?: TreeNode): Promise { + if (!this.uri) { + if (!element) { + return [this.root]; + } else if (element.contextValue === CONTEXT_HDFS_SELECT_CLUSTER_ROOT) { + const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; + return allConfigurations.filter( + conf => !!conf.webhdfs_uri + ).map( + conf => new SelectClusterNode(conf, element) + ); + } else { + return; + } + } else { + if (!element) { + return [this.root]; + } else if (element.contextValue === CONTEXT_HDFS_FOLDER || element.contextValue === CONTEXT_HDFS_ROOT) { + const provider: HDFSFileSystemProvider | undefined = (await getSingleton(HDFS)).provider; + if (!provider) { + return; + } + const uri: Uri = element.resourceUri!; + const res: IFileList = await provider.readDirectory(uri); + return [ + ...res.filter( + ([name, type]) => type === FileType.Directory + ).sort( + ([name1, type1], [name2, type2]) => name1.localeCompare(name2) + ).map( + ([name, type]) => new FolderNode(Util.uriPathAppend(uri, name), element) + ), + ...res.filter( + ([name, type]) => type === FileType.File + ).sort( + ([name1, type1], [name2, type2]) => name1.localeCompare(name2) + ).map( + ([name, type]) => new FileNode(Util.uriPathAppend(uri, name), element) + ) + ]; + } else { + return; + } + + } + } + + public getParent(element: TreeNode): TreeNode | undefined { + return element.parent; + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/container/jobListTreeView.ts b/contrib/pai_vscode/src/pai/container/jobListTreeView.ts new file mode 100644 index 0000000000..60f8390d5b --- /dev/null +++ b/contrib/pai_vscode/src/pai/container/jobListTreeView.ts @@ -0,0 +1,360 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ +/* tslint:disable:max-classes-per-file */ + +import { injectable } from 'inversify'; +import * as request from 'request-promise-native'; +import { + commands, window, workspace, Event, EventEmitter, TreeDataProvider, + TreeItem, TreeItemCollapsibleState, TreeView, WorkspaceConfiguration +} from 'vscode'; + +import { + COMMAND_CONTAINER_JOBLIST_MORE, COMMAND_CONTAINER_JOBLIST_REFRESH, + COMMAND_TREEVIEW_DOUBLECLICK, COMMAND_VIEW_JOB, + CONTEXT_JOBLIST_CLUSTER, + ICON_ELLIPSIS, + ICON_ERROR, + ICON_HISTORY, + ICON_LATEST, + ICON_LOADING, + ICON_OK, + ICON_PAI, + ICON_QUEUE, + ICON_RUN, + ICON_STOP, + SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE, + SETTING_JOB_JOBLIST_RECENTJOBSLENGTH, + SETTING_JOB_JOBLIST_REFERSHINTERVAL, + SETTING_SECTION_JOB, + VIEW_CONTAINER_JOBLIST +} from '../../common/constants'; +import { __ } from '../../common/i18n'; +import { getSingleton, Singleton } from '../../common/singleton'; +import { Util } from '../../common/util'; +import { getClusterName, ClusterManager } from '../clusterManager'; +import { IPAICluster, IPAIJobInfo } from '../paiInterface'; +import { PAIRestUri } from '../paiUri'; +import { RecentJobManager } from '../recentJobManager'; + +enum FilterType { + Recent = 0, + All = 1 +} + +enum LoadingState { + Finished = 0, + Loading = 1, + Error = 2 +} + +enum TreeDataType { + Cluster = 0, + Filter = 1, + Job = 2, + More = 3 +} + +/** + * Leaf node representing job on PAI + */ +export class JobNode extends TreeItem { + private static statusIcons: { [status in IPAIJobInfo['state']]: string | undefined } = { + SUCCEEDED: ICON_OK, + FAILED: ICON_ERROR, + WAITING: ICON_QUEUE, + STOPPED: ICON_STOP, + RUNNING: ICON_RUN, + UNKNOWN: undefined + }; + + public constructor(jobInfo: IPAIJobInfo, config: IPAICluster) { + super(jobInfo.name); + this.command = { + title: __('treeview.joblist.view'), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [COMMAND_VIEW_JOB, jobInfo, config] + }; + const icon: string | undefined = JobNode.statusIcons[jobInfo.state]; + if (icon) { + this.iconPath = Util.resolvePath(icon); + } + } +} + +/** + * Expand job list when chosen + */ +class ShowMoreNode extends TreeItem { + public constructor(cluster: IClusterData) { + super(__('treeview.joblist.more')); + this.command = { + title: __('treeview.joblist.more'), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [COMMAND_CONTAINER_JOBLIST_MORE, cluster] + }; + this.iconPath = Util.resolvePath(ICON_ELLIPSIS); + } +} + +/** + * Secondary node containing filtered job list + */ +class FilterNode extends TreeItem { + public constructor(type: FilterType, loadingState: LoadingState) { + if (type === FilterType.Recent) { + super(__('treeview.joblist.recent'), TreeItemCollapsibleState.Expanded); + } else { + super(__('treeview.joblist.all'), TreeItemCollapsibleState.Collapsed); + } + this.iconPath = Util.resolvePath( + loadingState === LoadingState.Loading ? ICON_LOADING : + loadingState === LoadingState.Error ? ICON_ERROR : + type === FilterType.Recent ? ICON_LATEST : ICON_HISTORY); + } +} + +/** + * Root node representing PAI cluster + */ +export class ClusterNode extends TreeItem { + public readonly index: number; + public constructor(configuration: IPAICluster, index: number) { + super(getClusterName(configuration), TreeItemCollapsibleState.Collapsed); + this.index = index; + this.iconPath = Util.resolvePath(ICON_PAI); + this.contextValue = CONTEXT_JOBLIST_CLUSTER; + } +} + +interface IClusterData { + type: TreeDataType.Cluster; + config: IPAICluster; + index: number; + shownAmount: number; + loadingState: LoadingState; + jobs: IPAIJobInfo[]; + lastShownAmount?: number; +} + +interface IFilterData { + type: TreeDataType.Filter; + filterType: FilterType; + clusterIndex: number; +} + +interface IJobData { + type: TreeDataType.Job; + job: IPAIJobInfo; + clusterIndex: number; + filterType: FilterType; +} + +interface IMoreData { + type: TreeDataType.More; + clusterIndex: number; +} + +type ITreeData = IClusterData | IFilterData | IJobData | IMoreData; + +/** + * Contributes to the tree view of cluster job list + */ +@injectable() +export class JobListTreeDataProvider extends Singleton implements TreeDataProvider { + private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); + public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line + + private clusters: IClusterData[] = []; + private readonly treeView: TreeView; + private refreshTimer: NodeJS.Timer | undefined; + + constructor() { + super(); + this.treeView = window.createTreeView(VIEW_CONTAINER_JOBLIST, { treeDataProvider: this }); + this.context.subscriptions.push( + commands.registerCommand(COMMAND_CONTAINER_JOBLIST_REFRESH, () => this.refresh()), + commands.registerCommand( + COMMAND_CONTAINER_JOBLIST_MORE, + (cluster: IClusterData) => { + if (cluster.jobs.length <= cluster.shownAmount) { + return; + } + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + cluster.lastShownAmount = cluster.shownAmount; + cluster.shownAmount += settings.get(SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE)!; + void this.refresh(cluster.index, false); + } + ) + ); + } + + public async refresh(index: number = -1, reload: boolean = true): Promise { + if (index === -1 || !this.clusters[index]) { + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; + this.clusters = allConfigurations.map((config, i) => ({ + type: TreeDataType.Cluster, + index: i, + config, + loadingState: LoadingState.Finished, + jobs: [], + shownAmount: settings.get(SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE)! + })); + this.onDidChangeTreeDataEmitter.fire(); + if (reload) { + void this.reloadJobs(); + } + } else { + this.onDidChangeTreeDataEmitter.fire(this.clusters[index]); + if (reload) { + void this.reloadJobs(index); + } + } + } + + public getTreeItem(element: ITreeData): TreeItem { + switch (element.type) { + case TreeDataType.Cluster: + return new ClusterNode(element.config, element.index); + case TreeDataType.Filter: + return new FilterNode(element.filterType, this.clusters[element.clusterIndex].loadingState); + case TreeDataType.Job: + return new JobNode(element.job, this.clusters[element.clusterIndex].config); + case TreeDataType.More: + return new ShowMoreNode(this.clusters[element.clusterIndex]); + default: + throw new Error('Unexpected node type'); + } + } + + public async getChildren(element?: ITreeData): Promise { + if (!element) { + // Root nodes: configurations + return this.clusters; + } + switch (element.type) { + case TreeDataType.Cluster: + { + return [ + { type: TreeDataType.Filter, filterType: FilterType.Recent, clusterIndex: element.index }, + { type: TreeDataType.Filter, filterType: FilterType.All, clusterIndex: element.index } + ]; + } + case TreeDataType.Filter: + if (element.filterType === FilterType.Recent) { + const cluster: IClusterData = this.clusters[element.clusterIndex]; + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + const recentMaxLen: number = settings.get(SETTING_JOB_JOBLIST_RECENTJOBSLENGTH)!; + const recentJobs: string[] | undefined = (await getSingleton(RecentJobManager)).allRecentJobs[cluster.index] || []; + const result: IJobData[] = []; + for (const name of recentJobs.slice(0, recentMaxLen)) { + const foundJob: IPAIJobInfo | undefined = cluster.jobs.find(job => job.name === name); + if (foundJob) { + result.push({ + type: TreeDataType.Job, + job: foundJob, + clusterIndex: element.clusterIndex, + filterType: element.filterType + }); + } + } + return result; + } else { + const cluster: IClusterData = this.clusters[element.clusterIndex]; + const result: (IJobData | IMoreData)[] = cluster.jobs.slice(0, cluster.shownAmount).map( + job => ({ + type: TreeDataType.Job, + job, + clusterIndex: element.clusterIndex, + filterType: element.filterType + }) + ); + if (cluster.lastShownAmount && cluster.lastShownAmount !== cluster.shownAmount) { + setImmediate(i => this.treeView.reveal(result[i]), cluster.lastShownAmount - 1); + cluster.lastShownAmount = cluster.shownAmount; + } + if (cluster.jobs.length > cluster.shownAmount) { + result.push({ type: TreeDataType.More, clusterIndex: element.clusterIndex }); + } + return result; + } + case TreeDataType.Job: + case TreeDataType.More: + return undefined; + default: + } + } + + public getParent(element: ITreeData): ITreeData | undefined { + if (element.type === TreeDataType.Job) { + return { + type: TreeDataType.Filter, + filterType: element.filterType, + clusterIndex: element.clusterIndex + }; + } + if (element.type === TreeDataType.Filter) { + return this.clusters[element.clusterIndex]; + } + } + + public async revealLatestJob(clusterIndex: number, jobName: string): Promise { + const job: IPAIJobInfo | undefined = this.clusters[clusterIndex].jobs.find(j => j.name === jobName); + if (job) { + /** + * Note: treeView.reveal() will obtain the node's parent and + * traverse to ancestors (upwards) until a expanded node has been found, + * then go back down and expand nodes via getChildren() on demand + */ + await this.treeView.reveal( + { + type: TreeDataType.Job, + job: job, + clusterIndex: clusterIndex, + filterType: FilterType.Recent + }, + { focus: true } + ); + } + } + + public onActivate(): Promise { + return this.refresh(); + } + + public async onDeactivate(): Promise { + if (this.refreshTimer) { + clearTimeout(this.refreshTimer); + } + this.treeView.dispose(); + } + + private async reloadJobs(index: number = -1): Promise { + if (this.refreshTimer) { + clearTimeout(this.refreshTimer); + } + const clusters: IClusterData[] = index !== -1 ? [this.clusters[index]] : this.clusters; + await Promise.all(clusters.map(async cluster => { + cluster.loadingState = LoadingState.Loading; + this.onDidChangeTreeDataEmitter.fire(cluster); + try { + cluster.jobs = await request.get( + PAIRestUri.jobs(cluster.config), + { json: true } + ); + cluster.loadingState = LoadingState.Finished; + } catch (e) { + Util.err('treeview.joblist.error', [e.message || e]); + cluster.loadingState = LoadingState.Error; + } + this.onDidChangeTreeDataEmitter.fire(cluster); + })); + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + const interval: number = settings.get(SETTING_JOB_JOBLIST_REFERSHINTERVAL)!; + this.refreshTimer = setTimeout(this.reloadJobs.bind(this), interval * 1000); + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/hdfs.ts b/contrib/pai_vscode/src/pai/hdfs.ts index 0ce4a71dac..6161b28c84 100644 --- a/contrib/pai_vscode/src/pai/hdfs.ts +++ b/contrib/pai_vscode/src/pai/hdfs.ts @@ -15,13 +15,16 @@ import { promisify } from 'util'; import * as vscode from 'vscode'; import { - COMMAND_HDFS_DOWNLOAD, COMMAND_HDFS_UPLOAD_FILES, COMMAND_HDFS_UPLOAD_FOLDERS, COMMAND_OPEN_HDFS, OCTICON_CLOUDUPLOAD + COMMAND_HDFS_DOWNLOAD, COMMAND_HDFS_UPLOAD_FILES, COMMAND_HDFS_UPLOAD_FOLDERS, COMMAND_OPEN_HDFS, + ENUM_HDFS_EXPLORER_LOCATION, OCTICON_CLOUDUPLOAD, SETTING_HDFS_EXPLORER_LOCATION, SETTING_SECTION_HDFS } from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; + import { ClusterManager } from './clusterManager'; -import { ConfigurationNode } from './configurationTreeDataProvider'; +import { ClusterExplorerChildNode } from './configurationTreeDataProvider'; +import { HDFSTreeDataProvider } from './container/hdfsTreeView'; import { IPAICluster } from './paiInterface'; import { createWebHDFSClient, IHDFSClient, IHDFSStatResult } from './webhdfs-workaround'; @@ -94,11 +97,17 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { statResult = await this.stat(uri); if (statResult.type !== vscode.FileType.Directory) { throw vscode.FileSystemError.FileExists(uri); + } else { + // pass + } + } catch (e) { + if (uri.path === '/') { + throw e; } - } catch { await this.createDirectory(Util.uriPathPop(uri)); try { await (await this.getClient(uri)).mkdir(path.join('/', uri.path)); + this.onDidChangeFileEmitter.fire([{ type: vscode.FileChangeType.Created, uri }]); } catch (ex) { throw new vscode.FileSystemError(ex); } @@ -108,6 +117,7 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { public async delete(uri: vscode.Uri, options: {recursive: boolean}): Promise { try { await (await this.getClient(uri)).unlink(path.join('/', uri.path), options.recursive); + this.onDidChangeFileEmitter.fire([{ type: vscode.FileChangeType.Deleted, uri }]); } catch (ex) { throw new vscode.FileSystemError(ex); } @@ -248,79 +258,81 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { const client: IHDFSClient = (await this.getClient(uri)); const filePath: string = path.join('/', uri.path); await vscode.window.withProgress( - { - location: vscode.ProgressLocation.Notification, - title: __('hdfs.uploading', [filePath]), - cancellable: true - }, - (progress, cancellationToken) => new Promise(async (resolve, reject) => { - try { - if ((await client.stat(filePath)).type === 'DIRECTORY') { - reject(vscode.FileSystemError.FileIsADirectory(uri)); - return; - } - if (!options.overwrite) { - reject(vscode.FileSystemError.FileExists(uri)); - return; - } - } catch { - if (!options.create) { - reject(vscode.FileSystemError.FileNotFound(uri)); - return; + { + location: vscode.ProgressLocation.Notification, + title: __('hdfs.uploading', [filePath]), + cancellable: true + }, + (progress, cancellationToken) => new Promise(async (resolve, reject) => { + try { + if ((await client.stat(filePath)).type === 'DIRECTORY') { + reject(vscode.FileSystemError.FileIsADirectory(uri)); + return; + } + if (!options.overwrite) { + reject(vscode.FileSystemError.FileExists(uri)); + return; + } + } catch { + if (!options.create) { + reject(vscode.FileSystemError.FileNotFound(uri)); + return; + } } - } - const local: fs.ReadStream = streamifier.createReadStream(content); - let writeAmount: number = 0; - const transform: Transform = new Transform({ - transform: (chunk: string | Buffer, encoding: string, callback: Function) => { - writeAmount += chunk.length; - progress.report({ - message: __('hdfs.progress', [ - (writeAmount / content.length * 100).toFixed(0), writeAmount, content.length - ]), - increment: chunk.length / content.length * 100 - }); - callback(null, chunk); + const local: fs.ReadStream = streamifier.createReadStream(content); + let writeAmount: number = 0; + const transform: Transform = new Transform({ + transform: (chunk: string | Buffer, encoding: string, callback: Function) => { + writeAmount += chunk.length; + progress.report({ + message: __('hdfs.progress', [ + (writeAmount / content.length * 100).toFixed(0), writeAmount, content.length + ]), + increment: chunk.length / content.length * 100 + }); + callback(null, chunk); + } + }); + const stream: Request = await client.createRobustWriteStream(filePath); + let error: any; + + function cleanup(): void { + local.unpipe(); + transform.unpipe(); + local.destroy(); + transform.destroy(); + stream.destroy(); } - }); - const stream: Request = await client.createRobustWriteStream(filePath); - let error: any; - - function cleanup(): void { - local.unpipe(); - transform.unpipe(); - local.destroy(); - transform.destroy(); - stream.destroy(); - } - cancellationToken.onCancellationRequested(() => { - error = true; - cleanup(); - reject(__('hdfs.write.cancelled')); - }); + cancellationToken.onCancellationRequested(() => { + error = true; + cleanup(); + reject(__('hdfs.write.cancelled')); + }); - local.once('error', err => { - error = err; - cleanup(); - reject(new vscode.FileSystemError(err)); - }); - stream.once('error', err => { - error = err; - cleanup(); - reject(new vscode.FileSystemError(err)); - }); + local.once('error', err => { + error = err; + cleanup(); + reject(new vscode.FileSystemError(err)); + }); + stream.once('error', err => { + error = err; + cleanup(); + reject(new vscode.FileSystemError(err)); + }); - stream.once('finish', () => { - cleanup(); - if (!error) { - resolve(); - } - }); + stream.once('finish', () => { + cleanup(); + if (!error) { + resolve(); + } + }); - // TODO: local.pipe(transform).pipe(stream); is not working due to unknown reason...maybe a bug in node-webhdfs? - local.pipe(transform).pipe(stream); - })); + // TODO: local.pipe(transform).pipe(stream); is not working due to unknown reason...maybe a bug in node-webhdfs? + local.pipe(transform).pipe(stream); + }) + ); + this.onDidChangeFileEmitter.fire([{ type: vscode.FileChangeType.Created, uri }]); } public async copy(source: vscode.Uri, destination: vscode.Uri, options: { overwrite: boolean }): Promise { @@ -460,65 +472,81 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { */ @injectable() export class HDFS extends Singleton { + public readonly provider: HDFSFileSystemProvider; private UPLOADFILES: string = __('hdfs.dialog.label.upload-files'); private UPLOADFOLDER: string = __('hdfs.dialog.label.upload-folders'); private DOWNLOADHERE: string = __('hdfs.dialog.label.download'); - private _provider: HDFSFileSystemProvider | undefined; - public get provider(): HDFSFileSystemProvider | undefined { - return this._provider; - } - constructor() { super(); console.log('Registering HDFS...'); - try { - this._provider = new HDFSFileSystemProvider(); - this.context.subscriptions.push( - vscode.workspace.registerFileSystemProvider('webhdfs', this._provider, { isCaseSensitive: true }) - ); - console.log('HDFS registered as webhdfs:/...'); - } catch (ex) { - Util.err('hdfs.initialization.error', [ex]); - } + this.provider = new HDFSFileSystemProvider(); + this.context.subscriptions.push( + vscode.workspace.registerFileSystemProvider('webhdfs', this.provider, { isCaseSensitive: true }) + ); + console.log('HDFS registered as webhdfs:/...'); this.context.subscriptions.push( vscode.commands.registerCommand( COMMAND_OPEN_HDFS, - (node: ConfigurationNode) => this.open(node.index) + async (node?: ClusterExplorerChildNode | IPAICluster) => { + if (!node) { + const manager: ClusterManager = await getSingleton(ClusterManager); + const index: number | undefined = await manager.pick(); + if (index === undefined) { + return; + } + await this.open(manager.allConfigurations[index]); + } else if (node instanceof ClusterExplorerChildNode) { + await this.open((await getSingleton(ClusterManager)).allConfigurations[node.index]); + } else { + await this.open(node); + } + } ), - vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FILES, this.uploadFiles.bind(this)), - vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FOLDERS, this.uploadFolders.bind(this)), - vscode.commands.registerCommand(COMMAND_HDFS_DOWNLOAD, this.download.bind(this)) + vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FILES, async (param: vscode.Uri | vscode.TreeItem) => { + await this.uploadFiles(this.unpackParam(param)); + }), + vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FOLDERS, async (param: vscode.Uri | vscode.TreeItem) => { + await this.uploadFolders(this.unpackParam(param)); + }), + vscode.commands.registerCommand(COMMAND_HDFS_DOWNLOAD, async (param: vscode.Uri | vscode.TreeItem) => { + await this.download(this.unpackParam(param)); + }) ); } - public async open(index: number): Promise { - const configuration: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations[index]; - if (!configuration.webhdfs_uri) { + public async open(conf: IPAICluster): Promise { + if (!conf.webhdfs_uri) { Util.err('hdfs.initialization.missingconfiguration'); return; } - let start: number = 0; - let deleteCount: number = 0; - if (vscode.workspace.workspaceFolders) { - start = vscode.workspace.workspaceFolders.findIndex(folder => folder.uri.scheme === 'webhdfs'); - if (start >= 0) { - deleteCount = 1; - } else { - start = vscode.workspace.workspaceFolders.length; + const setting: string | undefined = vscode.workspace.getConfiguration(SETTING_SECTION_HDFS).get(SETTING_HDFS_EXPLORER_LOCATION); + if (setting === ENUM_HDFS_EXPLORER_LOCATION.explorer) { + let start: number = 0; + let deleteCount: number = 0; + if (vscode.workspace.workspaceFolders) { + start = vscode.workspace.workspaceFolders.findIndex(folder => folder.uri.scheme === 'webhdfs'); + if (start >= 0) { + deleteCount = 1; + } else { + start = vscode.workspace.workspaceFolders.length; + } } - } - try { - // this._provider!.addClient(getHDFSUriAuthority(configuration)); - await vscode.commands.executeCommand('workbench.view.explorer'); - void vscode.window.showInformationMessage(__('hdfs.open.prompt', [configuration.webhdfs_uri])); - vscode.workspace.updateWorkspaceFolders(start, deleteCount, { - uri: vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(configuration)}/`), - name: __('hdfs.workspace.title', [configuration.webhdfs_uri]) - }); - // Extension may be reloaded at this point due to workspace changes - } catch (ex) { - Util.err('hdfs.open.error', [ex]); + try { + // this.provider!.addClient(getHDFSUriAuthority(configuration)); + await vscode.commands.executeCommand('workbench.view.explorer'); + void vscode.window.showInformationMessage(__('hdfs.open.prompt', [conf.webhdfs_uri])); + vscode.workspace.updateWorkspaceFolders(start, deleteCount, { + uri: vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(conf)}/`), + name: __('hdfs.workspace.title', [conf.webhdfs_uri]) + }); + // Extension may be reloaded at this point due to workspace changes + } catch (ex) { + Util.err('hdfs.open.error', [ex]); + } + } else { + const provider: HDFSTreeDataProvider = await getSingleton(HDFSTreeDataProvider); + provider.setUri(vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(conf)}/`)); } } @@ -534,7 +562,19 @@ export class HDFS extends Singleton { } } - public async download(from: vscode.Uri): Promise { + private unpackParam(param: vscode.Uri | vscode.TreeItem): vscode.Uri { + if (param instanceof vscode.TreeItem) { + if (param.resourceUri !== undefined) { + return param.resourceUri; + } else { + throw new Error('Invalid HDFS operation param'); + } + } else { + return param; + } + } + + private async download(from: vscode.Uri): Promise { const files: vscode.Uri[] | undefined = await vscode.window.showOpenDialog({ canSelectFiles: false, canSelectFolders: true, @@ -547,7 +587,7 @@ export class HDFS extends Singleton { await this.provider!.copy(from, files[0], { overwrite: true }); } - public async uploadFiles(target: vscode.Uri): Promise { + private async uploadFiles(target: vscode.Uri): Promise { const files: vscode.Uri[] | undefined = await vscode.window.showOpenDialog({ canSelectFiles: true, canSelectMany: true, @@ -559,7 +599,7 @@ export class HDFS extends Singleton { return this.upload(files, target); } - public async uploadFolders(target: vscode.Uri): Promise { + private async uploadFolders(target: vscode.Uri): Promise { const folders: vscode.Uri[] | undefined = await vscode.window.showOpenDialog({ canSelectFiles: false, canSelectFolders: true, diff --git a/contrib/pai_vscode/src/pai/paiInterface.ts b/contrib/pai_vscode/src/pai/paiInterface.ts index 5ca4a2faf3..c0ef9fe468 100644 --- a/contrib/pai_vscode/src/pai/paiInterface.ts +++ b/contrib/pai_vscode/src/pai/paiInterface.ts @@ -33,4 +33,17 @@ export interface IPAIJobConfig { codeDir: string; outputDir: string; taskRoles: IPAITaskRole[]; +} + +export interface IPAIJobInfo { + name: string; + username: string; + state: 'SUCCEEDED' | 'FAILED' | 'WAITING' | 'STOPPED' | 'RUNNING' | 'UNKNOWN'; + subState: 'FRAMEWORK_COMPLETED' | 'FRAMEWORK_WAITING'; + executionType: 'START' | 'STOP'; + retries: number; + createdTime: number; + completedTime: number; + appExitCode: number; + virtualCluster: string; } \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/paiJobManager.ts b/contrib/pai_vscode/src/pai/paiJobManager.ts index 52b6dac74f..0cdbfb5cea 100644 --- a/contrib/pai_vscode/src/pai/paiJobManager.ts +++ b/contrib/pai_vscode/src/pai/paiJobManager.ts @@ -8,10 +8,9 @@ import * as fs from 'fs-extra'; import * as globby from 'globby'; import { injectable } from 'inversify'; import * as JSONC from 'jsonc-parser'; -import { isEmpty } from 'lodash'; +import { isEmpty, isNil } from 'lodash'; import * as os from 'os'; import * as path from 'path'; -import * as querystring from 'querystring'; import * as request from 'request-promise-native'; import * as uuid from 'uuid'; import * as vscode from 'vscode'; @@ -27,13 +26,16 @@ import { import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ClusterManager, getClusterIdentifier, getClusterWebPortalUri } from './clusterManager'; -import { ConfigurationNode } from './configurationTreeDataProvider'; + +import { getClusterIdentifier, ClusterManager } from './clusterManager'; +import { ClusterExplorerChildNode } from './configurationTreeDataProvider'; import { getHDFSUriAuthority, HDFS, HDFSFileSystemProvider } from './hdfs'; import { IPAICluster, IPAIJobConfig, IPAITaskRole } from './paiInterface'; import opn = require('opn'); // tslint:disable-line import unixify = require('unixify'); // tslint:disable-line +import { PAIRestUri, PAIWebPortalUri } from './paiUri'; +import { RecentJobManager } from './recentJobManager'; interface ITokenItem { token: string; expireTime: number; @@ -60,7 +62,6 @@ interface IJobInput { */ @injectable() export class PAIJobManager extends Singleton { - private static readonly API_PREFIX: string = 'api/v1'; private static readonly TIMEOUT: number = 60 * 1000; private static readonly SIMULATION_DOCKERFILE_FOLDER: string = '.pai_simulator'; private static readonly propertiesToBeReplaced: (keyof IPAIJobConfig)[] = [ @@ -84,7 +85,7 @@ export class PAIJobManager extends Singleton { this.context.subscriptions.push( vscode.commands.registerCommand( COMMAND_CREATE_JOB_CONFIG, - async (input?: ConfigurationNode | vscode.Uri) => { + async (input?: ClusterExplorerChildNode | vscode.Uri) => { if (input instanceof vscode.Uri) { await PAIJobManager.generateJobConfig(input.fsPath); } else { @@ -94,10 +95,10 @@ export class PAIJobManager extends Singleton { ), vscode.commands.registerCommand( COMMAND_SIMULATE_JOB, - async (input?: ConfigurationNode | vscode.Uri) => { + async (input?: ClusterExplorerChildNode | vscode.Uri) => { if (input instanceof vscode.Uri) { await this.simulate({ jobConfigPath: input.fsPath }); - } else if (input instanceof ConfigurationNode) { + } else if (input instanceof ClusterExplorerChildNode) { await this.simulate({ clusterIndex: input.index }); } else { await this.simulate(); @@ -106,10 +107,10 @@ export class PAIJobManager extends Singleton { ), vscode.commands.registerCommand( COMMAND_SUBMIT_JOB, - async (input?: ConfigurationNode | vscode.Uri) => { + async (input?: ClusterExplorerChildNode | vscode.Uri) => { if (input instanceof vscode.Uri) { await this.submitJob({ jobConfigPath: input.fsPath }); - } else if (input instanceof ConfigurationNode) { + } else if (input instanceof ClusterExplorerChildNode) { await this.submitJob({ clusterIndex: input.index }); } else { await this.submitJob(); @@ -132,7 +133,7 @@ export class PAIJobManager extends Singleton { parent = fileFolders[0].uri.fsPath; } } - defaultSaveDir = path.join(parent, `${name}.pai.json`); + defaultSaveDir = path.join(parent, `${name}.pai.jsonc`); config = { jobName: '', image: 'aiplatform/pai.build.base', @@ -160,7 +161,7 @@ export class PAIJobManager extends Singleton { } script = path.relative(parent, script); const jobName: string = path.basename(script, path.extname(script)); - defaultSaveDir = path.join(parent, `${jobName}.pai.json`); + defaultSaveDir = path.join(parent, `${jobName}.pai.jsonc`); config = { jobName, image: 'aiplatform/pai.build.base', @@ -181,10 +182,17 @@ export class PAIJobManager extends Singleton { } const saveDir: vscode.Uri | undefined = await vscode.window.showSaveDialog({ - defaultUri: vscode.Uri.file(defaultSaveDir) + defaultUri: vscode.Uri.file(defaultSaveDir), + filters: { + JSON: ['json', 'jsonc'] + } }); if (saveDir) { - await fs.writeJSON(saveDir.fsPath, config, { spaces: 4 }); + if (saveDir.fsPath.endsWith('.jsonc')) { + await fs.writeFile(saveDir.fsPath, await Util.generateCommentedJSON(config, 'pai_job_config.schema.json')); + } else { + await fs.writeJSON(saveDir.fsPath, config, { spaces: 4 }); + } await vscode.window.showTextDocument(saveDir); } } @@ -282,7 +290,7 @@ export class PAIJobManager extends Singleton { param.config.jobName = `${param.config.jobName}_${uuid().substring(0, 8)}`; } else { try { - await request.get(`${this.getRestUrl(param.cluster)}/user/${param.cluster.username}/jobs/${param.config.jobName}`, { + await request.get(PAIRestUri.jobDetail(param.cluster, param.cluster.username, param.config.jobName), { headers: { Authorization: `Bearer ${await this.getToken(param.cluster)}` }, timeout: PAIJobManager.TIMEOUT, json: true @@ -325,21 +333,19 @@ export class PAIJobManager extends Singleton { // send job submission request statusBarItem.text = `${OCTICON_CLOUDUPLOAD} ${__('job.request.status')}`; try { - await request.post(`${this.getRestUrl(param.cluster)}/user/${param.cluster.username}/jobs`, { + await request.post(PAIRestUri.jobs(param.cluster, param.cluster.username), { headers: { Authorization: `Bearer ${await this.getToken(param.cluster)}` }, form: param.config, timeout: PAIJobManager.TIMEOUT, json: true }); + void (await getSingleton(RecentJobManager)).enqueueRecentJobs(param.cluster, param.config.jobName); const open: string = __('job.submission.success.open'); void vscode.window.showInformationMessage( __('job.submission.success'), open ).then(async res => { - const url: string = `${getClusterWebPortalUri(param.cluster!)}/view.html?${querystring.stringify({ - username: param.cluster!.username, - jobName: param.config.jobName - })}`; + const url: string = await PAIWebPortalUri.jobDetail(param.cluster!, param.cluster!.username, param.config.jobName); if (res === open) { await Util.openExternally(url); } @@ -542,6 +548,9 @@ export class PAIJobManager extends Singleton { jobConfigPath = jobConfigUrl![0].fsPath; } const config: IPAIJobConfig = JSONC.parse(await fs.readFile(jobConfigPath, 'utf8')); + if (isNil(config)) { + Util.err('job.prepare.config.invalid'); + } const error: string | undefined = await Util.validateJSON(config, SCHEMA_JOB_CONFIG); if (error) { throw new Error(error); @@ -571,21 +580,11 @@ export class PAIJobManager extends Singleton { return result; } - private getRestUrl(cluster: IPAICluster): string { - let url: string = cluster.rest_server_uri; - if (!url.endsWith('/')) { - url += '/'; - } - url += PAIJobManager.API_PREFIX; - - return Util.fixURL(url); - } - private async getToken(cluster: IPAICluster): Promise { const id: string = getClusterIdentifier(cluster); let item: ITokenItem | undefined = this.cachedTokens.get(id); if (!item || Date.now() > item.expireTime) { - const result: any = await request.post(`${this.getRestUrl(cluster)}/token`, { + const result: any = await request.post(PAIRestUri.token(cluster), { form: { username: cluster.username, password: cluster.password, @@ -618,9 +617,15 @@ export class PAIJobManager extends Singleton { }); const fsProvider: HDFSFileSystemProvider = (await getSingleton(HDFS)).provider!; let codeDir: string = param.config.codeDir; - if (codeDir.startsWith('$PAI_DEFAULT_FS_URI')) { - codeDir = codeDir.substring('$PAI_DEFAULT_FS_URI'.length); + if (codeDir.startsWith('hdfs://') || codeDir.startsWith('webhdfs://')) { + throw new Error(__('job.upload.invalid-code-dir')); + } else { + if (codeDir.startsWith('$PAI_DEFAULT_FS_URI')) { + codeDir = codeDir.substring('$PAI_DEFAULT_FS_URI'.length); + } + codeDir = path.posix.resolve('/', codeDir); } + const codeUri: vscode.Uri = vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(param.cluster!)}${codeDir}`); const total: number = projectFiles.length; @@ -654,7 +659,7 @@ export class PAIJobManager extends Singleton { return true; } catch (e) { - Util.err('job.upload.error', [e]); + Util.err('job.upload.error', [e.message]); return false; } } diff --git a/contrib/pai_vscode/src/pai/paiUri.ts b/contrib/pai_vscode/src/pai/paiUri.ts new file mode 100644 index 0000000000..72b4ea6003 --- /dev/null +++ b/contrib/pai_vscode/src/pai/paiUri.ts @@ -0,0 +1,70 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ + +import * as querystring from 'querystring'; +import * as request from 'request-promise-native'; + +import { Util } from '../common/util'; + +import { IPAICluster } from './paiInterface'; + +export namespace PAIRestUri { + const API_PREFIX: string = 'api/v1'; + + export function getRestUrl(cluster: IPAICluster): string { + let url: string = cluster.rest_server_uri; + if (!url.endsWith('/')) { + url += '/'; + } + url += API_PREFIX; + + return Util.fixURL(url); + } + + export function token(cluster: IPAICluster): string { + return `${getRestUrl(cluster)}/token`; + } + + export function jobDetail(cluster: IPAICluster, username: string, jobName: string): string { + return `${getRestUrl(cluster)}/user/${username}/jobs/${jobName}`; + } + + export function jobs(cluster: IPAICluster, username?: string): string { + if (username) { + return `${getRestUrl(cluster)}/user/${username}/jobs`; + } + return `${getRestUrl(cluster)}/jobs`; + } +} + +export namespace PAIWebPortalUri { + export function getClusterWebPortalUri(conf: IPAICluster): string { + return conf.web_portal_uri || conf.rest_server_uri.split(':')[0]; + } + + export async function isOldJobLinkAvailable(cluster: IPAICluster): Promise { + const link: string = `${getClusterWebPortalUri(cluster)}/view.html`; + try { + await request.get(Util.fixURL(link)); + return true; + } catch { + return false; + } + } + + export async function jobDetail(cluster: IPAICluster, username: string, jobName: string): Promise { + const oldLink: boolean = await isOldJobLinkAvailable(cluster); + return `${getClusterWebPortalUri(cluster)}/${oldLink ? 'view' : 'job-detail'}.html?${querystring.stringify({ + username, + jobName + })}`; + } + + export async function jobs(cluster: IPAICluster): Promise { + const oldLink: boolean = await isOldJobLinkAvailable(cluster); + return `${getClusterWebPortalUri(cluster)}/${oldLink ? 'view' : 'job-list'}.html`; + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/paiWebpages.ts b/contrib/pai_vscode/src/pai/paiWebpages.ts index 4e10233c2d..cc366cdbf7 100644 --- a/contrib/pai_vscode/src/pai/paiWebpages.ts +++ b/contrib/pai_vscode/src/pai/paiWebpages.ts @@ -8,14 +8,17 @@ import { injectable } from 'inversify'; import * as vscode from 'vscode'; import { - COMMAND_LIST_JOB, COMMAND_OPEN_DASHBOARD, COMMAND_TREEVIEW_OPEN_PORTAL + COMMAND_LIST_JOB, COMMAND_OPEN_DASHBOARD, COMMAND_TREEVIEW_OPEN_PORTAL, COMMAND_VIEW_JOB } from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ClusterManager, getClusterName, getClusterWebPortalUri } from './clusterManager'; -import { ConfigurationNode } from './configurationTreeDataProvider'; -import { IPAICluster } from './paiInterface'; + +import { getClusterName, ClusterManager } from './clusterManager'; +import { ClusterExplorerChildNode } from './configurationTreeDataProvider'; +import { ClusterNode } from './container/jobListTreeView'; +import { IPAICluster, IPAIJobInfo } from './paiInterface'; +import { PAIWebPortalUri } from './paiUri'; const paiDashboardPropertyLabelMapping: { [propertyName: string]: string } = { grafana_uri: 'Grafana', @@ -38,11 +41,15 @@ export class PAIWebpages extends Singleton { ), vscode.commands.registerCommand( COMMAND_TREEVIEW_OPEN_PORTAL, - (node: ConfigurationNode) => this.openDashboardFromTreeView(node.index) + (node: ClusterExplorerChildNode) => this.openDashboardFromTreeView(node.index) ), vscode.commands.registerCommand( COMMAND_LIST_JOB, - (node: ConfigurationNode) => this.listJobs(node.index) + (node: ClusterExplorerChildNode | ClusterNode) => this.listJobs(node.index) + ), + vscode.commands.registerCommand( + COMMAND_VIEW_JOB, + this.viewJob.bind(this) ) ); } @@ -55,7 +62,7 @@ export class PAIWebpages extends Singleton { const config: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations![index]; const options: vscode.QuickPickItem[] = []; - const paiUrl: string = getClusterWebPortalUri(config); + const paiUrl: string = PAIWebPortalUri.getClusterWebPortalUri(config); options.push({ label: __('webpage.dashboard.webportal', [getClusterName(config)]), detail: paiUrl @@ -83,13 +90,18 @@ export class PAIWebpages extends Singleton { public async openDashboardFromTreeView(index: number): Promise { const config: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations![index]; - const url: string = getClusterWebPortalUri(config); + const url: string = PAIWebPortalUri.getClusterWebPortalUri(config); await Util.openExternally(url); } public async listJobs(index: number): Promise { const config: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations![index]; - const url: string = getClusterWebPortalUri(config) + '/view.html'; + const url: string = await PAIWebPortalUri.jobs(config); + await Util.openExternally(url); + } + + public async viewJob(jobInfo: IPAIJobInfo, config: IPAICluster): Promise { + const url: string = await PAIWebPortalUri.jobDetail(config, jobInfo.username, jobInfo.name); await Util.openExternally(url); } } diff --git a/contrib/pai_vscode/src/pai/recentJobManager.ts b/contrib/pai_vscode/src/pai/recentJobManager.ts new file mode 100644 index 0000000000..502f0e1b33 --- /dev/null +++ b/contrib/pai_vscode/src/pai/recentJobManager.ts @@ -0,0 +1,84 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ + +import { injectable } from 'inversify'; +import * as vscode from 'vscode'; + +import { + SETTING_JOB_JOBLIST_RECENTJOBSLENGTH, SETTING_SECTION_JOB +} from '../common/constants'; +import { __ } from '../common/i18n'; +import { getSingleton, Singleton } from '../common/singleton'; + +import { IPAICluster } from './paiInterface'; + +import { ClusterManager } from './clusterManager'; +import { JobListTreeDataProvider } from './container/jobListTreeView'; + +/** + * Manager class for cluster configurations + */ +@injectable() +export class RecentJobManager extends Singleton { + private static readonly RECENT_JOBS_KEY: string = 'openpai.recentJobs'; + + private onClusterChangeDisposable: vscode.Disposable | undefined; + private recentJobs: (string[] | undefined)[] | undefined; + + public async onActivate(): Promise { + this.onClusterChangeDisposable = (await getSingleton(ClusterManager)).onDidChange(modification => { + switch (modification.type) { + case 'EDIT': + this.allRecentJobs[modification.index] = []; + break; + case 'REMOVE': + this.allRecentJobs.splice(modification.index, 1); + break; + case 'RESET': + this.recentJobs = []; + break; + default: + } + void this.saveRecentJobs(); + }); + this.recentJobs = this.context.globalState.get(RecentJobManager.RECENT_JOBS_KEY) || []; + } + + public get allRecentJobs(): (string[] | undefined)[] { + return this.recentJobs!; + } + + public async saveRecentJobs(index: number = -1): Promise { + await this.context.globalState.update(RecentJobManager.RECENT_JOBS_KEY, this.allRecentJobs); + const provider: JobListTreeDataProvider = await getSingleton(JobListTreeDataProvider); + await provider.refresh(index); + if (index !== -1) { + const latestJobName: string | undefined = (this.allRecentJobs[index] || [])[0]; + if (latestJobName) { + await provider.revealLatestJob(index, latestJobName); + } + } + } + + public async enqueueRecentJobs(cluster: IPAICluster, jobName: string): Promise { + const index: number = (await getSingleton(ClusterManager)).allConfigurations.findIndex(c => c === cluster); + if (index === -1) { + return; + } + const list: string[] = this.allRecentJobs[index] = this.allRecentJobs[index] || []; + list.unshift(jobName); + const settings: vscode.WorkspaceConfiguration = vscode.workspace.getConfiguration(SETTING_SECTION_JOB); + const maxLen: number = settings.get(SETTING_JOB_JOBLIST_RECENTJOBSLENGTH)!; + list.splice(maxLen); // Make sure not longer than maxLen + await this.saveRecentJobs(index); + } + + public onDeactivate(): void { + if (this.onClusterChangeDisposable) { + this.onClusterChangeDisposable.dispose(); + } + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/root.ts b/contrib/pai_vscode/src/root.ts index 954f6d76ba..c1ca6856e4 100644 --- a/contrib/pai_vscode/src/root.ts +++ b/contrib/pai_vscode/src/root.ts @@ -5,18 +5,26 @@ */ import { Singleton } from './common/singleton'; +import { TreeViewHelper } from './common/treeViewHelper'; import { UtilClass } from './common/util'; import { ClusterManager } from './pai/clusterManager'; import { ConfigurationTreeDataProvider } from './pai/configurationTreeDataProvider'; +import { HDFSTreeDataProvider } from './pai/container/hdfsTreeView'; +import { JobListTreeDataProvider } from './pai/container/jobListTreeView'; import { HDFS } from './pai/hdfs'; import { PAIJobManager } from './pai/paiJobManager'; import { PAIWebpages } from './pai/paiWebpages'; +import { RecentJobManager } from './pai/recentJobManager'; export const allSingletonClasses: { new(...arg: any[]): Singleton }[] = [ UtilClass, ClusterManager, + RecentJobManager, + TreeViewHelper, ConfigurationTreeDataProvider, PAIJobManager, PAIWebpages, - HDFS + HDFS, + HDFSTreeDataProvider, + JobListTreeDataProvider ]; \ No newline at end of file diff --git a/contrib/pai_vscode/src/test/clusterConfiguration.test.ts b/contrib/pai_vscode/src/test/clusterConfiguration.test.ts index f855c6d668..631f87a125 100644 --- a/contrib/pai_vscode/src/test/clusterConfiguration.test.ts +++ b/contrib/pai_vscode/src/test/clusterConfiguration.test.ts @@ -5,6 +5,7 @@ */ // tslint:disable:align import * as assert from 'assert'; + import { getSingleton } from '../common/singleton'; import { ClusterManager } from '../pai/clusterManager'; diff --git a/contrib/pai_vscode/tslint.json b/contrib/pai_vscode/tslint.json index 690b0fc847..1ee738e7f3 100644 --- a/contrib/pai_vscode/tslint.json +++ b/contrib/pai_vscode/tslint.json @@ -141,7 +141,7 @@ "no-inferrable-types": false, "no-multiline-string": true, "no-null-keyword": false, - "no-parameter-properties": false, + "no-parameter-properties": true, "no-relative-imports": false, "no-require-imports": true, "no-shadowed-variable": true, @@ -158,7 +158,13 @@ "object-literal-sort-keys": false, "one-variable-per-declaration": true, "only-arrow-functions": false, - "ordered-imports": true, + "ordered-imports": [ + true, + { + "named-imports-order": "lowercase-first", + "grouped-imports": true + } + ], "prefer-array-literal": false, "prefer-const": true, "prefer-for-of": true, @@ -201,7 +207,8 @@ "import-spacing": true, "indent": [ true, - "spaces" + "spaces", + 4 ], "linebreak-style": true, "newline-before-return": false,