From 60e6b3e73f25e07d152638f2e99e69d0bd5aaba1 Mon Sep 17 00:00:00 2001 From: Jayaram Kancherla Date: Tue, 23 Apr 2024 07:50:44 -0700 Subject: [PATCH] Show progress bars when downloading assets, update README (#257) * Display download progress for remotely hosted assets * Fix race condition between inputs and available options, avoiding duplicate downloads of the same asset * Show logs for download progress * Update README * Bump app version --- README.md | 17 ++++++ package.json | 2 +- src/DownloadToaster.js | 60 +++++++++++++++++++ src/components/AnalysisMode/index.js | 65 ++++++++++++++++++++- src/workers/helpers.js | 87 +++++++++++++++++++++++++++- src/workers/scran.worker.js | 3 +- 6 files changed, 227 insertions(+), 7 deletions(-) create mode 100644 src/DownloadToaster.js diff --git a/README.md b/README.md index a2590ef3..e93bf6c9 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,23 @@ and diagnostic plots from the individual analysis steps. - Clicking on "What's happening" will show logs describing how long each step of the analysis took (and any errors during the analysis). - Clicking Export will save the analysis either to the browser or download the analysis as a .kana file. Loading these files will restore the state of the application +If you use **Kana** for analysis or exploration, consider citing our JOSS publication - + +```bibtex +@article{Kana2023, + doi = {10.21105/joss.05603}, + url = {https://doi.org/10.21105/joss.05603}, + year = {2023}, + publisher = {The Open Journal}, + volume = {8}, + number = {89}, + pages = {5603}, + author = {Aaron Tin Long Lun and Jayaram Kancherla}, + title = {Powering single-cell analyses in the browser with WebAssembly}, + journal = {Journal of Open Source Software} +} +``` + ## For developers ***Check out [Contributing](./CONTRIBUTING.md) for guidelines on opening issues and pull requests.*** diff --git a/package.json b/package.json index 7ec6ae68..745fb445 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "kana", "description": "Single-cell data analysis in the browser", - "version": "3.0.22", + "version": "3.0.23", "author": { "name": "Jayaram Kancherla", "email": "jayaram.kancherla@gmail.com", diff --git a/src/DownloadToaster.js b/src/DownloadToaster.js new file mode 100644 index 00000000..28ef2026 --- /dev/null +++ b/src/DownloadToaster.js @@ -0,0 +1,60 @@ +import { + OverlayToaster, + Position, + ProgressBar, + Classes, +} from "@blueprintjs/core"; + +import { Tooltip2 } from "@blueprintjs/popover2"; + +import classNames from "classnames"; + +export const DownloadToaster = OverlayToaster.create({ + className: "recipe-toaster", + position: Position.TOP_RIGHT, +}); + +let download_toasters = {}; + +export function setProgress(id, total, progress) { + if (total !== null) { + download_toasters["total"] = total; + download_toasters["progress"] = progress; + } + + if (progress !== null) { + let tprogress = + (Math.round((progress * 100) / download_toasters["total"]) / 100) * 100; + + download_toasters["progress"] = tprogress; + } +} + +export function renderProgress(progress, url) { + return { + icon: "cloud-download", + message: ( + <> + <> + Downloading asset from{" "} + {url}} + minimal={true} + usePortal={false} + > + {new URL(url).hostname} + + + = 100, + })} + intent={progress < 100 ? "primary" : "success"} + value={progress / 100} + /> + + ), + timeout: progress < 100 ? 0 : 1000, + }; +} diff --git a/src/components/AnalysisMode/index.js b/src/components/AnalysisMode/index.js index 3d3ba119..324caa4a 100644 --- a/src/components/AnalysisMode/index.js +++ b/src/components/AnalysisMode/index.js @@ -42,6 +42,8 @@ import Gallery from "../Gallery/index"; import { AppToaster } from "../../AppToaster"; +import { renderProgress, DownloadToaster } from "../../DownloadToaster"; + import { AppContext } from "../../context/AppContext"; import pkgVersion from "../../../package.json"; @@ -59,6 +61,8 @@ const scranWorker = new Worker( let logs = []; +let download_toasters = {}; + export function AnalysisMode(props) { // true until wasm is initialized const [loading, setLoading] = useState(true); @@ -135,6 +139,8 @@ export function AnalysisMode(props) { const [initDims, setInitDims] = useState(null); const [inputData, setInputData] = useState(null); + const [preInputDone, setPreInputDone] = useState(false); + // STEP: QC; for all three, RNA, ADT, CRISPR // dim sizes const [qcDims, setQcDims] = useState(null); @@ -307,6 +313,7 @@ export function AnalysisMode(props) { useEffect(() => { if (wasmInitialized && preInputFiles) { if (preInputFiles.files) { + setPreInputDone(false); scranWorker.postMessage({ type: "PREFLIGHT_INPUT", payload: { @@ -318,7 +325,7 @@ export function AnalysisMode(props) { }, [preInputFiles, wasmInitialized]); useEffect(() => { - if (wasmInitialized && preInputFiles) { + if (wasmInitialized && preInputFiles && preInputDone && preInputOptions) { if (preInputFiles.files && preInputOptions.options.length > 0) { scranWorker.postMessage({ type: "PREFLIGHT_OPTIONS", @@ -329,7 +336,7 @@ export function AnalysisMode(props) { }); } } - }, [preInputOptions, wasmInitialized]); + }, [preInputOptions, wasmInitialized, preInputDone]); // NEW analysis: files are imported into Kana useEffect(() => { @@ -854,7 +861,7 @@ export function AnalysisMode(props) { scranWorker.onmessage = (msg) => { const payload = msg.data; - // console.log("ON MAIN::RCV::", payload); + // console.log("IN ANALYSIS MODE, ON MAIN::RCV::", payload); // process any error messages if (payload) { @@ -904,6 +911,57 @@ export function AnalysisMode(props) { } } + if (payload.type.startsWith("DOWNLOAD")) { + if (payload.download == "START") { + download_toasters[payload.url] = { + total: payload.total_bytes, + progress: 0, + }; + download_toasters[payload.url]["key"] = DownloadToaster.show( + renderProgress(0, payload.url) + ); + + add_to_logs("start", `Download asset from ${payload.url}`, "started"); + } else if (payload.download == "PROGRESS") { + let tprogress = + (Math.round( + (payload.downloaded_bytes * 100) / + download_toasters[payload.url]["total"] + ) / + 100) * + 100; + + if (tprogress < 100) { + download_toasters[payload.url]["progress"] = tprogress; + + download_toasters[payload.url]["key"] = DownloadToaster.show( + renderProgress(tprogress, payload.url), + download_toasters[payload.url]["key"] + ); + } + add_to_logs("progress", `Downloading ${tprogress}% done`, ""); + } else if (payload.download == "COMPLETE") { + download_toasters[payload.url]["progress"] = 100; + + download_toasters[payload.url]["key"] = DownloadToaster.show( + renderProgress(100, payload.url), + download_toasters[payload.url]["key"] + ); + + add_to_logs( + "complete", + `Asset downloaded from ${payload.url}`, + "finished" + ); + + setTimeout(() => { + delete download_toasters[payload.url]; + }, 500); + } + + return; + } + const { resp, type } = payload; if (type === "INIT") { @@ -927,6 +985,7 @@ export function AnalysisMode(props) { } else if (type === "PREFLIGHT_INPUT_DATA") { if (resp.details) { setPreInputFilesStatus(resp.details); + setPreInputDone(true); } } else if (type === "PREFLIGHT_OPTIONS_DATA") { if (resp) { diff --git a/src/workers/helpers.js b/src/workers/helpers.js index 55b564e8..fbf44897 100644 --- a/src/workers/helpers.js +++ b/src/workers/helpers.js @@ -6,8 +6,91 @@ import * as downloads from "./DownloadsDBHandler.js"; // Evade CORS problems and enable caching. const proxy = "https://cors-proxy.aaron-lun.workers.dev"; async function proxyAndCache(url) { - let buffer = await downloads.get(proxy + "/" + encodeURIComponent(url)); - return new Uint8Array(buffer); + const url_with_proxy = proxy + "/" + encodeURIComponent(url); + + try { + const out = await fetchWithProgress( + url_with_proxy, + (cl) => { + postMessage({ + type: `DOWNLOAD for url: ` + String(url), + download: "START", + url: String(url), + total_bytes: String(cl), + msg: "Total size is " + String(cl) + " bytes!", + }); + return url_with_proxy; + }, + (id, sofar) => { + postMessage({ + type: `DOWNLOAD for url: ` + String(url), + download: "PROGRESS", + url: String(url), + downloaded_bytes: String(sofar), + msg: "Progress so far, got " + String(sofar) + " bytes!", + }); + }, + (id, total) => { + postMessage({ + type: `DOWNLOAD for url: ` + String(url), + download: "COMPLETE", + url: String(url), + msg: "Finished, got " + String(total) + " bytes!", + }); + } + ); + + return out; + } catch (error) { + // console.log("oops error", error) + postMessage({ + type: `DOWNLOAD for url: ` + String(url), + download: "START", + url: String(url), + total_bytes: 100, + }); + let buffer = await downloads.get(url_with_proxy); + postMessage({ + type: `DOWNLOAD for url: ` + String(url), + download: "COMPLETE", + url: String(url), + }); + return new Uint8Array(buffer); + } +} + +async function fetchWithProgress(url, startFun, iterFun, endFun) { + const res = await fetch(url); + if (!res.ok) { + throw new Error("oops, failed to download '" + url + "'"); + } + + const cl = res.headers.get("content-length"); // WARNING: this might be NULL! + const id = startFun(cl); + + const reader = res.body.getReader(); + const chunks = []; + let total = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + total += value.length; + iterFun(id, total); + } + + let output = new Uint8Array(total); + let start = 0; + for (const x of chunks) { + output.set(x, start); + start += x.length; + } + + endFun(id, total); + return output; } bakana.CellLabellingState.setDownload(proxyAndCache); diff --git a/src/workers/scran.worker.js b/src/workers/scran.worker.js index cf8dbb8a..cb302a34 100644 --- a/src/workers/scran.worker.js +++ b/src/workers/scran.worker.js @@ -16,6 +16,7 @@ import { fetchStepSummary, describeColumn, isArrayOrView, + fetchWithProgress, } from "./helpers.js"; import { code } from "../utils/utils.js"; /***************************************/ @@ -306,7 +307,7 @@ var loaded; onmessage = function (msg) { const { type, payload } = msg.data; - // console.log("WORKER::RCV::", type, payload); + // console.log("SCRAN.WORKER ::RCV::", type, payload); let fatal = false; if (type === "INIT") {