Add sketch of WASM integration

package.json

@@ -4,6 +4,7 @@
   "author": "Wolfgang Beyer",
   "license": "MIT",
   "dependencies": {
+    "@bjorn3/browser_wasi_shim": "^0.2.17",
     "@emotion/react": "^11.11.0",
     "@emotion/styled": "^11.11.0",
     "@fortawesome/fontawesome-svg-core": "^6.4.0",
@@ -28,10 +29,12 @@
     "es-dirname": "^0.1.0",
     "express": "^4.18.2",
     "fs-extra": "^10.1.0",
+    "gbz-base": "^0.1.0-alpha.0",
     "gh-pages": "^4.0.0",
     "markdown-to-jsx": "^7.2.0",
     "multer": "^1.4.5-lts.1",
     "node-cron": "^3.0.2",
+    "patch-package": "^8.0.0",
     "path-is-inside": "^1.0.2",
     "polyfill-object.fromentries": "^1.0.1",
     "prop-types": "^15.8.1",
@@ -63,7 +66,8 @@
     "predeploy": "npm run build",
     "deploy": "gh-pages -d build",
     "serve": "node ./src/server.mjs",
-    "format": "prettier --write \"**/*.+(mjs|js|css)\""
+    "format": "prettier --write \"**/*.+(mjs|js|css)\"",
+    "postinstall": "patch-package"
   },
   "eslintConfig": {
     "extends": "react-app"
@@ -86,7 +90,7 @@
   "jest": {
     "resetMocks": false,
     "transformIgnorePatterns": [
-      "node_modules/(?!(@streamparser/json)/)"
+      "node_modules/(?!(@streamparser/json|@bjorn3/browser_wasi_shim)/)"
     ]
   }
 }

patches/@bjorn3+browser_wasi_shim+0.2.17.patch

@@ -0,0 +1,14 @@
+diff --git a/node_modules/@bjorn3/browser_wasi_shim/package.json b/node_modules/@bjorn3/browser_wasi_shim/package.json
+index af9de55..2e7a121 100644
+--- a/node_modules/@bjorn3/browser_wasi_shim/package.json
++++ b/node_modules/@bjorn3/browser_wasi_shim/package.json
+@@ -21,7 +21,8 @@
+   "exports": {
+     ".": {
+       "types": "./typings/index.d.ts",
+-      "import": "./dist/index.js"
++      "import": "./dist/index.js",
++      "default": "./dist/index.js"
+     }
+   },
+   "typings": "./typings/index.d.ts",

src/APIInterface.mjs

@@ -12,11 +12,29 @@ export class APIInterface {
 
   // Returns files used to determine what options are available in the track picker.
   // Returns object with keys: files, bedFiles.
+  // files holds an array of objects like {  name: string; type: filetype;}, where filetype is a file type like "graph".
+  // bedFiles just holds an array of strings.
   // cancelSignal is an AbortSignal that can be used to cancel the request.
   async getFilenames(cancelSignal) {
     throw new Error("getFilenames function not implemented");
   }
 
+  // Get notifications (via calls to handler()) when the set of filenames available from getFilenames() has changed.
+  // Returns a subscription object that should be kept around as long as you still want updates.
+  // cancelSignal is an AbortSignal that can be used to cancel the stream of notifications.
+  subscribeToFilenameChanges(handler, cancelSignal) {
+    throw new Error("subscribeToFilenameChanges function not implemented");
+  }
+
+  // Upload a file.
+  // fileType is a track type like "graph" or "read".
+  // file is the file data (Blob or File).
+  // cancelSignal is an AbortSignal that can be used to cancel the upload.
+  // Resolves with the file name that can be used to refer to the uploaded file.
+  async putFile(fileType, file, cancelSignal) {
+    throw new Error("putFile function not implemented");
+  }
+
   // Takes in a bedfile path or a url pointing to a raw bed file.
   // Returns object with key: bedRegions.
   // bedRegions contains information extrapolated from each line of the bedfile.

src/App.js

@@ -15,6 +15,7 @@ import { dataOriginTypes } from "./enums";
 import "./config-client.js";
 import { config } from "./config-global.mjs";
 import ServerAPI from "./ServerAPI.mjs";
+import { GBZBaseAPI } from "./GBZBaseAPI.mjs";
 
 const EXAMPLE_TRACKS = [
   // Fake tracks for the generated examples.
@@ -46,6 +47,17 @@ class App extends Component {
   constructor(props) {
     super(props);
 
+    // See if the WASM API is available.
+    // Right now this just tests and logs, but eventually we will be able to use it.
+    let gbzApi = new GBZBaseAPI();
+    gbzApi.available().then((working) => {
+      if (working) {
+        console.log("WASM API implementation available!");
+      } else {
+        console.error("WASM API implementation not available!");
+      }
+    });
+
     this.APIInterface = new ServerAPI(props.apiUrl);
 
     console.log("App component starting up with API URL: " + props.apiUrl);
@@ -183,15 +195,13 @@ class App extends Component {
           setDataOrigin={this.setDataOrigin}
           setColorSetting={this.setColorSetting}
           dataOrigin={this.state.dataOrigin}
-          apiUrl={this.props.apiUrl}
           defaultViewTarget={this.defaultViewTarget}
           getCurrentViewTarget={this.getCurrentViewTarget}
           APIInterface={this.APIInterface}
         />
         <TubeMapContainer
           viewTarget={this.state.viewTarget}
           dataOrigin={this.state.dataOrigin}
-          apiUrl={this.props.apiUrl}
           visOptions={this.state.visOptions}
           APIInterface={this.APIInterface}
         />

src/GBZBaseAPI.mjs

@@ -0,0 +1,210 @@
+import { APIInterface } from "./APIInterface.mjs";
+import { WASI, File, OpenFile } from "@bjorn3/browser_wasi_shim";
+
+// TODO: The Webpack way to get the WASM would be something like:
+//import QueryWasm from "gbz-base/target/wasm32-wasi/release/query.wasm";
+// if the export mapping is broken, or
+//import QueryWasm from "gbz-base/query.wasm";
+// if it is working. In Jest, not only is the export mapping not working, but
+// also it can't get us a fetch-able string from the import like Webpack does.
+// So we will need some fancy Jest config to mock the WASM file into a js
+// module that does *something*, and also to mock fetch() into something that
+// can fetch it. Or else we need to hide that all behind something that can
+// fetch the WASM on either Webpack or Jest with its own strategies/by being
+// swapped out.
+
+// Resolve with the bytes or Response of the WASM query blob, on Jest or Webpack.
+async function getWasmBytes() {
+  let blobBytes = null;
+
+  if (!window["jest"]) {
+    // Not running on Jest, we should be able to dynamic import a binary asset
+    // by export name and get the bytes, and Webpack will handle it.
+    try {
+      let blobImport = await import("gbz-base/query.wasm");
+      return fetch(blobImport.default);
+    } catch (e) {
+      console.error("Could not dynamically import WASM blob.", e);
+      // Leave blobBytes unset to try a fallback method.
+    }
+  }
+
+  if (!blobBytes) {
+    // Either we're on Jest, or the dynamic import didn't work (maybe we're on
+    // plain Node?).
+    //
+    // Try to open the file from the filesystem.
+    //
+    // Don't actually try and ship the filesystem module in the browser though:
+    // see <https://webpack.js.org/api/module-methods/#webpackignore>
+    let fs = await import(/* webpackIgnore: true */ "fs-extra");
+    blobBytes = await fs.readFile("node_modules/gbz-base/target/wasm32-wasi/release/query.wasm");
+  } 
+
+  console.log("Got blob bytes: ", blobBytes);
+  return blobBytes;
+}
+
+/**
+ * API implementation that uses tools compiled to WebAssembly, client-side.
+ */
+export class GBZBaseAPI extends APIInterface {
+  constructor() {
+    super();
+
+    // We can take user uploads, in which case we need to hold on to them somewhere.
+    // This holds all the file objects.
+    this.files = [];
+
+    // We need to index all their names by type.
+    this.filesByType = {};
+
+    // This is a promise for the compiled WebAssembly blob.
+    this.compiledWasm = undefined;
+  }
+
+  // Make sure our WASM backend is ready.
+  async setUp() {
+    if (this.compiledWasm === undefined) {
+      // Kick off and save exactly one request to get and load the WASM bytes.
+      this.compiledWasm = getWasmBytes().then((result) => {
+        if (result instanceof Response) {
+          // If a fetch request was made, compile as it streams in
+          return WebAssembly.compileStreaming(result);
+        } else {
+          // We have all the bytes, so compile right away.
+          // TODO: Put this logic in the function?
+          return WebAssembly.compile(result);
+        }
+      });
+    }
+
+    // Wait for the bytes to be available.
+    this.compiledWasm = await this.compiledWasm;
+  }
+
+  // Make a call into the WebAssembly code and return the result.
+  async callWasm(argv) {
+    if (argv.length < 1) {
+      // We need at least one command line argument to be the program name.
+      throw new Error("Not safe to invoke main() without program name");
+    }
+
+    // Make sure this.compiledWasm is set.
+    // TODO: Change to an accessor method?
+    await this.setUp();
+
+    // Define the places to store program input and output
+    let stdin = new File([]);
+    let stdout = new File([]);
+    let stderr = new File([]);
+
+    // Environment variables as NAME=value strings
+    const environment = ["RUST_BACKTRACE=full"];
+
+    // File descriptors for the process in number order
+    let file_descriptors = [new OpenFile(stdin), new OpenFile(stdout), new OpenFile(stderr)];
+
+    // Set up the WASI interface
+    let wasi = new WASI(argv, environment, file_descriptors);
+
+    // Set up the WebAssembly run
+    let instantiation = await WebAssembly.instantiate(this.compiledWasm, {
+        "wasi_snapshot_preview1": wasi.wasiImport,
+    });
+
+    try {
+      // Make the WASI system call main
+      let returnCode = wasi.start(instantiation);
+      console.log("Return code:", returnCode);
+    } finally {
+      // The WASM code can throw right out of the WASI shim if Rust panics.
+      console.log("Standard Output:", new TextDecoder().decode(stdout.data));
+      console.log("Standard Error:", new TextDecoder().decode(stderr.data));
+    }
+  }
+
+  // Return true if the WASM setup is working, and false otherwise.
+  async available() {
+    try {
+      await this.callWasm(["query", "--help"]);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /////////
+  // Tube Map API implementation
+  /////////
+
+  async getChunkedData(viewTarget, cancelSignal) {
+    return {
+      graph: {},
+      gam: {},
+      region: null,
+      coloredNodes: [],
+    };
+  }
+
+  async getFilenames(cancelSignal) {
+    // Set up an empty response.
+    let response = {
+      files: [],
+      bedFiles: [],
+    };
+
+    for (let type of this.filesByType) {
+      if (type === "bed") {
+        // Just send all these files in bedFiles.
+        response.bedFiles = this.filesByType[type];
+      } else {
+        for (let fileName of this.filesByType[type]) {
+          // We sens a name/type record for each non-BED file
+          response.files.push({ name: fileName, type: type });
+        }
+      }
+    }
+
+    return response;
+  }
+
+  subscribeToFilenameChanges(handler, cancelSignal) {
+    return {};
+  }
+
+  async putFile(fileType, file, cancelSignal) {
+    // We track files just by array index.
+    let fileName = this.files.length.toString();
+    // Just hang on to the File object.
+    this.files.push(file);
+
+    if (this.filesByType[fileType] === undefined) {
+      this.filesByType[fileType] = [];
+    }
+    // Index the name we produced by type.
+    this.filesByType[fileType].push(fileName);
+
+    return fileName;
+  }
+
+  async getBedRegions(bedFile, cancelSignal) {
+    return {
+      bedRegions: [],
+    };
+  }
+
+  async getPathNames(graphFile, cancelSignal) {
+    return {
+      pathNames: [],
+    };
+  }
+
+  async getChunkTracks(bedFile, chunk, cancelSignal) {
+    return {
+      tracks: [],
+    };
+  }
+}
+
+export default GBZBaseAPI;

src/GBZBaseAPI.test.js

@@ -0,0 +1,38 @@
+import { GBZBaseAPI } from "./GBZBaseAPI.mjs";
+
+import fs from "fs-extra";
+
+it("can be constructed", () => {
+  let api = new GBZBaseAPI();
+});
+
+it("can self-test its WASM setup", async () => {
+  let api = new GBZBaseAPI();
+  let working = await api.available();
+  expect(working).toBeTruthy();
+});
+
+it("can have a file uploaded", async () => {
+  let api = new GBZBaseAPI();
+
+  // We need to make sure we make a jsdom File (which is a jsdom Blob), and not
+  // a Node Blob, for our test file. Otherwise it doesn't work with jsdom's
+  // upload machinery.
+  // See for example <https://github.com/vitest-dev/vitest/issues/2078> for
+  // background on the many flavors of Blob.
+  const fileData = await fs.readFileSync("exampleData/cactus.vg");
+  // Since a Node Buffer is an ArrayBuffer, we can use it to make a jsdom File.
+  // We need to put the data block in an enclosing array, or else the block
+  // will be iterated and each byte will be stringified and *those* bytes will
+  // be uploaded.
+  const file = new window.File([fileData], "cactus.vg", {
+    type: "application/octet-stream",
+  });
+
+  // Set up for canceling the upload
+  let controller = new AbortController();
+
+  let uploadName = await api.putFile("graph", file, controller.signal);
+
+  expect(uploadName).toBeTruthy();
+});