Allow hooking into the filesystem and providing a persistent backend (like absurd-sql)

Makefile

@@ -62,7 +62,7 @@ EMFLAGS_DEBUG = \
 	-s ASSERTIONS=1 \
 	-O1
 
-BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc
+BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc out/vfs.bc
 
 OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js
 
@@ -73,19 +73,21 @@ EMFLAGS_PRE_JS_FILES = \
 
 EXPORTED_METHODS_JSON_FILES = src/exported_functions.json src/exported_runtime_methods.json
 
+FS_EXTERN_PATH = "$(realpath -s ./src/fs-externs.js)"
+
 all: optimized debug worker
 
 .PHONY: debug
 debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js
 
 dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
-	$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
+	EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
 	mv $@ out/tmp-raw.js
 	cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
 	rm out/tmp-raw.js
 
 dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
-	$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
+	EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
 	mv $@ out/tmp-raw.js
 	cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
 	rm out/tmp-raw.js
@@ -94,19 +96,19 @@ dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FI
 optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js
 
 dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
-	$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
+	EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
 	mv $@ out/tmp-raw.js
 	cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
 	rm out/tmp-raw.js
 
 dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
-	$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
+	EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
 	mv $@ out/tmp-raw.js
 	cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
 	rm out/tmp-raw.js
 
 dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
-	$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
+	EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
 	mv $@ out/tmp-raw.js
 	cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
 	rm out/tmp-raw.js
@@ -156,6 +158,11 @@ out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION)
 	# Generate llvm bitcode
 	$(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
 
+out/vfs.bc: src/vfs.c sqlite-src/$(SQLITE_AMALGAMATION)
+	mkdir -p out
+	# Generate llvm bitcode
+	$(EMCC) $(CFLAGS) -s LINKABLE=1 -I sqlite-src/$(SQLITE_AMALGAMATION) -c src/vfs.c -o $@
+
 # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
 # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js
 # 	tar --create --gzip $^ > $@

src/api.js

@@ -811,13 +811,20 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
     * @memberof module:SqlJs
     * Open a new database either by creating a new one or opening an existing
     * one stored in the byte array passed in first argument
-    * @param {number[]} data An array of bytes representing
-    * an SQLite database file
+    * @param {number[]|string} data An array of bytes representing
+    * an SQLite database file or a path
+    * @param {Object} opts Options to specify a filename
     */
-    function Database(data) {
-        this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
-        if (data != null) {
+    function Database(data, { filename = false } = {}) {
+        if(filename === false) {
+          this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
+          this.memoryFile = true;
+          if (data != null) {
             FS.createDataFile("/", this.filename, data, true, true);
+          }
+        }
+        else {
+          this.filename = data;
         }
         this.handleError(sqlite3_open(this.filename, apiTemp));
         this.db = getValue(apiTemp, "i32");
@@ -1103,7 +1110,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         Object.values(this.functions).forEach(removeFunction);
         this.functions = {};
         this.handleError(sqlite3_close_v2(this.db));
-        FS.unlink("/" + this.filename);
+
+        if(this.memoryFile) {
+          FS.unlink("/" + this.filename);
+        }
         this.db = null;
     };
 
@@ -1231,4 +1241,49 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
 
     // export Database to Module
     Module.Database = Database;
+
+    // Because emscripten doesn't allow us to handle `ioctl`, we need
+    // to manually install lock/unlock methods. Unfortunately we need
+    // to keep track of a mapping of `sqlite_file*` pointers to filename
+    // so that we can tell our filesystem which files to lock/unlock
+    var sqliteFiles = new Map();
+
+    Module["register_for_idb"] = (customFS) => {
+      var SQLITE_BUSY = 5;
+
+      function open(namePtr, file) {
+        var path = UTF8ToString(namePtr);
+        sqliteFiles.set(file, path);
+      }
+
+      function lock(file, lockType) {
+        var path = sqliteFiles.get(file);
+        var success = customFS.lock(path, lockType)
+        return success? 0 : SQLITE_BUSY;
+      }
+
+      function unlock(file,lockType) {
+        var path = sqliteFiles.get(file);
+        customFS.unlock(path, lockType)
+        return 0;
+      }
+
+      let lockPtr = addFunction(lock, 'iii');
+      let unlockPtr = addFunction(unlock, 'iii');
+      let openPtr = addFunction(open, 'vii');
+      Module["_register_for_idb"](lockPtr, unlockPtr, openPtr)
+    }
+
+    // TODO: This isn't called from anywhere yet. We need to
+    // somehow cleanup closed files from `sqliteFiles`
+    Module["cleanup_file"] = (path) => {
+      let filesInfo = [...sqliteFiles.entries()]
+      let fileInfo = filesInfo.find(f => f[1] === path);
+      sqliteFiles.delete(fileInfo[0])
+    }
+
+    Module["reset_filesystem"] = () => {
+      FS.root = null;
+      FS.staticInit();
+    }
 };

src/exported_functions.json

@@ -41,5 +41,7 @@
 "_sqlite3_result_int",
 "_sqlite3_result_int64",
 "_sqlite3_result_error",
+"_sqlite3_vfs_find",
+"_register_for_idb",
 "_RegisterExtensionFunctions"
 ]

src/exported_runtime_methods.json

@@ -3,5 +3,6 @@
 "stackAlloc",
 "stackSave",
 "stackRestore",
-"UTF8ToString"
+"UTF8ToString",
+"FS"
 ]

src/fs-externs.js

@@ -0,0 +1,48 @@
+/**
+ * @externs
+ */
+
+Module.FS = class {
+  constructor() {
+    this.ErrnoError = class {};
+  }
+  mount() {}
+  isRoot() {}
+  isFile() {}
+  isDir() {}
+  stat() {}
+  /** @return {FSNode} */
+  lookupPath() {}
+  /** @return {FSNode} */
+  lookupNode() {}
+  /** @return {FSNode} */
+  createNode() {}
+  /** @return {FSNode} */
+  mknod() {}
+};
+
+Module.FS.FSNode = class {
+  constructor() {
+    this.node_ops = {
+      getattr: () => {},
+      setattr: () => {},
+      lookup: () => {},
+      mknod: () => {},
+      rename: () => {},
+      unlink: () => {},
+      rmdir: () => {},
+      reaaddir: () => {},
+      symlink: () => {},
+      readlink: () => {}
+    };
+
+    this.stream_ops = {
+      llseek: () => {},
+      read: () => {},
+      write: () => {},
+      allocate: () => {},
+      mmap: () => {},
+      msync: () => {}
+    };
+  }
+};

src/vfs.c

@@ -0,0 +1,47 @@
+#include <sqlite3.h>
+#include <stdio.h>
+
+static int (*defaultOpen)(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags);
+
+static void (*fsOpen)(const char *, void*);
+static int (*fsLock)(sqlite3_file *file, int);
+static int (*fsUnlock)(sqlite3_file *file, int);
+
+static int blockDeviceCharacteristics(sqlite3_file* file) {
+    return SQLITE_IOCAP_SAFE_APPEND |
+        SQLITE_IOCAP_SEQUENTIAL |
+        SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
+}
+
+static int block_lock(sqlite3_file *file, int lock) {
+    return fsLock(file, lock);
+}
+
+static int block_unlock(sqlite3_file *file, int lock) {
+    return fsUnlock(file, lock);
+}
+
+static int block_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags) {
+    int res = defaultOpen(vfs, zName, file, flags, pOutFlags);
+
+    sqlite3_io_methods* methods = (sqlite3_io_methods*)file->pMethods;
+    methods->xDeviceCharacteristics = blockDeviceCharacteristics;
+    methods->xLock = block_lock;
+    methods->xUnlock = block_unlock;
+
+    fsOpen(zName, (void*)file);
+
+    return res;
+}
+
+void register_for_idb(int(*lockFile)(sqlite3_file*,int), int(*unlockFile)(sqlite3_file*,int), void(*openFile)(const char*, void*)) {
+    sqlite3_vfs *vfs = sqlite3_vfs_find("unix");
+    defaultOpen = vfs->xOpen;
+
+    vfs->xOpen = block_open;
+    sqlite3_vfs_register(vfs, 1);
+
+    fsLock = lockFile;
+    fsUnlock = unlockFile;
+    fsOpen = openFile;
+}