diff --git a/Makefile b/Makefile index a4ce5ebc..a70c27c4 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ EMFLAGS_DEBUG = \ -s ASSERTIONS=1 \ -O1 -BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc +BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc out/vfs.bc OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js @@ -73,19 +73,21 @@ EMFLAGS_PRE_JS_FILES = \ EXPORTED_METHODS_JSON_FILES = src/exported_functions.json src/exported_runtime_methods.json +FS_EXTERN_PATH = "$(realpath -s ./src/fs-externs.js)" + all: optimized debug worker .PHONY: debug debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js @@ -94,19 +96,19 @@ dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FI optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js @@ -156,6 +158,11 @@ out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION) # Generate llvm bitcode $(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ +out/vfs.bc: src/vfs.c sqlite-src/$(SQLITE_AMALGAMATION) + mkdir -p out + # Generate llvm bitcode + $(EMCC) $(CFLAGS) -s LINKABLE=1 -I sqlite-src/$(SQLITE_AMALGAMATION) -c src/vfs.c -o $@ + # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js # tar --create --gzip $^ > $@ diff --git a/src/api.js b/src/api.js index 854bf32e..d2a20dfb 100644 --- a/src/api.js +++ b/src/api.js @@ -811,13 +811,20 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { * @memberof module:SqlJs * Open a new database either by creating a new one or opening an existing * one stored in the byte array passed in first argument - * @param {number[]} data An array of bytes representing - * an SQLite database file + * @param {number[]|string} data An array of bytes representing + * an SQLite database file or a path + * @param {Object} opts Options to specify a filename */ - function Database(data) { - this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); - if (data != null) { + function Database(data, { filename = false } = {}) { + if(filename === false) { + this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); + this.memoryFile = true; + if (data != null) { FS.createDataFile("/", this.filename, data, true, true); + } + } + else { + this.filename = data; } this.handleError(sqlite3_open(this.filename, apiTemp)); this.db = getValue(apiTemp, "i32"); @@ -1103,7 +1110,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { Object.values(this.functions).forEach(removeFunction); this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); - FS.unlink("/" + this.filename); + + if(this.memoryFile) { + FS.unlink("/" + this.filename); + } this.db = null; }; @@ -1231,4 +1241,49 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // export Database to Module Module.Database = Database; + + // Because emscripten doesn't allow us to handle `ioctl`, we need + // to manually install lock/unlock methods. Unfortunately we need + // to keep track of a mapping of `sqlite_file*` pointers to filename + // so that we can tell our filesystem which files to lock/unlock + var sqliteFiles = new Map(); + + Module["register_for_idb"] = (customFS) => { + var SQLITE_BUSY = 5; + + function open(namePtr, file) { + var path = UTF8ToString(namePtr); + sqliteFiles.set(file, path); + } + + function lock(file, lockType) { + var path = sqliteFiles.get(file); + var success = customFS.lock(path, lockType) + return success? 0 : SQLITE_BUSY; + } + + function unlock(file,lockType) { + var path = sqliteFiles.get(file); + customFS.unlock(path, lockType) + return 0; + } + + let lockPtr = addFunction(lock, 'iii'); + let unlockPtr = addFunction(unlock, 'iii'); + let openPtr = addFunction(open, 'vii'); + Module["_register_for_idb"](lockPtr, unlockPtr, openPtr) + } + + // TODO: This isn't called from anywhere yet. We need to + // somehow cleanup closed files from `sqliteFiles` + Module["cleanup_file"] = (path) => { + let filesInfo = [...sqliteFiles.entries()] + let fileInfo = filesInfo.find(f => f[1] === path); + sqliteFiles.delete(fileInfo[0]) + } + + Module["reset_filesystem"] = () => { + FS.root = null; + FS.staticInit(); + } }; diff --git a/src/exported_functions.json b/src/exported_functions.json index b93b07d2..c792b657 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -41,5 +41,7 @@ "_sqlite3_result_int", "_sqlite3_result_int64", "_sqlite3_result_error", +"_sqlite3_vfs_find", +"_register_for_idb", "_RegisterExtensionFunctions" ] diff --git a/src/exported_runtime_methods.json b/src/exported_runtime_methods.json index 13a8efb8..e8915a6f 100644 --- a/src/exported_runtime_methods.json +++ b/src/exported_runtime_methods.json @@ -3,5 +3,6 @@ "stackAlloc", "stackSave", "stackRestore", -"UTF8ToString" +"UTF8ToString", +"FS" ] diff --git a/src/fs-externs.js b/src/fs-externs.js new file mode 100644 index 00000000..a82e2463 --- /dev/null +++ b/src/fs-externs.js @@ -0,0 +1,48 @@ +/** + * @externs + */ + +Module.FS = class { + constructor() { + this.ErrnoError = class {}; + } + mount() {} + isRoot() {} + isFile() {} + isDir() {} + stat() {} + /** @return {FSNode} */ + lookupPath() {} + /** @return {FSNode} */ + lookupNode() {} + /** @return {FSNode} */ + createNode() {} + /** @return {FSNode} */ + mknod() {} +}; + +Module.FS.FSNode = class { + constructor() { + this.node_ops = { + getattr: () => {}, + setattr: () => {}, + lookup: () => {}, + mknod: () => {}, + rename: () => {}, + unlink: () => {}, + rmdir: () => {}, + reaaddir: () => {}, + symlink: () => {}, + readlink: () => {} + }; + + this.stream_ops = { + llseek: () => {}, + read: () => {}, + write: () => {}, + allocate: () => {}, + mmap: () => {}, + msync: () => {} + }; + } +}; diff --git a/src/vfs.c b/src/vfs.c new file mode 100644 index 00000000..412bad87 --- /dev/null +++ b/src/vfs.c @@ -0,0 +1,47 @@ +#include +#include + +static int (*defaultOpen)(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags); + +static void (*fsOpen)(const char *, void*); +static int (*fsLock)(sqlite3_file *file, int); +static int (*fsUnlock)(sqlite3_file *file, int); + +static int blockDeviceCharacteristics(sqlite3_file* file) { + return SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL | + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; +} + +static int block_lock(sqlite3_file *file, int lock) { + return fsLock(file, lock); +} + +static int block_unlock(sqlite3_file *file, int lock) { + return fsUnlock(file, lock); +} + +static int block_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags) { + int res = defaultOpen(vfs, zName, file, flags, pOutFlags); + + sqlite3_io_methods* methods = (sqlite3_io_methods*)file->pMethods; + methods->xDeviceCharacteristics = blockDeviceCharacteristics; + methods->xLock = block_lock; + methods->xUnlock = block_unlock; + + fsOpen(zName, (void*)file); + + return res; +} + +void register_for_idb(int(*lockFile)(sqlite3_file*,int), int(*unlockFile)(sqlite3_file*,int), void(*openFile)(const char*, void*)) { + sqlite3_vfs *vfs = sqlite3_vfs_find("unix"); + defaultOpen = vfs->xOpen; + + vfs->xOpen = block_open; + sqlite3_vfs_register(vfs, 1); + + fsLock = lockFile; + fsUnlock = unlockFile; + fsOpen = openFile; +}