Add db.iterateStatements (#429)

 db.iterateStatements allows giving sql.js an SQL string and iterating over statements objects created from that string

* Initial working version of statement iteration.

* Tests written and running green.

* Resolved linter issues.

* Modified approach based on PR feedback; simple testing works, automated tests and documentation to be written.

* Testing and documentation written.

* Undid prior commit (accidentally committed change from sql-wasm.js to sql-wasm-debug.js)

* Applied all suggested modifications.

* Documentation fixes.

* Improve the documentation of db#iterateStatements

* Add @implements annotations for StatementIterator

* Reformat test code

* Fix the type definition of StatementIterator.StatementIteratorResult

Co-authored-by: ophir <pere.jobs@gmail.com>

Makefile

@@ -23,7 +23,8 @@ CFLAGS = \
 	-DSQLITE_DISABLE_LFS \
 	-DSQLITE_ENABLE_FTS3 \
 	-DSQLITE_ENABLE_FTS3_PARENTHESIS \
-	-DSQLITE_THREADSAFE=0
+	-DSQLITE_THREADSAFE=0 \
+	-DSQLITE_ENABLE_NORMALIZE
 
 # When compiling to WASM, enabling memory-growth is not expected to make much of an impact, so we enable it for all builds
 # Since tihs is a library and not a standalone executable, we don't want to catch unhandled Node process exceptions

examples/repl.html

@@ -31,4 +31,4 @@
 			document.getElementById('error').innerHTML = error;
 		};
 	</script>
-</body>
+</body>

src/api.js

@@ -3,6 +3,7 @@
     FS
     HEAP8
     Module
+    _malloc
     _free
     addFunction
     allocate
@@ -14,6 +15,9 @@
     stackAlloc
     stackRestore
     stackSave
+    UTF8ToString
+    stringToUTF8
+    lengthBytesUTF8
 */
 
 "use strict";
@@ -80,6 +84,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         "number",
         ["number", "string", "number", "number", "number"]
     );
+    var sqlite3_sql = cwrap("sqlite3_sql", "string", ["number"]);
+    var sqlite3_normalized_sql = cwrap(
+        "sqlite3_normalized_sql",
+        "string",
+        ["number"]
+    );
     var sqlite3_prepare_v2_sqlptr = cwrap(
         "sqlite3_prepare_v2",
         "number",
@@ -446,6 +456,29 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         return rowObject;
     };
 
+    /** Get the SQL string used in preparing this statement.
+     @return {string} The SQL string
+     */
+    Statement.prototype["getSQL"] = function getSQL() {
+        return sqlite3_sql(this.stmt);
+    };
+
+    /** Get the SQLite's normalized version of the SQL string used in
+    preparing this statement.  The meaning of "normalized" is not
+    well-defined: see {@link https://sqlite.org/c3ref/expanded_sql.html
+    the SQLite documentation}.
+
+     @example
+     db.run("create table test (x integer);");
+     stmt = db.prepare("select * from test where x = 42");
+     // returns "SELECT*FROM test WHERE x=?;"
+
+     @return {string} The normalized SQL string
+     */
+    Statement.prototype["getNormalizedSQL"] = function getNormalizedSQL() {
+        return sqlite3_normalized_sql(this.stmt);
+    };
+
     /** Shorthand for bind + step + reset
     Bind the values, execute the statement, ignoring the rows it returns,
     and resets it
@@ -605,6 +638,138 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         return res;
     };
 
+    /**
+     * @classdesc
+     * An iterator over multiple SQL statements in a string,
+     * preparing and returning a Statement object for the next SQL
+     * statement on each iteration.
+     *
+     * You can't instantiate this class directly, you have to use a
+     * {@link Database} object in order to create a statement iterator
+     *
+     * {@see Database#iterateStatements}
+     *
+     * @example
+     * // loop over and execute statements in string sql
+     * for (let statement of db.iterateStatements(sql) {
+     *     statement.step();
+     *     // get results, etc.
+     *     // do not call statement.free() manually, each statement is freed
+     *     // before the next one is parsed
+     * }
+     *
+     * // capture any bad query exceptions with feedback
+     * // on the bad sql
+     * let it = db.iterateStatements(sql);
+     * try {
+     *     for (let statement of it) {
+     *         statement.step();
+     *     }
+     * } catch(e) {
+     *     console.log(
+     *         `The SQL string "${it.getRemainingSQL()}" ` +
+     *         `contains the following error: ${e}`
+     *     );
+     * }
+     *
+     * @implements {Iterator<Statement>}
+     * @implements {Iterable<Statement>}
+     * @constructs StatementIterator
+     * @memberof module:SqlJs
+     * @param {string} sql A string containing multiple SQL statements
+     * @param {Database} db The database from which this iterator was created
+     */
+    function StatementIterator(sql, db) {
+        this.db = db;
+        var sz = lengthBytesUTF8(sql) + 1;
+        this.sqlPtr = _malloc(sz);
+        if (this.sqlPtr === null) {
+            throw new Error("Unable to allocate memory for the SQL string");
+        }
+        stringToUTF8(sql, this.sqlPtr, sz);
+        this.nextSqlPtr = this.sqlPtr;
+        this.nextSqlString = null;
+        this.activeStatement = null;
+    }
+
+    /**
+     * @typedef {{ done:true, value:undefined } |
+     *           { done:false, value:Statement}}
+     *           StatementIterator.StatementIteratorResult
+     * @property {Statement} value the next available Statement
+     * (as returned by {@link Database.prepare})
+     * @property {boolean} done true if there are no more available statements
+     */
+
+    /** Prepare the next available SQL statement
+     @return {StatementIterator.StatementIteratorResult}
+     @throws {String} SQLite error or invalid iterator error
+     */
+    StatementIterator.prototype["next"] = function next() {
+        if (this.sqlPtr === null) {
+            return { done: true };
+        }
+        if (this.activeStatement !== null) {
+            this.activeStatement["free"]();
+            this.activeStatement = null;
+        }
+        if (!this.db.db) {
+            this.finalize();
+            throw new Error("Database closed");
+        }
+        var stack = stackSave();
+        var pzTail = stackAlloc(4);
+        setValue(apiTemp, 0, "i32");
+        setValue(pzTail, 0, "i32");
+        try {
+            this.db.handleError(sqlite3_prepare_v2_sqlptr(
+                this.db.db,
+                this.nextSqlPtr,
+                -1,
+                apiTemp,
+                pzTail
+            ));
+            this.nextSqlPtr = getValue(pzTail, "i32");
+            var pStmt = getValue(apiTemp, "i32");
+            if (pStmt === NULL) {
+                this.finalize();
+                return { done: true };
+            }
+            this.activeStatement = new Statement(pStmt, this.db);
+            this.db.statements[pStmt] = this.activeStatement;
+            return { value: this.activeStatement, done: false };
+        } catch (e) {
+            this.nextSqlString = UTF8ToString(this.nextSqlPtr);
+            this.finalize();
+            throw e;
+        } finally {
+            stackRestore(stack);
+        }
+    };
+
+    StatementIterator.prototype.finalize = function finalize() {
+        _free(this.sqlPtr);
+        this.sqlPtr = null;
+    };
+
+    /** Get any un-executed portions remaining of the original SQL string
+     @return {String}
+     */
+    StatementIterator.prototype["getRemainingSQL"] = function getRemainder() {
+        // iff an exception occurred, we set the nextSqlString
+        if (this.nextSqlString !== null) return this.nextSqlString;
+        // otherwise, convert from nextSqlPtr
+        return UTF8ToString(this.nextSqlPtr);
+    };
+
+    /* implement Iterable interface */
+
+    if (typeof Symbol === "function" && typeof Symbol.iterator === "symbol") {
+        StatementIterator.prototype[Symbol.iterator] = function iterator() {
+            return this;
+        };
+    }
+
     /** @classdesc
     * Represents an SQLite database
     * @constructs Database
@@ -844,6 +1009,27 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         return stmt;
     };
 
+    /** Iterate over multiple SQL statements in a SQL string.
+     * This function returns an iterator over {@link Statement} objects.
+     * You can use a for..of loop to execute the returned statements one by one.
+     * @param {string} sql a string of SQL that can contain multiple statements
+     * @return {StatementIterator} the resulting statement iterator
+     * @example <caption>Get the results of multiple SQL queries</caption>
+     * const sql_queries = "SELECT 1 AS x; SELECT '2' as y";
+     * for (const statement of db.iterateStatements(sql_queries)) {
+     *     statement.step(); // Execute the statement
+     *     const sql = statement.getSQL(); // Get the SQL source
+     *     const result = statement.getAsObject(); // Get the row of data
+     *     console.log(sql, result);
+     * }
+     * // This will print:
+     * // 'SELECT 1 AS x;' { x: 1 }
+     * // " SELECT '2' as y" { y: '2' }
+     */
+    Database.prototype["iterateStatements"] = function iterateStatements(sql) {
+        return new StatementIterator(sql, this);
+    };
+
     /** Exports the contents of the database to a binary array
     @return {Uint8Array} An array of bytes of the SQLite3 database file
      */

src/exported_functions.json

@@ -7,6 +7,8 @@
 "_sqlite3_errmsg",
 "_sqlite3_changes",
 "_sqlite3_prepare_v2",
+"_sqlite3_sql",
+"_sqlite3_normalized_sql",
 "_sqlite3_bind_text",
 "_sqlite3_bind_blob",
 "_sqlite3_bind_double",

src/exported_runtime_methods.json

@@ -2,5 +2,6 @@
 "cwrap",
 "stackAlloc",
 "stackSave",
-"stackRestore"
+"stackRestore",
+"UTF8ToString"
 ]

test/test_statement_iterator.js

@@ -0,0 +1,107 @@
+exports.test = function (SQL, assert) {
+  // Create a database
+  var db = new SQL.Database();
+
+  // Multiline SQL
+  var sqlstr = "CREATE TABLE test (x text, y integer);\n"
+    + "INSERT INTO test\n"
+    + "VALUES ('hello', 42), ('goodbye', 17);\n"
+    + "SELECT * FROM test;\n"
+    + " -- nothing here";
+  var sqlstart = "CREATE TABLE test (x text, y integer);"
+
+  // Manual iteration
+  // Get an iterator
+  var it = db.iterateStatements(sqlstr);
+
+  // Get first item
+  var x = it.next();
+  assert.equal(x.done, false, "Valid iterator object produced");
+  assert.equal(x.value.getSQL(), sqlstart, "Statement is for first query only");
+  assert.equal(it.getRemainingSQL(), sqlstr.slice(sqlstart.length), "Remaining sql retrievable");
+
+  // execute the first query
+  x.value.step();
+
+  // get and execute the second query
+  x = it.next();
+  assert.equal(x.done, false, "Second query found");
+  x.value.step();
+
+  // get and execute the third query
+  x = it.next();
+  assert.equal(x.done, false, "Third query found");
+  x.value.step();
+  assert.deepEqual(x.value.getColumnNames(), ['x', 'y'], "Third query is SELECT");
+
+  // check for additional queries
+  x = it.next();
+  assert.deepEqual(x, { done: true }, "Done reported after last query");
+
+  // additional iteration does nothing
+  x = it.next();
+  assert.deepEqual(x, { done: true }, "Done reported when iterating past completion");
+
+  db.run("DROP TABLE test;");
+
+  // for...of
+  var count = 0;
+  for (let statement of db.iterateStatements(sqlstr)) {
+    statement.step();
+    count = count + 1;
+  }
+  assert.equal(count, 3, "For loop iterates correctly");
+
+  var badsql = "SELECT 1 as x;garbage in, garbage out";
+
+  // bad sql will stop iteration
+  it = db.iterateStatements(badsql);
+  x = it.next();
+  x.value.step();
+  assert.deepEqual(x.value.getAsObject(), { x: 1 }, "SQL before bad statement executes successfully");
+  assert.throws(function () { it.next() }, /syntax error/, "Bad SQL stops iteration with exception");
+  assert.deepEqual(it.next(), { done: true }, "Done reported when iterating after exception");
+
+  // valid SQL executes, remaining SQL accessible after exception
+  it = db.iterateStatements(badsql);
+  var remains = '';
+  try {
+    for (let statement of it) {
+      statement.step();
+    }
+  } catch {
+    remains = it.getRemainingSQL();
+  }
+  assert.equal(remains, "garbage in, garbage out", "Remaining SQL accessible after exception");
+
+  // From the doc example on the iterateStatements method
+  const results = [];
+  const sql_queries = "SELECT 1 AS x; SELECT '2' as y";
+  for (const statement of db.iterateStatements(sql_queries)) {
+    statement.step(); // Fetch one line of result from the statement
+    const sql = statement.getSQL();
+    const result = statement.getAsObject();
+    results.push({ sql, result });
+  }
+  console.log(results);
+  assert.deepEqual(results, [
+    { sql: 'SELECT 1 AS x;', result: { x: 1 } },
+    { sql: " SELECT '2' as y", result: { y: '2' } }
+  ], "The code example from the documentation works");
+};
+
+if (module == require.main) {
+  const target_file = process.argv[2];
+  const sql_loader = require('./load_sql_lib');
+  sql_loader(target_file).then((sql) => {
+    require('test').run({
+      'test statement iterator': function (assert) {
+        exports.test(sql, assert);
+      }
+    });
+  })
+    .catch((e) => {
+      console.error(e);
+      assert.fail(e);
+    });
+}