From 24a6a12833a77affbe938e2ed4103415ccb35eb8 Mon Sep 17 00:00:00 2001 From: dogquery <> Date: Fri, 3 Jul 2020 23:25:30 +0900 Subject: [PATCH 01/36] initial commit. --- src/api.js | 216 ++++++++++++++++++++++--------- test/test_aggregate_functions.js | 17 +++ 2 files changed, 171 insertions(+), 62 deletions(-) create mode 100644 test/test_aggregate_functions.js diff --git a/src/api.js b/src/api.js index 854bf32e..b5f14f12 100644 --- a/src/api.js +++ b/src/api.js @@ -1131,81 +1131,90 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return sqlite3_changes(this.db); }; - /** Register a custom function with SQLite - @example Register a simple function - db.create_function("addOne", function (x) {return x+1;}) - db.exec("SELECT addOne(1)") // = 2 + var extract_blob = function extract_blob(ptr) { + var size = sqlite3_value_bytes(ptr); + var blob_ptr = sqlite3_value_blob(ptr); + var blob_arg = new Uint8Array(size); + for (var j = 0; j < size; j += 1) { + blob_arg[j] = HEAP8[blob_ptr + j]; + } + return blob_arg; + }; - @param {string} name the name of the function as referenced in - SQL statements. - @param {function} func the actual function to be executed. - @return {Database} The database object. Useful for method chaining - */ + var parseFunctionArguments = function parseFunctionArguments(argc, argv) { + var args = []; + for (var i = 0; i < argc; i += 1) { + var value_ptr = getValue(argv + (4 * i), "i32"); + var value_type = sqlite3_value_type(value_ptr); + var arg; + if ( + value_type === SQLITE_INTEGER + || value_type === SQLITE_FLOAT + ) { + arg = sqlite3_value_double(value_ptr); + } else if (value_type === SQLITE_TEXT) { + arg = sqlite3_value_text(value_ptr); + } else if (value_type === SQLITE_BLOB) { + arg = extract_blob(value_ptr); + } else arg = null; + args.push(arg); + } + return args; + }; + var setFunctionResult = function setFunctionResult(cx, result) { + switch (typeof result) { + case "boolean": + sqlite3_result_int(cx, result ? 1 : 0); + break; + case "number": + sqlite3_result_double(cx, result); + break; + case "string": + sqlite3_result_text(cx, result, -1, -1); + break; + case "object": + if (result === null) { + sqlite3_result_null(cx); + } else if (result.length != null) { + var blobptr = allocate(result, "i8", ALLOC_NORMAL); + sqlite3_result_blob(cx, blobptr, result.length, -1); + _free(blobptr); + } else { + sqlite3_result_error(cx, ( + "Wrong API use : tried to return a value " + + "of an unknown type (" + result + ")." + ), -1); + } + break; + default: + sqlite3_result_null(cx); + } + }; + + /** Register a custom function with SQLite + @example Register a simple function + db.create_function("addOne", function (x) {return x+1;}) + db.exec("SELECT addOne(1)") // = 2 + + @param {string} name the name of the function as referenced in + SQL statements. + @param {function} func the actual function to be executed. + @return {Database} The database object. Useful for method chaining + */ Database.prototype["create_function"] = function create_function( name, func ) { function wrapped_func(cx, argc, argv) { + var args = parseFunctionArguments(argc, argv); var result; - function extract_blob(ptr) { - var size = sqlite3_value_bytes(ptr); - var blob_ptr = sqlite3_value_blob(ptr); - var blob_arg = new Uint8Array(size); - for (var j = 0; j < size; j += 1) { - blob_arg[j] = HEAP8[blob_ptr + j]; - } - return blob_arg; - } - var args = []; - for (var i = 0; i < argc; i += 1) { - var value_ptr = getValue(argv + (4 * i), "i32"); - var value_type = sqlite3_value_type(value_ptr); - var arg; - if ( - value_type === SQLITE_INTEGER - || value_type === SQLITE_FLOAT - ) { - arg = sqlite3_value_double(value_ptr); - } else if (value_type === SQLITE_TEXT) { - arg = sqlite3_value_text(value_ptr); - } else if (value_type === SQLITE_BLOB) { - arg = extract_blob(value_ptr); - } else arg = null; - args.push(arg); - } try { result = func.apply(null, args); } catch (error) { sqlite3_result_error(cx, error, -1); return; } - switch (typeof result) { - case "boolean": - sqlite3_result_int(cx, result ? 1 : 0); - break; - case "number": - sqlite3_result_double(cx, result); - break; - case "string": - sqlite3_result_text(cx, result, -1, -1); - break; - case "object": - if (result === null) { - sqlite3_result_null(cx); - } else if (result.length != null) { - var blobptr = allocate(result, ALLOC_NORMAL); - sqlite3_result_blob(cx, blobptr, result.length, -1); - _free(blobptr); - } else { - sqlite3_result_error(cx, ( - "Wrong API use : tried to return a value " - + "of an unknown type (" + result + ")." - ), -1); - } - break; - default: - sqlite3_result_null(cx); - } + setFunctionResult(cx, result); } if (Object.prototype.hasOwnProperty.call(this.functions, name)) { removeFunction(this.functions[name]); @@ -1229,6 +1238,89 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return this; }; + /** Register a custom aggregate with SQLite + @example Register a aggregate function + db.create_aggregate( + "js_sum", + function () { return { sum: 0 }; }, + function (state, value) { state.sum+=value; }, + function (state) { return state.sum; } + ); + db.exec("CREATE TABLE test (col); INSERT INTO test VALUES (1), (2)"); + db.exec("SELECT js_sum(col) FROM test"); // = 3 + + @param {string} name the name of the aggregate as referenced in + SQL statements. + @param {function} init the actual function to be executed on initialize. + @param {function} step the actual function to be executed on step by step. + @param {function} finalize the actual function to be executed on finalize. + @return {Database} The database object. Useful for method chaining + */ + Database.prototype["create_aggregate"] = function create_aggregate( + name, + init, + step, + finalize + ) { + var state; + function wrapped_step(cx, argc, argv) { + if (!state) { + state = init(); + } + var args = parseFunctionArguments(argc, argv); + var mergedArgs = [state].concat(args); + try { + step.apply(null, mergedArgs); + } catch (error) { + sqlite3_result_error(cx, error, -1); + } + } + function wrapped_finalize(cx) { + var result; + try { + result = finalize.apply(null, [state]); + } catch (error) { + sqlite3_result_error(cx, error, -1); + state = null; + return; + } + setFunctionResult(cx, result); + state = null; + } + + if (Object.prototype.hasOwnProperty.call(this.functions, name)) { + removeFunction(this.functions[name]); + delete this.functions[name]; + } + if (Object.prototype.hasOwnProperty.call( + this.functions, + name + "__finalize" + )) { + removeFunction(this.functions[name + "__finalize"]); + delete this.functions[name + "__finalize"]; + } + // The signature of the wrapped function is : + // void wrapped(sqlite3_context *db, int argc, sqlite3_value **argv) + var step_ptr = addFunction(wrapped_step, "viii"); + // The signature of the wrapped function is : + // void wrapped(sqlite3_context *db) + var finalize_ptr = addFunction(wrapped_finalize, "vi"); + this.functions[name] = step_ptr; + this.functions[name + "__finalize"] = finalize_ptr; + this.handleError(sqlite3_create_function_v2( + this.db, + name, + step.length - 1, + SQLITE_UTF8, + 0, + 0, + step_ptr, + finalize_ptr, + 0 + )); + return this; + }; + // export Database to Module Module.Database = Database; }; diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js new file mode 100644 index 00000000..5a0ed16e --- /dev/null +++ b/test/test_aggregate_functions.js @@ -0,0 +1,17 @@ +exports.test = function (SQL, assert) { + var db = new SQL.Database(); + + db.create_aggregate( + "js_sum", + function () { return { sum: 0 }; }, + function (state, value) { state.sum += value; }, + function (state) { return state.sum; } + ); + + db.exec("CREATE TABLE test (col);"); + db.exec("INSERT INTO test VALUES (1), (2), (3);"); + var result = db.exec("SELECT js_sum(col) FROM test;"); + assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); + + // TODO: Add test cases.. +} \ No newline at end of file From fad9ba66470428c2d64b857181b2770b0543b005 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:24:55 -0400 Subject: [PATCH 02/36] documentation --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index bfc130b4..b779eabb 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,45 @@ db.create_function("add_js", add); // Run a query in which the function is used db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world' +// You can create aggregation functions, by passing a name and three functions +// to `db.create_aggregate`: +// +// - an init function. This function receives no arguments and will be called +// when the aggregate begins. Returns a state object that will be passed to the +// other two functions if you need to track state. +// - a step function. This function receives as a first argument the state +// object created in init, as well as the values received in the step. It +// will be called on every value to be aggregated. Does not return anything. +// - a finalizer. This function receives one argument, the state object, and +// returns the final value of the aggregate +// +// Here is an example aggregation function, `json_agg`, which will collect all +// input values and return them as a JSON array: +db.create_aggregate( + "json_agg", + function() { + // This is the init function, which returns a state object: + return { + values: [] + }; + }, + function(state, val) { + // This is the step function, which will store each value it receives in + // the values array of the state object + state.values.push(val); + }, + function(state) { + // This is the finalize function, which converts the received values from + // the state object into a JSON array and returns that + return JSON.stringify(state.values); + } +); + +// Now if you run this query: +var result = db.exec("SELECT json_agg(somecol) FROM atable;"); + +// result will be a json-encoded string representing each value of `somecol` in `atable`. + // Export the database to an Uint8Array containing the SQLite database file const binaryArray = db.export(); ``` From d191caa34cbfc39e85b10612429cbf7b59d77b5b Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:29:03 -0400 Subject: [PATCH 03/36] remove no-longer-valid type --- src/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api.js b/src/api.js index b5f14f12..0eef62b6 100644 --- a/src/api.js +++ b/src/api.js @@ -1176,7 +1176,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { if (result === null) { sqlite3_result_null(cx); } else if (result.length != null) { - var blobptr = allocate(result, "i8", ALLOC_NORMAL); + var blobptr = allocate(result, ALLOC_NORMAL); sqlite3_result_blob(cx, blobptr, result.length, -1); _free(blobptr); } else { From 0d937a7c67368204fa16f88fa1c3f9aa5dd1b240 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:29:52 -0400 Subject: [PATCH 04/36] close over state initialization for performance --- src/api.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/api.js b/src/api.js index 0eef62b6..1fdf68c0 100644 --- a/src/api.js +++ b/src/api.js @@ -1262,11 +1262,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { step, finalize ) { - var state; + var state = init(); function wrapped_step(cx, argc, argv) { - if (!state) { - state = init(); - } var args = parseFunctionArguments(argc, argv); var mergedArgs = [state].concat(args); try { From 8fd3f8a20a002007a6e457051eab3e6bae2fc8fa Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:30:07 -0400 Subject: [PATCH 05/36] link documentation in comment --- src/api.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/api.js b/src/api.js index 1fdf68c0..a0a4e525 100644 --- a/src/api.js +++ b/src/api.js @@ -1304,6 +1304,13 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var finalize_ptr = addFunction(wrapped_finalize, "vi"); this.functions[name] = step_ptr; this.functions[name + "__finalize"] = finalize_ptr; + + // passing null to the sixth parameter defines this as an aggregate + // function + // + // > An aggregate SQL function requires an implementation of xStep and + // > xFinal and NULL pointer must be passed for xFunc. + // - http://www.sqlite.org/c3ref/create_function.html this.handleError(sqlite3_create_function_v2( this.db, name, From ba733ba041c04c38ba4b1cb08098172e74c964ce Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:30:38 -0400 Subject: [PATCH 06/36] more testing --- test/test_aggregate_functions.js | 46 +++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 5a0ed16e..30c7b475 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -2,7 +2,7 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "js_sum", + "sum", function () { return { sum: 0 }; }, function (state, value) { state.sum += value; }, function (state) { return state.sum; } @@ -10,8 +10,46 @@ exports.test = function (SQL, assert) { db.exec("CREATE TABLE test (col);"); db.exec("INSERT INTO test VALUES (1), (2), (3);"); - var result = db.exec("SELECT js_sum(col) FROM test;"); + var result = db.exec("SELECT sum(col) FROM test;"); assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); - // TODO: Add test cases.. -} \ No newline at end of file + db.create_aggregate( + "percentile", + function () { return { vals: [], pctile: null }; }, // init + function (state, value, pctile) { + state.vals.push(value); + }, + function (state) { + return percentile(state.vals, state.pctile); + } + ); + var result = db.exec("SELECT percentile(col, 20) FROM test;"); + assert.equal(result[0].values[0][0], 1, "Aggregate function with two args"); + + db.create_aggregate( + "json_agg", + function() { return { vals: [] }; }, + function(state, val) { state.vals.push(val); }, + function(state) { return JSON.stringify(state.vals); } + ); + + db.exec("CREATE TABLE test2 (col, col2);"); + db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); + var result = db.exec("SELECT json_agg(col) FROM test2;"); + assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); +} + +// helper function to calculate a percentile from an array. Will modify the +// array in-place. +function percentile(arr, p) { + arr.sort(); + const pos = (arr.length - 1) * (p / 100); + const base = Math.floor(pos); + const rest = pos - base; + if (arr[base + 1] !== undefined) { + return arr[base] + rest * (arr[base + 1] - arr[base]); + } else { + return arr[base]; + } +}; + From 9e6b46290a73a38f7ccb0bb7a14df30796920a26 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Mon, 5 Sep 2022 20:07:54 -0400 Subject: [PATCH 07/36] run tests if they're main --- test/test_aggregate_functions.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 30c7b475..243a2d1f 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -53,3 +53,18 @@ function percentile(arr, p) { } }; +if (module == require.main) { + const target_file = process.argv[2]; + const sql_loader = require('./load_sql_lib'); + sql_loader(target_file).then((sql)=>{ + require('test').run({ + 'test functions': function(assert, done){ + exports.test(sql, assert, done); + } + }); + }) + .catch((e)=>{ + console.error(e); + assert.fail(e); + }); +} From 573afa71e8b2fe6daca1db327851e642cac95772 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Mon, 5 Sep 2022 22:17:40 -0400 Subject: [PATCH 08/36] accept a single arg --- src/api.js | 24 +++++++++++++---------- test/test_aggregate_functions.js | 33 +++++++++++++++++--------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/api.js b/src/api.js index a0a4e525..0a3110b8 100644 --- a/src/api.js +++ b/src/api.js @@ -1251,23 +1251,27 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { @param {string} name the name of the aggregate as referenced in SQL statements. - @param {function} init the actual function to be executed on initialize. - @param {function} step the actual function to be executed on step by step. - @param {function} finalize the actual function to be executed on finalize. + @param {object} Aggregate function containing three functions @return {Database} The database object. Useful for method chaining */ Database.prototype["create_aggregate"] = function create_aggregate( name, - init, - step, - finalize + aggregateFunctions ) { - var state = init(); + if (!aggregateFunctions.hasOwnProperty("init") || + !aggregateFunctions.hasOwnProperty("step") || + !aggregateFunctions.hasOwnProperty("finalize")) + throw "An aggregate function must have init, step and finalize properties"; + + var state; function wrapped_step(cx, argc, argv) { + if (!state) { + state = aggregateFunctions["init"].apply(null); + } var args = parseFunctionArguments(argc, argv); var mergedArgs = [state].concat(args); try { - step.apply(null, mergedArgs); + aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { sqlite3_result_error(cx, error, -1); } @@ -1275,7 +1279,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { function wrapped_finalize(cx) { var result; try { - result = finalize.apply(null, [state]); + result = aggregateFunctions["finalize"].apply(null, [state]); } catch (error) { sqlite3_result_error(cx, error, -1); state = null; @@ -1314,7 +1318,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { this.handleError(sqlite3_create_function_v2( this.db, name, - step.length - 1, + aggregateFunctions["step"].length - 1, SQLITE_UTF8, 0, 0, diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 243a2d1f..a7ec77a3 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -2,10 +2,11 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "sum", - function () { return { sum: 0 }; }, - function (state, value) { state.sum += value; }, - function (state) { return state.sum; } + "sum", { + init: function () { return { sum: 0 }; }, + step: function (state, value) { state.sum += value; }, + finalize: function (state) { return state.sum; } + } ); db.exec("CREATE TABLE test (col);"); @@ -14,23 +15,25 @@ exports.test = function (SQL, assert) { assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); db.create_aggregate( - "percentile", - function () { return { vals: [], pctile: null }; }, // init - function (state, value, pctile) { - state.vals.push(value); - }, - function (state) { - return percentile(state.vals, state.pctile); + "percentile", { + init: function () { return { vals: [], pctile: null }; }, // init + step: function (state, value, pctile) { + state.vals.push(value); + }, + finalize: function (state) { + return percentile(state.vals, state.pctile); + } } ); var result = db.exec("SELECT percentile(col, 20) FROM test;"); assert.equal(result[0].values[0][0], 1, "Aggregate function with two args"); db.create_aggregate( - "json_agg", - function() { return { vals: [] }; }, - function(state, val) { state.vals.push(val); }, - function(state) { return JSON.stringify(state.vals); } + "json_agg", { + init: function() { return { vals: [] }; }, + step: function(state, val) { state.vals.push(val); }, + finalize: function(state) { return JSON.stringify(state.vals); } + } ); db.exec("CREATE TABLE test2 (col, col2);"); From a3abdcb6eb36d55a36942f27761dc629d4a662bb Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Mon, 5 Sep 2022 23:41:37 -0400 Subject: [PATCH 09/36] this kind of works but I'm abandoning this branch Basically it seems that the sqlite extension pattern of 'allocate a struct and stick it in the context pointer' is not going to work for us here. I wonder if using the id of the pointer returned by sqlite3_aggregate_context would be enough? Since no two functions could use the same pointer, per https://www.sqlite.org/c3ref/aggregate_context.html ? --- src/api.js | 10 ++++++++++ src/exported_functions.json | 1 + 2 files changed, 11 insertions(+) diff --git a/src/api.js b/src/api.js index 0a3110b8..9cc1c72d 100644 --- a/src/api.js +++ b/src/api.js @@ -69,6 +69,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var SQLITE_FLOAT = 2; var SQLITE_TEXT = 3; var SQLITE_BLOB = 4; + var SQLITE_NULL = 5; // var - Encodings, used for registering functions. var SQLITE_UTF8 = 1; // var - cwrap function @@ -225,6 +226,14 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { "", ["number", "string", "number"] ); + + // https://www.sqlite.org/c3ref/aggregate_context.html + // void *sqlite3_aggregate_context(sqlite3_context*, int nBytes) + var sqlite3_aggregate_context = cwrap( + "sqlite3_aggregate_context", + "number", + ["number", "number"] + ); var registerExtensionFunctions = cwrap( "RegisterExtensionFunctions", "number", @@ -1265,6 +1274,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var state; function wrapped_step(cx, argc, argv) { + var p = sqlite3_aggregate_context(cx, 999); if (!state) { state = aggregateFunctions["init"].apply(null); } diff --git a/src/exported_functions.json b/src/exported_functions.json index b93b07d2..324017ae 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -41,5 +41,6 @@ "_sqlite3_result_int", "_sqlite3_result_int64", "_sqlite3_result_error", +"_sqlite3_aggregate_context", "_RegisterExtensionFunctions" ] From 9daf01f942ba656c0bfbbc3eafbe254f6aa9da67 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 00:01:41 -0400 Subject: [PATCH 10/36] a middle road sqlite3_agg_context solution --- src/api.js | 51 ++++++++++++++++++++++++-------- test/test_aggregate_functions.js | 6 ++-- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/api.js b/src/api.js index 9cc1c72d..5d79dfd7 100644 --- a/src/api.js +++ b/src/api.js @@ -69,7 +69,6 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var SQLITE_FLOAT = 2; var SQLITE_TEXT = 3; var SQLITE_BLOB = 4; - var SQLITE_NULL = 5; // var - Encodings, used for registering functions. var SQLITE_UTF8 = 1; // var - cwrap function @@ -1267,36 +1266,64 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { name, aggregateFunctions ) { - if (!aggregateFunctions.hasOwnProperty("init") || - !aggregateFunctions.hasOwnProperty("step") || - !aggregateFunctions.hasOwnProperty("finalize")) - throw "An aggregate function must have init, step and finalize properties"; + if (!Object.hasOwnProperty.call(aggregateFunctions, "step") + || !Object.hasOwnProperty.call( + aggregateFunctions, "finalize" + ) + ) { + throw "An aggregate function must have step and finalize " + + "properties"; + } + + // state is a state array; we'll use the pointer p to serve as the + // key for where we hold our state so that multiple invocations of + // this function never step on each other + var state = {}; - var state; function wrapped_step(cx, argc, argv) { - var p = sqlite3_aggregate_context(cx, 999); - if (!state) { - state = aggregateFunctions["init"].apply(null); + // The first time the sqlite3_aggregate_context(C,N) routine is + // called for a particular aggregate function, SQLite allocates N + // bytes of memory, zeroes out that memory, and returns a pointer + // to the new memory. + // + // We're going to use that pointer as a key to our state array, + // since using sqlite3_aggregate_context as it's meant to be used + // through webassembly seems to be very difficult. Just allocate + // one byte. + var p = sqlite3_aggregate_context(cx, 1); + + // If this is the first invocation of wrapped_step, state[p] + if (!state[p]) { + if (Object.hasOwnProperty.call(aggregateFunctions, "init")) { + state[p] = aggregateFunctions["init"].apply(null); + } else { + state[p] = []; + } } + var args = parseFunctionArguments(argc, argv); - var mergedArgs = [state].concat(args); + var mergedArgs = [state[p]].concat(args); try { aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { sqlite3_result_error(cx, error, -1); } } + function wrapped_finalize(cx) { var result; + var p = sqlite3_aggregate_context(cx, 1); try { - result = aggregateFunctions["finalize"].apply(null, [state]); + result = aggregateFunctions["finalize"].apply(null, [state[p]]); } catch (error) { sqlite3_result_error(cx, error, -1); state = null; return; } setFunctionResult(cx, result); - state = null; + + // clear the state for this invocation + delete state[p]; } if (Object.prototype.hasOwnProperty.call(this.functions, name)) { diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index a7ec77a3..aa4467aa 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -30,15 +30,15 @@ exports.test = function (SQL, assert) { db.create_aggregate( "json_agg", { - init: function() { return { vals: [] }; }, - step: function(state, val) { state.vals.push(val); }, - finalize: function(state) { return JSON.stringify(state.vals); } + step: function(state, val) { state.push(val); }, + finalize: function(state) { return JSON.stringify(state); } } ); db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); var result = db.exec("SELECT json_agg(col) FROM test2;"); + console.log("output: ", result[0].values); assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); } From ec5c72b059c2b3323c9712ab0df26eba33d63450 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:42:50 -0400 Subject: [PATCH 11/36] try out auto-updating state --- src/api.js | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/api.js b/src/api.js index 5d79dfd7..0443a61e 100644 --- a/src/api.js +++ b/src/api.js @@ -1267,24 +1267,23 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { aggregateFunctions ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") - || !Object.hasOwnProperty.call( - aggregateFunctions, "finalize" - ) ) { - throw "An aggregate function must have step and finalize " - + "properties"; + throw "An aggregate function must have a step property"; } - // state is a state array; we'll use the pointer p to serve as the + aggregateFunctions["init"] ||= (() => null) + aggregateFunctions["finalize"] ||= ((state) => state) + + // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of // this function never step on each other var state = {}; function wrapped_step(cx, argc, argv) { - // The first time the sqlite3_aggregate_context(C,N) routine is - // called for a particular aggregate function, SQLite allocates N - // bytes of memory, zeroes out that memory, and returns a pointer - // to the new memory. + // > The first time the sqlite3_aggregate_context(C,N) routine is + // > called for a particular aggregate function, SQLite allocates N + // > bytes of memory, zeroes out that memory, and returns a pointer + // > to the new memory. // // We're going to use that pointer as a key to our state array, // since using sqlite3_aggregate_context as it's meant to be used @@ -1292,19 +1291,15 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // one byte. var p = sqlite3_aggregate_context(cx, 1); - // If this is the first invocation of wrapped_step, state[p] + // If this is the first invocation of wrapped_step, call `init` if (!state[p]) { - if (Object.hasOwnProperty.call(aggregateFunctions, "init")) { - state[p] = aggregateFunctions["init"].apply(null); - } else { - state[p] = []; - } + state[p] = aggregateFunctions["init"].apply(null); } var args = parseFunctionArguments(argc, argv); var mergedArgs = [state[p]].concat(args); try { - aggregateFunctions["step"].apply(null, mergedArgs); + state[p] = aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { sqlite3_result_error(cx, error, -1); } @@ -1340,6 +1335,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // The signature of the wrapped function is : // void wrapped(sqlite3_context *db, int argc, sqlite3_value **argv) var step_ptr = addFunction(wrapped_step, "viii"); + // The signature of the wrapped function is : // void wrapped(sqlite3_context *db) var finalize_ptr = addFunction(wrapped_finalize, "vi"); From a927950821a5de68d673432f69e644ac22d4c300 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:43:08 -0400 Subject: [PATCH 12/36] improve quantile test, add multiple agg test --- test/test_aggregate_functions.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index aa4467aa..09971184 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -1,11 +1,13 @@ exports.test = function (SQL, assert) { + function assertFloat(got, expected, message="", sigma=0.001) { + assert.ok(got > expected - sigma && got < expected + sigma, message); + } + var db = new SQL.Database(); db.create_aggregate( "sum", { - init: function () { return { sum: 0 }; }, - step: function (state, value) { state.sum += value; }, - finalize: function (state) { return state.sum; } + step: function (state, value) { return (state || 0) + value; }, } ); @@ -18,28 +20,32 @@ exports.test = function (SQL, assert) { "percentile", { init: function () { return { vals: [], pctile: null }; }, // init step: function (state, value, pctile) { + state.pctile = pctile; state.vals.push(value); + return state; }, finalize: function (state) { return percentile(state.vals, state.pctile); } } ); - var result = db.exec("SELECT percentile(col, 20) FROM test;"); - assert.equal(result[0].values[0][0], 1, "Aggregate function with two args"); + result = db.exec("SELECT percentile(col, 80) FROM test;"); + assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); db.create_aggregate( "json_agg", { - step: function(state, val) { state.push(val); }, + step: function(state, val) { state = (state || []); state.push(val); return state; }, finalize: function(state) { return JSON.stringify(state); } } ); db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); - var result = db.exec("SELECT json_agg(col) FROM test2;"); - console.log("output: ", result[0].values); + result = db.exec("SELECT json_agg(col) FROM test2;"); assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); + + result = db.exec("SELECT json_agg(col), json_agg(col2) FROM test2;"); + assert.deepEqual(result[0].values[0].map(JSON.parse), [["four score", "and seven", "years ago"], [12, 7, 1]], "Multiple aggregations at once"); } // helper function to calculate a percentile from an array. Will modify the From e643bd9cfa8baa78c56922c05e886e9fa35bec7d Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:52:02 -0400 Subject: [PATCH 13/36] add a null to the test --- test/test_aggregate_functions.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 09971184..eecd3156 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -12,7 +12,7 @@ exports.test = function (SQL, assert) { ); db.exec("CREATE TABLE test (col);"); - db.exec("INSERT INTO test VALUES (1), (2), (3);"); + db.exec("INSERT INTO test VALUES (1), (2), (3), (null);"); var result = db.exec("SELECT sum(col) FROM test;"); assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); @@ -20,8 +20,10 @@ exports.test = function (SQL, assert) { "percentile", { init: function () { return { vals: [], pctile: null }; }, // init step: function (state, value, pctile) { - state.pctile = pctile; - state.vals.push(value); + if (value && !isNaN(value)) { + state.pctile = pctile; + state.vals.push(value); + } return state; }, finalize: function (state) { From 2cbdb0e058164ec14e6e521e5c057f86060abaea Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:58:48 -0400 Subject: [PATCH 14/36] acorn fails to parse ||=, whatever --- src/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api.js b/src/api.js index 0443a61e..f299745e 100644 --- a/src/api.js +++ b/src/api.js @@ -1268,11 +1268,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") ) { - throw "An aggregate function must have a step property"; + throw "An aggregate function must have a step function"; } - aggregateFunctions["init"] ||= (() => null) - aggregateFunctions["finalize"] ||= ((state) => state) + aggregateFunctions["init"] = aggregateFunctions["init"] || (() => null); + aggregateFunctions["finalize"] = aggregateFunctions["finalize"] + || ((state) => state); // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of From b9ccd48f515503a4ff79e4c6d7afd28b89e3a2f8 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 10:12:08 -0400 Subject: [PATCH 15/36] make eslint happy --- src/api.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/api.js b/src/api.js index f299745e..b54c9ef2 100644 --- a/src/api.js +++ b/src/api.js @@ -1271,9 +1271,13 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { throw "An aggregate function must have a step function"; } - aggregateFunctions["init"] = aggregateFunctions["init"] || (() => null); + // Default initializer and finalizer + function init() { return null; } + function finalize(state) { return state; } + + aggregateFunctions["init"] = aggregateFunctions["init"] || init; aggregateFunctions["finalize"] = aggregateFunctions["finalize"] - || ((state) => state); + || finalize; // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of From ac548d429b16f618c0a6e1f1d81783062ff5cf48 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 08:24:37 -0400 Subject: [PATCH 16/36] make initial_value an argument --- src/api.js | 8 +++----- test/test_aggregate_functions.js | 17 +++++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/api.js b/src/api.js index b54c9ef2..ecb77aec 100644 --- a/src/api.js +++ b/src/api.js @@ -1264,6 +1264,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { */ Database.prototype["create_aggregate"] = function create_aggregate( name, + initial_value, aggregateFunctions ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") @@ -1271,11 +1272,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { throw "An aggregate function must have a step function"; } - // Default initializer and finalizer - function init() { return null; } + // Default finalizer function finalize(state) { return state; } - - aggregateFunctions["init"] = aggregateFunctions["init"] || init; aggregateFunctions["finalize"] = aggregateFunctions["finalize"] || finalize; @@ -1298,7 +1296,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // If this is the first invocation of wrapped_step, call `init` if (!state[p]) { - state[p] = aggregateFunctions["init"].apply(null); + state[p] = initial_value; } var args = parseFunctionArguments(argc, argv); diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index eecd3156..f9b4ff2d 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -6,8 +6,8 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "sum", { - step: function (state, value) { return (state || 0) + value; }, + "sum", 0, { + step: function (state, value) { return state + value; }, } ); @@ -17,10 +17,11 @@ exports.test = function (SQL, assert) { assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); db.create_aggregate( - "percentile", { - init: function () { return { vals: [], pctile: null }; }, // init + "percentile", { vals: [], pctile: null }, + { step: function (state, value, pctile) { - if (value && !isNaN(value)) { + var typ = typeof value; + if (typ == "number" || typ == "bigint") { state.pctile = pctile; state.vals.push(value); } @@ -35,9 +36,9 @@ exports.test = function (SQL, assert) { assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); db.create_aggregate( - "json_agg", { - step: function(state, val) { state = (state || []); state.push(val); return state; }, - finalize: function(state) { return JSON.stringify(state); } + "json_agg", [], { + step: (state, val) => [...state, val], + finalize: (state) => JSON.stringify(state), } ); From bf22aa164c469338b9c7e5d3db05b931ead5b55d Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 08:42:36 -0400 Subject: [PATCH 17/36] test step and finalize exceptions --- test/test_aggregate_functions.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index f9b4ff2d..87cafe90 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -45,10 +45,34 @@ exports.test = function (SQL, assert) { db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); result = db.exec("SELECT json_agg(col) FROM test2;"); - assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); + assert.deepEqual( + JSON.parse(result[0].values[0]), + ["four score", "and seven", "years ago"], + "Aggregate function that returns JSON" + ); result = db.exec("SELECT json_agg(col), json_agg(col2) FROM test2;"); - assert.deepEqual(result[0].values[0].map(JSON.parse), [["four score", "and seven", "years ago"], [12, 7, 1]], "Multiple aggregations at once"); + assert.deepEqual( + result[0].values[0].map(JSON.parse), + [["four score", "and seven", "years ago"], [12, 7, 1]], + "Multiple aggregations at once" + ); + + db.create_aggregate("throws_step", 0, {step: (state, value) => { throw "bananas" }}) + assert.throws( + () => db.exec("SELECT throws_step(col) FROM test;"), + "Error: bananas", + "Handles exception in a step function" + ); + + db.create_aggregate("throws_finalize", 0, { + step: (state, value) => state + value, + finalize: (state) => { throw "shoes" }}) + assert.throws( + () => db.exec("SELECT throws_finalize(col) FROM test;"), + "Error: shoes", + "Handles exception in a finalize function" + ); } // helper function to calculate a percentile from an array. Will modify the From 55858e9fbd80127da39a7503522bd5616698a06f Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 08:55:32 -0400 Subject: [PATCH 18/36] add memory leak test --- test/test_aggregate_redefinition.js | 80 +++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/test_aggregate_redefinition.js diff --git a/test/test_aggregate_redefinition.js b/test/test_aggregate_redefinition.js new file mode 100644 index 00000000..9ed47dfa --- /dev/null +++ b/test/test_aggregate_redefinition.js @@ -0,0 +1,80 @@ +exports.test = function(sql, assert) { + // Test 1: Create a database, Register single function, close database, repeat 1000 times + for (var i = 1; i <= 1000; i++) + { + let lastStep = i == 1000; + let db = new sql.Database(); + try { + db.create_aggregate("TestFunction"+i, 0, {step: (state, value) => i}) + } catch(e) { + assert.ok( + false, + "Test 1: Recreate database "+i+"th times and register aggregate" + +" function failed with exception:"+e + ); + db.close(); + break; + } + var result = db.exec("SELECT TestFunction"+i+"(1)"); + var result_str = result[0]["values"][0][0]; + if(result_str != i || lastStep) + { + assert.equal( + result_str, + i, + "Test 1: Recreate database "+i+"th times and register aggregate function" + ); + db.close(); + break; + } + db.close(); + } + + // Test 2: Create a database, Register same function 1000 times, close database + { + let db = new sql.Database(); + for (var i = 1; i <= 1000; i++) + { + let lastStep = i == 1000; + try { + db.create_aggregate("TestFunction", 0, {step: (state, value) => i}) + } catch(e) { + assert.ok( + false, + "Test 2: Reregister aggregate function "+i+"th times failed with" + +" exception:"+e + ); + break; + } + var result = db.exec("SELECT TestFunction(1)"); + var result_str = result[0]["values"][0][0]; + if(result_str != i || lastStep) + { + assert.equal( + result_str, + i, + "Test 2: Reregister function "+i+"th times" + ); + break; + } + } + db.close(); + } +}; + + +if (module == require.main) { + const target_file = process.argv[2]; + const sql_loader = require('./load_sql_lib'); + sql_loader(target_file).then((sql)=>{ + require('test').run({ + 'test creating multiple functions': function(assert){ + exports.test(sql, assert); + } + }); + }) + .catch((e)=>{ + console.error(e); + assert.fail(e); + }); +} From 9a0c185d30697537d9c774ba262326f82f33faa0 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 09:03:00 -0400 Subject: [PATCH 19/36] update docs to current interface --- README.md | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b779eabb..67c32274 100644 --- a/README.md +++ b/README.md @@ -75,37 +75,25 @@ db.create_function("add_js", add); // Run a query in which the function is used db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world' -// You can create aggregation functions, by passing a name and three functions -// to `db.create_aggregate`: +// You can create custom aggregation functions, by passing a name, an initial +// value, and two functions to `db.create_aggregate`: // -// - an init function. This function receives no arguments and will be called -// when the aggregate begins. Returns a state object that will be passed to the -// other two functions if you need to track state. // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It -// will be called on every value to be aggregated. Does not return anything. +// will be called on every value to be aggregated, and its return value +// will be used as the state for the next iteration. // - a finalizer. This function receives one argument, the state object, and -// returns the final value of the aggregate +// returns the final value of the aggregate. It can be omitted, in which case +// the value of the `state` variable will be used. // // Here is an example aggregation function, `json_agg`, which will collect all // input values and return them as a JSON array: db.create_aggregate( "json_agg", - function() { - // This is the init function, which returns a state object: - return { - values: [] - }; - }, - function(state, val) { - // This is the step function, which will store each value it receives in - // the values array of the state object - state.values.push(val); - }, - function(state) { - // This is the finalize function, which converts the received values from - // the state object into a JSON array and returns that - return JSON.stringify(state.values); + [], + { + step: (state, val) => state.push(val), + finalize: (state) => JSON.stringify(state), } ); From 2445107948ccc348551e729f75eaf0b7fd1b530e Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 09:16:58 -0400 Subject: [PATCH 20/36] delete state in exception handlers --- src/api.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/api.js b/src/api.js index ecb77aec..cc2abcfb 100644 --- a/src/api.js +++ b/src/api.js @@ -1295,6 +1295,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var p = sqlite3_aggregate_context(cx, 1); // If this is the first invocation of wrapped_step, call `init` + // + // Make sure that every path through the step and finalize + // functions deletes the value state[p] when it's done so we don't + // leak memory and possibly stomp the init value of future calls if (!state[p]) { state[p] = initial_value; } @@ -1304,6 +1308,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { try { state[p] = aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { + delete state[p]; sqlite3_result_error(cx, error, -1); } } @@ -1314,13 +1319,14 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { try { result = aggregateFunctions["finalize"].apply(null, [state[p]]); } catch (error) { + delete state[p]; sqlite3_result_error(cx, error, -1); state = null; return; } + setFunctionResult(cx, result); - // clear the state for this invocation delete state[p]; } From 5b62cf60b82238c16f8af451b70423ae0065e50a Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 09:19:58 -0400 Subject: [PATCH 21/36] remove null state --- src/api.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api.js b/src/api.js index cc2abcfb..93e905db 100644 --- a/src/api.js +++ b/src/api.js @@ -1321,7 +1321,6 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { } catch (error) { delete state[p]; sqlite3_result_error(cx, error, -1); - state = null; return; } From 062f147e605d4c65093cba8613baca15f674fde0 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 10:51:04 -0400 Subject: [PATCH 22/36] return init function and document object --- src/api.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/api.js b/src/api.js index 93e905db..5dbf4613 100644 --- a/src/api.js +++ b/src/api.js @@ -1259,21 +1259,33 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { @param {string} name the name of the aggregate as referenced in SQL statements. - @param {object} Aggregate function containing three functions + @param {object} Aggregate function containing at least a step function. + Valid keys for this object are: + - init: a function receiving no arguments and returning an initial + value for the aggregate function. The initial value will be + null if this key is omitted. + - step (required): a function receiving the current state and one to + many values and returning an updated state value. + Will receive the value from init for the first step. + - finalize: a function returning the final value of the aggregate + function. If omitted, the value returned by the last step + wil be used as the final value. @return {Database} The database object. Useful for method chaining */ Database.prototype["create_aggregate"] = function create_aggregate( name, - initial_value, aggregateFunctions ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") ) { - throw "An aggregate function must have a step function"; + throw "An aggregate function must have a step function in " + name; } - // Default finalizer + // Default initializer and finalizer + function init() { return null; } function finalize(state) { return state; } + + aggregateFunctions["init"] = aggregateFunctions["init"] || init; aggregateFunctions["finalize"] = aggregateFunctions["finalize"] || finalize; @@ -1299,8 +1311,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // Make sure that every path through the step and finalize // functions deletes the value state[p] when it's done so we don't // leak memory and possibly stomp the init value of future calls - if (!state[p]) { - state[p] = initial_value; + if (!Object.hasOwnProperty.call(state, p)) { + state[p] = aggregateFunctions["init"].apply(null); } var args = parseFunctionArguments(argc, argv); From 7aff1aeda4deee6ef8889833295b8b5a7e5e9d28 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 10:51:16 -0400 Subject: [PATCH 23/36] more tests and update back to init function --- test/test_aggregate_functions.js | 54 +++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 87cafe90..59029b93 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -6,8 +6,8 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "sum", 0, { - step: function (state, value) { return state + value; }, + "sum", { + step: function (state, value) { return (state || 0) + value; }, } ); @@ -17,11 +17,12 @@ exports.test = function (SQL, assert) { assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); db.create_aggregate( - "percentile", { vals: [], pctile: null }, + "percentile", { + init: function() { return { vals: [], pctile: null }}, step: function (state, value, pctile) { var typ = typeof value; - if (typ == "number" || typ == "bigint") { + if (typ == "number" || typ == "bigint") { // ignore nulls state.pctile = pctile; state.vals.push(value); } @@ -36,7 +37,8 @@ exports.test = function (SQL, assert) { assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); db.create_aggregate( - "json_agg", [], { + "json_agg", { + init: () => [], step: (state, val) => [...state, val], finalize: (state) => JSON.stringify(state), } @@ -58,21 +60,51 @@ exports.test = function (SQL, assert) { "Multiple aggregations at once" ); - db.create_aggregate("throws_step", 0, {step: (state, value) => { throw "bananas" }}) + db.create_aggregate("is_even", { + init: () => true, + step: state => !state + }); + result = db.exec("SELECT is_even() FROM (VALUES (1),(2),(0));"); + assert.deepEqual( + result[0].values[0][0], + 0, // this gets convert from "false" to an int by sqlite + "Aggregate functions respect falsy values" + ); + + db.create_aggregate("sum_non_zero", { + init: () => 0, + step: (state, value) => { + if (!value) throw "bananas"; + return state + value + } + }) assert.throws( - () => db.exec("SELECT throws_step(col) FROM test;"), + () => db.exec("SELECT sum_non_zero(column1) FROM (VALUES (1),(2),(0));"), "Error: bananas", "Handles exception in a step function" ); + assert.deepEqual( + db.exec("SELECT sum_non_zero(column1) FROM (VALUES (1),(2));")[0].values[0][0], + 3, + "Aggregate functions work after an exception has been thrown in step" + ); - db.create_aggregate("throws_finalize", 0, { - step: (state, value) => state + value, - finalize: (state) => { throw "shoes" }}) + db.create_aggregate("throws_finalize", { + step: (state, value) => (state || 0) + value, + finalize: (state) => { + if (!state) throw "shoes" + return state; + }}) assert.throws( - () => db.exec("SELECT throws_finalize(col) FROM test;"), + () => db.exec("SELECT throws_finalize(column1) FROM (VALUES (0));"), "Error: shoes", "Handles exception in a finalize function" ); + assert.deepEqual( + db.exec("SELECT throws_finalize(column1) FROM (VALUES (1),(2));")[0].values[0][0], + 3, + "Aggregate functions work after an exception has been thrown in finalize" + ); } // helper function to calculate a percentile from an array. Will modify the From 67f85e5ca600ffa1055bc7f2dbff82dceaf0baaf Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 11:05:55 -0400 Subject: [PATCH 24/36] update redefinition test for new interface --- test/test_aggregate_redefinition.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_aggregate_redefinition.js b/test/test_aggregate_redefinition.js index 9ed47dfa..05779720 100644 --- a/test/test_aggregate_redefinition.js +++ b/test/test_aggregate_redefinition.js @@ -5,7 +5,7 @@ exports.test = function(sql, assert) { let lastStep = i == 1000; let db = new sql.Database(); try { - db.create_aggregate("TestFunction"+i, 0, {step: (state, value) => i}) + db.create_aggregate("TestFunction"+i, {step: (state, value) => i}) } catch(e) { assert.ok( false, @@ -37,7 +37,7 @@ exports.test = function(sql, assert) { { let lastStep = i == 1000; try { - db.create_aggregate("TestFunction", 0, {step: (state, value) => i}) + db.create_aggregate("TestFunction", {step: (state, value) => i}) } catch(e) { assert.ok( false, From b8692d439cae6a2f5e84178b744b5a8497b3bbb2 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 11:59:48 -0400 Subject: [PATCH 25/36] update README to match fixed signature --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 67c32274..d4fb2c1a 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,11 @@ db.create_function("add_js", add); // Run a query in which the function is used db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world' -// You can create custom aggregation functions, by passing a name, an initial -// value, and two functions to `db.create_aggregate`: +// You can create custom aggregation functions, by passing a name +// and a set of functions to `db.create_aggregate`: // +// - a initialization function. This function receives no arguments and returns +// the initial value for the aggregation function // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It // will be called on every value to be aggregated, and its return value @@ -90,8 +92,8 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // input values and return them as a JSON array: db.create_aggregate( "json_agg", - [], { + init: () => [], step: (state, val) => state.push(val), finalize: (state) => JSON.stringify(state), } From b41e5cf7e6bf6b7afac76daea450091c944d6cf3 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 12:07:55 -0400 Subject: [PATCH 26/36] more consistent test formatting --- test/test_aggregate_functions.js | 52 ++++++++++++++------------------ 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 59029b93..d28c775f 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -5,44 +5,37 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); - db.create_aggregate( - "sum", { - step: function (state, value) { return (state || 0) + value; }, - } - ); + db.create_aggregate("sum", { + step: function (state, value) { return (state || 0) + value; }, + }); db.exec("CREATE TABLE test (col);"); db.exec("INSERT INTO test VALUES (1), (2), (3), (null);"); var result = db.exec("SELECT sum(col) FROM test;"); assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); - db.create_aggregate( - "percentile", - { - init: function() { return { vals: [], pctile: null }}, - step: function (state, value, pctile) { - var typ = typeof value; - if (typ == "number" || typ == "bigint") { // ignore nulls - state.pctile = pctile; - state.vals.push(value); - } - return state; - }, - finalize: function (state) { - return percentile(state.vals, state.pctile); + db.create_aggregate("percentile", { + init: function() { return { vals: [], pctile: null }}, + step: function (state, value, pctile) { + var typ = typeof value; + if (typ == "number" || typ == "bigint") { // ignore nulls + state.pctile = pctile; + state.vals.push(value); } + return state; + }, + finalize: function (state) { + return percentile(state.vals, state.pctile); } - ); + }); result = db.exec("SELECT percentile(col, 80) FROM test;"); assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); - db.create_aggregate( - "json_agg", { - init: () => [], - step: (state, val) => [...state, val], - finalize: (state) => JSON.stringify(state), - } - ); + db.create_aggregate("json_agg", { + init: () => [], + step: (state, val) => [...state, val], + finalize: (state) => JSON.stringify(state), + }); db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); @@ -77,7 +70,7 @@ exports.test = function (SQL, assert) { if (!value) throw "bananas"; return state + value } - }) + }); assert.throws( () => db.exec("SELECT sum_non_zero(column1) FROM (VALUES (1),(2),(0));"), "Error: bananas", @@ -94,7 +87,8 @@ exports.test = function (SQL, assert) { finalize: (state) => { if (!state) throw "shoes" return state; - }}) + } + }); assert.throws( () => db.exec("SELECT throws_finalize(column1) FROM (VALUES (0));"), "Error: shoes", From d257bbab67e3344a2339988742649764c5da9b70 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 12:39:51 -0400 Subject: [PATCH 27/36] Update README.md Co-authored-by: Ophir LOJKINE --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4fb2c1a..1d0068e9 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ db.create_aggregate( "json_agg", { init: () => [], - step: (state, val) => state.push(val), + step: (state, val) => [...state, val], finalize: (state) => JSON.stringify(state), } ); From e82c2862acfd1665642fb60c66adbde3af86717c Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 12:43:24 -0400 Subject: [PATCH 28/36] clarify what exactly the result will contain --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d0068e9..71e70416 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,7 @@ db.create_aggregate( // Now if you run this query: var result = db.exec("SELECT json_agg(somecol) FROM atable;"); - -// result will be a json-encoded string representing each value of `somecol` in `atable`. +console.log("You'll get a json-encoded list of values: ", result[0].values[0]) // Export the database to an Uint8Array containing the SQLite database file const binaryArray = db.export(); From b65457cbaae8d4f78716adb2055580ef3adb519b Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:31:22 +0200 Subject: [PATCH 29/36] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71e70416..9f50b830 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // You can create custom aggregation functions, by passing a name // and a set of functions to `db.create_aggregate`: // -// - a initialization function. This function receives no arguments and returns +// - an initialization function. This function receives no arguments and returns // the initial value for the aggregation function // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It From 8d2c2e0d053498c262f13efdd0fd32a894992b88 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:31:52 +0200 Subject: [PATCH 30/36] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f50b830..344788c6 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // You can create custom aggregation functions, by passing a name // and a set of functions to `db.create_aggregate`: // -// - an initialization function. This function receives no arguments and returns +// - an initialization function. This function receives no argument and returns // the initial value for the aggregation function // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It From f8f4a7c8740e72d39e16f74315952686c2d9ac45 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:37:25 +0200 Subject: [PATCH 31/36] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 344788c6..aa444607 100644 --- a/README.md +++ b/README.md @@ -78,15 +78,15 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // You can create custom aggregation functions, by passing a name // and a set of functions to `db.create_aggregate`: // -// - an initialization function. This function receives no argument and returns -// the initial value for the aggregation function -// - a step function. This function receives as a first argument the state -// object created in init, as well as the values received in the step. It -// will be called on every value to be aggregated, and its return value -// will be used as the state for the next iteration. -// - a finalizer. This function receives one argument, the state object, and +// - an `init` function. This function receives no argument and returns +// the initial value for the state of the aggregate function. +// - a `step` function. This function takes two arguments +// - the current state of the aggregation +// - a new value to aggregate to the state +// It should return a new value for the state. +// - a `finalize` function. This function receives a state object, and // returns the final value of the aggregate. It can be omitted, in which case -// the value of the `state` variable will be used. +// the final value of the state will be returned directly by the aggregate function. // // Here is an example aggregation function, `json_agg`, which will collect all // input values and return them as a JSON array: From bdaa1b6d45b0699bdc863bc74d892a919e25eefa Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:41:02 +0200 Subject: [PATCH 32/36] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa444607..00a5da28 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,9 @@ db.create_aggregate( } ); -// Now if you run this query: -var result = db.exec("SELECT json_agg(somecol) FROM atable;"); -console.log("You'll get a json-encoded list of values: ", result[0].values[0]) +```suggestion +db.exec("SELECT json_agg(column1) FROM (VALUES ('hello'), ('world'))"); +// -> The result of the query is the string '["hello","world"]' // Export the database to an Uint8Array containing the SQLite database file const binaryArray = db.export(); From e86d7ff81127fa764b42b54f2ea0cc9e4688ad3a Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:41:20 +0200 Subject: [PATCH 33/36] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 00a5da28..4e98ee5e 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,6 @@ db.create_aggregate( } ); -```suggestion db.exec("SELECT json_agg(column1) FROM (VALUES ('hello'), ('world'))"); // -> The result of the query is the string '["hello","world"]' From 423fc3615557333bba30c442bb3abd8daeddc875 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Thu, 8 Sep 2022 00:18:00 +0000 Subject: [PATCH 34/36] Improve documentation and type annotations --- src/api.js | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/api.js b/src/api.js index 5dbf4613..cb7052b1 100644 --- a/src/api.js +++ b/src/api.js @@ -1200,7 +1200,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { }; /** Register a custom function with SQLite - @example Register a simple function + @example Register a simple function db.create_function("addOne", function (x) {return x+1;}) db.exec("SELECT addOne(1)") // = 2 @@ -1247,30 +1247,33 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { }; /** Register a custom aggregate with SQLite - @example Register a aggregate function - db.create_aggregate( - "js_sum", - function () { return { sum: 0 }; }, - function (state, value) { state.sum+=value; }, - function (state) { return state.sum; } - ); - db.exec("CREATE TABLE test (col); INSERT INTO test VALUES (1), (2)"); - db.exec("SELECT js_sum(col) FROM test"); // = 3 + @example Register a custom sum function + db.create_aggregate("js_sum", { + init: () => 0, + step: (state, value) => state + value, + finalize: state => state + }); + db.exec("SELECT js_sum(column1) FROM (VALUES (1), (2))"); // = 3 @param {string} name the name of the aggregate as referenced in SQL statements. - @param {object} Aggregate function containing at least a step function. - Valid keys for this object are: - - init: a function receiving no arguments and returning an initial - value for the aggregate function. The initial value will be - null if this key is omitted. - - step (required): a function receiving the current state and one to - many values and returning an updated state value. - Will receive the value from init for the first step. - - finalize: a function returning the final value of the aggregate - function. If omitted, the value returned by the last step - wil be used as the final value. + @param {object} aggregateFunctions + object containing at least a step function. + @param {function(): T} [aggregateFunctions.init = ()=>null] + a function receiving no arguments and returning an initial + value for the aggregate function. The initial value will be + null if this key is omitted. + @param {function(T, any) : T} aggregateFunctions.step + a function receiving the current state and a value to aggregate + and returning a new state. + Will receive the value from init for the first step. + @param {function(T): any} [aggregateFunctions.finalize = (state)=>state] + a function returning the result of the aggregate function + given its final state. + If omitted, the value returned by the last step + will be used as the final value. @return {Database} The database object. Useful for method chaining + @template T */ Database.prototype["create_aggregate"] = function create_aggregate( name, From f8e7bd3012fbf00444c96f2d1a99b3865b0e39d4 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Thu, 8 Sep 2022 00:38:14 +0000 Subject: [PATCH 35/36] ignore documentation in eslintrc --- .eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.js b/.eslintrc.js index f730a261..02e6047c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -16,6 +16,7 @@ module.exports = { ignorePatterns: [ "/dist/", "/examples/", + "/documentation/", "/node_modules/", "/out/", "/src/shell-post.js", From 799ebcd1b70b4ac1f2c097e23cf6a1232c2b39a4 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Thu, 8 Sep 2022 00:39:08 +0000 Subject: [PATCH 36/36] reduce code size --- src/api.js | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/src/api.js b/src/api.js index cb7052b1..ec8c2fe7 100644 --- a/src/api.js +++ b/src/api.js @@ -1279,18 +1279,16 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { name, aggregateFunctions ) { - if (!Object.hasOwnProperty.call(aggregateFunctions, "step") - ) { - throw "An aggregate function must have a step function in " + name; - } - // Default initializer and finalizer - function init() { return null; } - function finalize(state) { return state; } + var init = aggregateFunctions["init"] + || function init() { return null; }; + var finalize = aggregateFunctions["finalize"] + || function finalize(state) { return state; }; + var step = aggregateFunctions["step"]; - aggregateFunctions["init"] = aggregateFunctions["init"] || init; - aggregateFunctions["finalize"] = aggregateFunctions["finalize"] - || finalize; + if (!step) { + throw "An aggregate function must have a step function in " + name; + } // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of @@ -1314,14 +1312,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // Make sure that every path through the step and finalize // functions deletes the value state[p] when it's done so we don't // leak memory and possibly stomp the init value of future calls - if (!Object.hasOwnProperty.call(state, p)) { - state[p] = aggregateFunctions["init"].apply(null); - } + if (!Object.hasOwnProperty.call(state, p)) state[p] = init(); var args = parseFunctionArguments(argc, argv); var mergedArgs = [state[p]].concat(args); try { - state[p] = aggregateFunctions["step"].apply(null, mergedArgs); + state[p] = step.apply(null, mergedArgs); } catch (error) { delete state[p]; sqlite3_result_error(cx, error, -1); @@ -1332,28 +1328,24 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var result; var p = sqlite3_aggregate_context(cx, 1); try { - result = aggregateFunctions["finalize"].apply(null, [state[p]]); + result = finalize(state[p]); } catch (error) { delete state[p]; sqlite3_result_error(cx, error, -1); return; } - setFunctionResult(cx, result); - delete state[p]; } - if (Object.prototype.hasOwnProperty.call(this.functions, name)) { + if (Object.hasOwnProperty.call(this.functions, name)) { removeFunction(this.functions[name]); delete this.functions[name]; } - if (Object.prototype.hasOwnProperty.call( - this.functions, - name + "__finalize" - )) { - removeFunction(this.functions[name + "__finalize"]); - delete this.functions[name + "__finalize"]; + var finalize_name = name + "__finalize"; + if (Object.hasOwnProperty.call(this.functions, finalize_name)) { + removeFunction(this.functions[finalize_name]); + delete this.functions[finalize_name]; } // The signature of the wrapped function is : // void wrapped(sqlite3_context *db, int argc, sqlite3_value **argv) @@ -1363,7 +1355,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // void wrapped(sqlite3_context *db) var finalize_ptr = addFunction(wrapped_finalize, "vi"); this.functions[name] = step_ptr; - this.functions[name + "__finalize"] = finalize_ptr; + this.functions[finalize_name] = finalize_ptr; // passing null to the sixth parameter defines this as an aggregate // function @@ -1374,7 +1366,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { this.handleError(sqlite3_create_function_v2( this.db, name, - aggregateFunctions["step"].length - 1, + step.length - 1, SQLITE_UTF8, 0, 0,