diff --git a/src/docs/FILTERS.md b/src/docs/FILTERS.md index 5759508..a15305b 100644 --- a/src/docs/FILTERS.md +++ b/src/docs/FILTERS.md @@ -1,5 +1,15 @@ # OpenAF processor filters +The OpenAF processor filters are applied in the following order: + +**input data** -> _path=..._ -> **apply transformers to data** -> _from=..._ -> _sql=..._ -> **output data** + +--- + +For more advanced cases they can also be applied in the following order: + +**input data** -> _ifrom=..._ -> _isql=..._ -> _path=..._ -> **apply transformers to data** -> _from=..._ -> _sql=..._ -> _opath=..._ -> **output data** + ## 🪚 Path The _path=_ filter tool is based on the JMESPath library. To see all the available options please refer to http://jmespath.org. Here are some examples: diff --git a/src/docs/USAGE.md b/src/docs/USAGE.md index 5db0df3..9858767 100644 --- a/src/docs/USAGE.md +++ b/src/docs/USAGE.md @@ -77,6 +77,7 @@ These options will change the parsed input data included any filters provided. | arraytomap | Boolean | If true will try to convert the input array to a map (see arraytomapkey, arraytomapkeepkey) | | arraytomapkeepkey | Boolean | If true and arraytomap=true the defined arraytomapkey won't be removed from each map | | arraytomapkey | String | For arraytomap=true defines the name of the map property that will be each element key (see arraytomapkeepkey) | +| cmlt | Boolean | If true will accumulate the input values into an output array (useful with loop) | | correcttypes | Boolean | If true will try to convert alpha-numeric field values with just numbers to number fields, string date fields to dates and boolean fields | | flatmap | Boolean | If true a map structure will be flat to just one level | | jsonschema | String | The JSON schema file to use for validation returning a map with a boolean valid and errors if exist | @@ -259,6 +260,17 @@ List of options to use when _in=llm_ or _llmprompt=..._: --- +## 🧾 CMLT transform options + +List of options to use when _cmlt=true_: + +| Option | Type | Description | +|--------|------|-------------| +| cmltch | String | A JSON/SLON OpenAF channel configuration string with type and options/url (defaults to simple) | +| cmltsize | Number | The number of input data values to keep (default 100). If -1 it will keep without a limit | + +--- + ## 🧾 Log output options List of options to use when _out=log_: diff --git a/src/include/transformFns.js b/src/include/transformFns.js index 31bcda5..2fc2673 100644 --- a/src/include/transformFns.js +++ b/src/include/transformFns.js @@ -5,6 +5,31 @@ var _transformFns = { return _t } }, + "cmlt" : r => { + if (toBoolean(params.cmlt)) { + var _r = (isArray(r) ? r : [ r ]) + params.cmltch = _$(params.cmltch, "cmltch").default("(type: simple)") + let cmltch = _fromJSSLON(params.cmltch) + if (isMap(cmltch)) { + if (isUnDef(cmltch.type)) _exit(-1, "cmltch.type is not defined.") + if (isDef(cmltch.lib)) loadLib(cmltch.lib) + if ($ch().list().indexOf("oafp::cmltdata") < 0) { + if (cmltch.type == "remote") { + $ch("oafp::cmltdata").createRemote(cmltch.url) + } else { + $ch("oafp::cmltdata").create(cmltch.type, cmltch.options) + } + let _sz = Number(_$(params.cmltsize, "cmltsize").isNumber().default(100)) - 1 + if (_sz > 0) $ch("oafp::cmltdata").subscribe(ow.ch.utils.getHousekeepSubscriber("oafp::cmltdata", _sz)) + } + + _r.forEach(_rt => $ch("oafp::cmltdata").set({ t: nowNano() }, _rt)) + return $ch("oafp::cmltdata").getAll() + } else { + _exit(-1, "Invalid cmltch parameter") + } + } + }, "jsonschemagen" : _r => { if (toBoolean(params.jsonschemagen)) { ow.loadObj() diff --git a/src/include/utilFns.js b/src/include/utilFns.js index dc7127e..68eb204 100644 --- a/src/include/utilFns.js +++ b/src/include/utilFns.js @@ -8,12 +8,29 @@ const _transform = r => { return r } const _$f = (r, options) => { + if (options.__ifrom) { + r = $from(r).query(af.fromNLinq(options.__ifrom.trim())) + delete options.__ifrom + } + if (options.__isql) { + var method = __ + if (isString(params.sqlfilter)) { + switch(params.sqlfilter.toLowerCase()) { + case "simple" : method = "nlinq"; break + case "advanced": method = "h2"; break + default : method = __ + } + } + if (isArray(r) && r.length > 0) r = $sql(r, options.__isql.trim(), method) + delete options.__isql + } if (options.__path) { r = $path(r, options.__path.trim()) delete options.__path } if (isString(r)) return _transform(r) + r = _transform(r) if (options.__from) { r = $from(r).query(af.fromNLinq(options.__from.trim())) @@ -31,7 +48,10 @@ const _$f = (r, options) => { if (isArray(r) && r.length > 0) r = $sql(r, options.__sql.trim(), method) delete options.__sql } - r = _transform(r) + if (options.__opath) { + r = $path(r, options.__opath.trim()) + delete options.__opath + } return r } diff --git a/src/oafp.source.js.hbs b/src/oafp.source.js.hbs index 83cb2b3..958d995 100755 --- a/src/oafp.source.js.hbs +++ b/src/oafp.source.js.hbs @@ -278,7 +278,18 @@ if (isDef(params.secKey)) { } // Set options -var options = { __format: params.format, __from: params.from, __sql: params.sql, __path: params.path, __csv: params.csv, __pause: params.pause, __key: params.__key } +var options = { + __format: params.format, + __from: params.from, + __ifrom: params.ifrom, + __isql: params.isql, + __sql: params.sql, + __path: params.path, + __opath: params.opath, + __csv: params.csv, + __pause: params.pause, + __key: params.__key +} // ndjson options /*if (params.type == "ndjson") { params.ndjsonjoin = toBoolean(_$(params.ndjsonjoin, "ndjsonjoin").isString().default(__))