Add cmlt transform option and update documentation

src/docs/FILTERS.md

@@ -1,5 +1,15 @@
 # OpenAF processor filters
 
+The OpenAF processor filters are applied in the following order:
+
+**input data** -> _path=..._ -> **apply transformers to data** -> _from=..._ -> _sql=..._ -> **output data** 
+
+---
+
+For more advanced cases they can also be applied in the following order:
+
+**input data** -> _ifrom=..._ -> _isql=..._ -> _path=..._ -> **apply transformers to data** -> _from=..._ -> _sql=..._ -> _opath=..._ -> **output data** 
+
 ## 🪚 Path
 
 The _path=_ filter tool is based on the JMESPath library. To see all the available options please refer to http://jmespath.org. Here are some examples:

src/docs/USAGE.md

@@ -77,6 +77,7 @@ These options will change the parsed input data included any filters provided.
 | arraytomap | Boolean | If true will try to convert the input array to a map (see arraytomapkey, arraytomapkeepkey) |
 | arraytomapkeepkey | Boolean | If true and arraytomap=true the defined arraytomapkey won't be removed from each map |
 | arraytomapkey | String | For arraytomap=true defines the name of the map property that will be each element key (see arraytomapkeepkey) |
+| cmlt | Boolean | If true will accumulate the input values into an output array (useful with loop) |
 | correcttypes | Boolean | If true will try to convert alpha-numeric field values with just numbers to number fields, string date fields to dates and boolean fields |
 | flatmap | Boolean | If true a map structure will be flat to just one level |
 | jsonschema | String | The JSON schema file to use for validation returning a map with a boolean valid and errors if exist |
@@ -259,6 +260,17 @@ List of options to use when _in=llm_ or _llmprompt=..._:
 
 ---
 
+## 🧾 CMLT transform options
+
+List of options to use when _cmlt=true_:
+
+| Option | Type | Description |
+|--------|------|-------------|
+| cmltch | String | A JSON/SLON OpenAF channel configuration string with type and options/url (defaults to simple) |
+| cmltsize | Number | The number of input data values to keep (default 100). If -1 it will keep without a limit |
+
+---
+
 ## 🧾 Log output options
 
 List of options to use when _out=log_:

src/include/transformFns.js

@@ -5,6 +5,31 @@ var _transformFns = {
             return _t
         }
     },
+    "cmlt"    : r => {
+        if (toBoolean(params.cmlt)) {
+            var _r = (isArray(r) ? r : [ r ])
+            params.cmltch = _$(params.cmltch, "cmltch").default("(type: simple)")
+            let cmltch = _fromJSSLON(params.cmltch)
+            if (isMap(cmltch)) {
+                if (isUnDef(cmltch.type)) _exit(-1, "cmltch.type is not defined.")
+                if (isDef(cmltch.lib)) loadLib(cmltch.lib)
+                if ($ch().list().indexOf("oafp::cmltdata") < 0) {
+                    if (cmltch.type == "remote") {
+                        $ch("oafp::cmltdata").createRemote(cmltch.url)
+                    } else {
+                        $ch("oafp::cmltdata").create(cmltch.type, cmltch.options)
+                    }
+                    let _sz = Number(_$(params.cmltsize, "cmltsize").isNumber().default(100)) - 1
+                    if (_sz > 0) $ch("oafp::cmltdata").subscribe(ow.ch.utils.getHousekeepSubscriber("oafp::cmltdata", _sz))
+                }
+
+                _r.forEach(_rt => $ch("oafp::cmltdata").set({ t: nowNano() }, _rt))
+                return $ch("oafp::cmltdata").getAll()
+            } else {
+                _exit(-1, "Invalid cmltch parameter")
+            }
+        }
+    },
     "jsonschemagen" : _r => {
         if (toBoolean(params.jsonschemagen)) {
             ow.loadObj()

src/include/utilFns.js

@@ -8,12 +8,29 @@ const _transform = r => {
     return r
 }
 const _$f = (r, options) => {
+    if (options.__ifrom) {
+        r = $from(r).query(af.fromNLinq(options.__ifrom.trim()))
+        delete options.__ifrom
+    }
+    if (options.__isql) {
+        var method = __
+        if (isString(params.sqlfilter)) {
+            switch(params.sqlfilter.toLowerCase()) {
+            case "simple"  : method = "nlinq"; break
+            case "advanced": method = "h2"; break
+            default        : method = __
+            }
+        }
+        if (isArray(r) && r.length > 0) r = $sql(r, options.__isql.trim(), method)
+        delete options.__isql
+    }
     if (options.__path) {
         r = $path(r, options.__path.trim())
         delete options.__path
     }
 
     if (isString(r)) return _transform(r)
+    r = _transform(r)
 
     if (options.__from) {
         r = $from(r).query(af.fromNLinq(options.__from.trim()))
@@ -31,7 +48,10 @@ const _$f = (r, options) => {
         if (isArray(r) && r.length > 0) r = $sql(r, options.__sql.trim(), method)
         delete options.__sql
     }
-    r = _transform(r)
+    if (options.__opath) {
+        r = $path(r, options.__opath.trim())
+        delete options.__opath
+    }
 
     return r
 }

src/oafp.source.js.hbs

@@ -278,7 +278,18 @@ if (isDef(params.secKey)) {
 }
 
 // Set options
-var options = { __format: params.format, __from: params.from, __sql: params.sql, __path: params.path, __csv: params.csv, __pause: params.pause, __key: params.__key }
+var options = { 
+    __format: params.format, 
+    __from: params.from, 
+    __ifrom: params.ifrom, 
+    __isql: params.isql, 
+    __sql: params.sql, 
+    __path: params.path, 
+    __opath: params.opath,
+    __csv: params.csv, 
+    __pause: params.pause, 
+    __key: params.__key 
+}
 // ndjson options
 /*if (params.type == "ndjson") {
     params.ndjsonjoin = toBoolean(_$(params.ndjsonjoin, "ndjsonjoin").isString().default(__))