unused-eval-load.js

// code below is an INCOMPLETE attempt to use EVAL instead of Function
// and so can use 'with' statement (so NOT strict) to create scope-like
// layers of globals. These would then be presented as an array
// of Proxy objects, with those above (i.e. lower index) overriding
// those lower down in the array (i.e. with higher indexes)


// todo: other misc notes below to be pulled into the main README.md file
// todo: write up findings from below and Realms and SES


/* do NOT "use strict"; because that invalidates using 'with' statement below */
/* do NOT use 'import' or 'export' because that will implicitly triggers strict mode */
/* hence, we use the module.exports below form (which does not trigger strict mode) */

// ON LOADING modules from server: if package.json has BROWSER field:
// - if string, download that
// - if object AND requested file has redirect, use that
// - BUT, how to know that a file is being redirected???
//      - for each request below node_modules,
//          - need to read package.json first and see if redirected?


// git commit -am ".."

/*
    // AMD: https://requirejs.org/docs/api.html

    // according to: https://requirejs.org/docs/commonjs.html
    // define(function(...){}) must ALWAYS be:
    // - either 'require'
    // - or 'require,exports,module' IN THAT ORDER and with those EXACT NAMES

    // We only implement a small subset of AMD, as likely generated by webpack and rollup
    // Manual AMD modules may not work with load module
    // - if need manual, use our x-define
    // - if need to use someone else's manual AMD, sorry...

    // what we implement:
    // - require, module, exports as deps
    // - no-deps + function()
    // - no-deps + function(require)
    //      - mods are pre-extracted: dep names are simple strings
    // - no-deps + function(require,exports,module) (in any order)
    // - require & requirejs: ([deps], execFcn())

    // prefix in names: text! css! html!

    // NO config
    // no jsonp

    // AMD reqs:
    // - url is ID unless ends with .js, or starts with /, or a url protocol (http://)
    //   then uses baseUrl/urlAsID
    // - require or requirejs (apparently)
    // from https://requirejs.org/docs/api.html#define 
    // - define({can be object}) defines an object module (also string or array or other non-function? why not)
    // - define(function definitionFcn(){}) no deps and function.length === 0; returned from definitionFcn is module
    // - define(function(require, exports, module) {}); - no deps 1, 2, or 3 parms
    //      - whoaaa
    // - require CAN be asked as a dependency: if dep is 'require' will return require code as a dependency: so, a reserved dep name
    // - require.toUrl function is NOT implemented

    // 'exports' is special dependency case; can then be used as expotrs.x inside defFcn

    // start something is: requirejs([deps], starter(){});

    // loadModule.config() returns a loading function

    // we do NOT honor requirejs.config({...})

            // AMD using the the require format: need to extract required deps
*/


/* reference for future exploration: loading modules using <script type=module> techniques

    A big issue with our strategy is that we always wrap downloaded code inside a Function/AsyncFunction
    for the dynamic module's initialization:
    - this permits passing global variables available to module's initialization
    - BUT, this prevent ES6 usage of 'import/export' statements since those CANNOT be used within functions

    todo: find a way to 'detect' pure ES6 modules ahead of time
          ...then, find way to import them as is (i.e. no transpilation required)

    becomes a 3rd way to try and load module (1st 2 are amd, cjs)
    - either with pure eval() OR create <script type=module> tag and add to body

    issues with ES6 modules: import 'x'; what is x? relative to page/website? or NPM module? but then, from where?
    - probably should be controlled from website, so possibly a 302/redirect or just direct download from website (from
      its own node_modules folder)

    note: eval does NOT support import/export syntax so can't use it to load es6 module

    Closest to this will be dynamic imports: import(url).then(...)
    - but not currently supported in Firefox (experimental with manual switch) or Edge (no timeline)
   

    // method below "works" but no way to extract resulting module (so, not really loading a module)

    async function loadAsScript({url, code, type = 'module'}) {

        // used to load code directly as script
        // in particular, works for ES6 modules (i.e. which use non-transpiled import/export statements)

        // with this method, use either url or code (code used if both specified)

        // read: https://developer.mozilla.org/en-US/docs/Web/API/HTMLScriptElement#Dynamically_importing_scripts

        // LOTS OF ISSUES with loading js code using <script> technique (below)

        // BIGGEST ISSUE: script "result" (i.e. the module, or its exports) is NOT accessible
        //                to the outside world: there is no way to IMPORT whatever the module
        //                is exporting. So no easy way to load individual modules for composition/usage
        //                by other modules (e.g. a plugin system)

        // Other issues:
        //  - scripts load in full context of app
        //      - could be good (if trusted scripts) or not
        // - CANNOT set GLOBALS as per Function/AsyncFunction except by setting them VIA 'window.'
        // - maybe that's not so bad

        return new Promise((resolve,reject) => {

            // required for both methods
            const body = document.body;//getElementsByTagName('body')[0];
            const script = document.createElement('script');
            script.type=type;//.setAttribute('type', type); // must be 'module' if js code uses import/export

            // make it unique (in case used for window/global custom onload method)
            const scriptID = ('module_' + Math.random() ).replace(/[^a-z0-9_]+/ig,'');
            // script.id = scriptID;//.setAttribute('id', scriptID); // must be 'module' if js code uses import/export

            function getModuleResults() {
                console.log('ES6 Module resolving to x', script, script.module, script.exports);
                resolve(script);
            }

            script.onerror = function(...args) {

                // REGARDLESS of method: will be called on ANY code execution error
                console.log('JAVSCRIPT MODULE LOAD ERROR', url || code, args, ';;;');
                reject(new Error(`ES6 Module loaded but has errors`));
            }

            if (code) { // favor this one (if both specified) since code already downloaded

                // only onerror can be triggered (never onload) so no 'native' way to know it finished loading
                // our workaround is to explicitly append our we-b-done method
                // will NOT work if code does not execute till the end (e.g. a top-level return)

                // will need a window/global name for callback
                const customOnLoad = scriptID + '_loading_complete';

                // so required to know when initialization code ends
                window[customOnLoad] = () => {
                    //console.log('ES6 Module loaded OK, looks like it worked!', url || code);
                    getModuleResults();
                }

                // then (only onerror can be triggered so append our custom onload)...
                script.appendChild(document.createTextNode(`"use strict";\n\n${code}\n\n;${customOnLoad}()`));
            }
            else { // uses src attribute (browser will do the download)

                // required: to know when loaded
                script.onload = function(...args) {
                    
                    // CALLED (after execution) if using src=url (and code is good)
                    // NOT CALLED when loading via srcCode

                    //console.log('ES6 MODULE ONLOAD CALLED: well, something happened', args, ';;;');
                    getModuleResults();
                }

                // both onload and onerror can be triggered
                script.setAttribute('src', url); 
            }

            // trigger loading process...
            body.appendChild(script);
        });
    }
*/

/*
    // AMD: https://requirejs.org/docs/api.html


    // according to: https://requirejs.org/docs/commonjs.html
    // define(function(...){}) must ALWAYS be:
    // - either 'require'
    // - or 'require,exports,module' IN THAT ORDER and with those EXACT NAMES

    // We only implement a small subset of AMD, as likely generated by webpack and rollup
    // Manual AMD modules may not work with load module
    // - if need manual, use our x-define
    // - if need to use someone else's manual AMD, sorry...

    // what we implement:
    // - require, module, exports as deps
    // - no-deps + function()
    // - no-deps + function(require)
    //      - mods are pre-extracted: dep names are simple strings
    // - no-deps + function(require,exports,module) (in any order)
    // - require & requirejs: ([deps], execFcn())

    // prefix in names: text! css! html!

    // NO config
    // no jsonp

    // AMD reqs:
    // - url is ID unless ends with .js, or starts with /, or a url protocol (http://)
    //   then uses baseUrl/urlAsID
    // - require or requirejs (apparently)
    // from https://requirejs.org/docs/api.html#define 
    // - define({can be object}) defines an object module (also string or array or other non-function? why not)
    // - define(function definitionFcn(){}) no deps and function.length === 0; returned from definitionFcn is module
    // - define(function(require, exports, module) {}); - no deps 1, 2, or 3 parms
    //      - whoaaa
    // - require CAN be asked as a dependency: if dep is 'require' will return require code as a dependency: so, a reserved dep name
    // - require.toUrl function is NOT implemented

    // 'exports' is special dependency case; can then be used as expotrs.x inside defFcn

    // start something is: requirejs([deps], starter(){});

    // loadModule.config() returns a loading function

    // we do NOT honor requirejs.config({...})

            // AMD using the the require format: need to extract required deps
*/

// RULE: in order to import CJS modules, some code transpiling is required. In some edge cases, the transpiling may fail.
// for those cases, changing the edge case(s) if best else use an AMD version of that module (used as-is, no transpilation)
// edge cases:
//  - a regex which contains // or /* (both likely errors)
//  - comment(s) between 'require' and the opening paren '(' or within the parentheses (i.e. before the closing paren)
//  - embedded tick-quoted-text inside another tick-quoted-text: e.g. const x = `this is some ${choice? `1` : `2`} text`;
//      - although, if no require in between, might work since tick marks should be balanced (right?)

/* More involved Strategy 2 (unused for now)
    // find out how many deps the moduleDefine function expects
    // WHOAAA: BUT if function uses ...ARGS format (i.e. the spread/rest operator), FUNCTION.LENGTH === 0!!!
    const numDeps = moduleDefine.length; // ...function.length returns how many parms (so, deps) are declared for it

    // WHOAAA, part 2: if numDeps === 0, may mean NO parms, or means ...parms: how to proceed???
    // if (numDeps === 0 && args.length > 0)
    //     throw new ModuleLoadError(`define method takes no parms but some dependencies declared\n\t[possible issue: 'define(...parms){}' declaration format NOT supported]`); // give users a hint

    // we allow for either an array of deps (traditional) or just comma-separated parms (for convenience when creating amd modules manually)
    // we're also ok with a mixture of strings and arrays (of strings only), though not clear why that would be the case
    // and we always work backwards on parms (from right to left) to allow for possibility of a module name at the front/leftmost position
    // (as per traditional, in case first/leftmost parm is module's 'name', as is typical of AMD define([mod-name,][...deps,] fcn(...depRefs){}))
    // IF a module name is specified, it remains UNUSED (not needed for modules loaded by URLs)
    // POSSIBLE: if single string parm left (i.e. module name), maybe register it as the module's name also: i.e. as an alias
    // - but what happens if another module wants that name: overwrite? remove both? keep first? keep both?
    // alternative: Array.flat() would be REALLY NICE here, but Edge does NOT support it (as of feb 6, 2019)
    // - https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/flat
    const externals = [];
    while(externals.length < numDeps) {
        const nextDep = args.pop(); // from right/back to left/front 
        if (typeof nextDep === 'string')
            externals.unshift(nextDep); // add to front/left of array
        else if (Array.isArray(nextDep)) {
            while (externals.length < numDeps && nextDep.length > 0) { // process nested deps
                const nd = nextDep.pop(); // take last one (so going from back to front)...
                if (typeof nd === 'string')
                    externals.unshift(nd); // add to front of array
                else 
                    throw new ModuleLoadError(`invalid dependency in AMD module definition - can only be a string (got type=${typeof nd})`);
            }
        }
        else 
            throw new ModuleLoadError(`invalid dependency in AMD module definition - can only be a string or an array of strings`);
    }
*/


// this is NOT secure
// - will still eval in GLOBAL context
// - we do have access to a general proxy which can easily be circumvented

// CANNOT use ES6 export to export this module because this would INVALIDATE our use of the 'with'
// statement insode executeCode; we MUST use 'module.exports' construct
module.exports = { executeCode, extractRequireDependencies};

/*
    AMD: 
        - we are AMD-like to work with webpack/rollup and npm
        - but we don' follow ALL its possibilities
            - e.g. 
            define('name', [...deps...], fcn onDepsReady(...deps...){})
                // defines a module (name becomes a private name for re-inclusion in deps)

        - if define(fcn) only
            - no deps, no name
            - and fcn has single parm
            - parm is 'require'
                - need to scan for it and extract 'string-only' deps

        require([...deps...], fcn onDepsReady(...deps...){}))
            // runs a module
            // if last parm is NOT function, it's a CJS require
                - allow anyway? --NO--
                    - would require pre-processing
                        - do a pre-download
                        - or convert to await
                            - top-level await is ok
                            - nested await will fail
                    - --BECAUSE-- if allow, custom code by/for us (no longer AMD)
                        - so maybe just change name? loadModule()?
                        - and if not a string use await?
                        - or always use await to keep simple
                        - allow multiple-at-once: 
                            const [mod1,mod2,mod3] = loadModule('mod1', 'mod2','mod3');

    CJS:
        require('name')
        require('ex' + 'pression).something
        require('dynamic' + 'ex' + 'pression)(execute-code)


*/

// 2 methods of executing code: EVAL and FUNCTION

// METHOD 1: EVAL
//export [see note above]
async function executeCode(srcCode, ...envs) {//privateEnv = {}, globalEnv = window) {

    const globalEnv = envs.pop();
    const privateEnv = envs.pop();

    // env === environment === context

    // important: we MUST NOT "use strict" anywhere in this file, else will prevent/break 'with' construct below
    // - for this code to work, we also CANNOT HAVE ANY 'import' or exports statements in this file
    //   else it will trigger strict-mode: strict mode prevents using 'with' below (would generate a SyntaxError)

    // IMPLEMENTATION is NOT secure:
    // - will still eval in GLOBAL context (albeit thinly protected because of our proxy)
    // - proxy can be easily jailbroken (i.e. circumvented) with '(0,eval)('this')'
    //   - unless we take extraordinary measures (take over root eval, Function/AsyncFunction 
    //     and Object.document.createElement(<script>))
    
    const usc = {}; // our Underlying Store/Cache, proxied as virtualPrivateEnv below
    const {$addNewVarsTo} = privateEnv, // settings for new var storage: 'private' or 'global' (or function returning either based on propName)
          newVarScope = varname => typeof $addNewVarsTo === 'function' ? $addNewVarsTo(propname) : ($addNewVarsTo || 'private');

    // will become the only "global" reference when eval used below
    // because will ALWAYS return true for ALL names (from its .has() method)
    const virtualPrivateEnv = new Proxy(usc, {

        // always always true: allows TRAPPING all names in srcCode from "outside world"
        has(target, name) { return true; }, 

        get(target, propName, receiver) {
            if (propName === Symbol.unscopables) 
                return {}; // important when using 'WITH' (as we are below)

            if (/^window$/.test(propName)) 
                return virtualPrivateEnv; // redirect back to us

            // give our private context priority
            if (propName in privateEnv) 
                return Reflect.get(privateEnv, propName);

            // fall back to global otherwise
            if (propName in globalEnv) 
                return Reflect.get(globalEnv, propName);

            throw new ReferenceError(`${propName} not defined`);
        },

        set(target, propName, value, receiver) {

            // CAN 'return false;' to signal that assignement failed 
            // BUT, in NON-STRICT mode (which this must be for us to be able to use 'with'),
            // failed assignments are SILENTLY IGNORED by browsers (as per standard implementations)
            // and so would not (cannot) be trapped by user...

            if (propName in privateEnv) 
                return Reflect.set(privateEnv, propName, value);

            if (propName in globalEnv) 
                return Reflect.set(globalEnv, propName, value);

            // propName (variable name) unknown: can add to extras, add to windows, or just fail
            // but for fail, we'd never know if it's var-declared or not (which could mean a typo)
            // AND, we've already said that this var exists (in .has() above) so...
            const scope = newVarScope(propName);
            if (/^(global|window|root|globalThis|main)$/i.test(scope))
                globalEnv[propName] = value;
            else if (/^private$/i.test(scope))
                privateEnv[propName] = value;
            else {
                // assignment is ignored
                //log.warning(`assignement to ${propName} ignored because of unknown scope (${scope})`);
                return false; // does this mean anything???
            }

            return true;
        },

        apply(target, propName, receiver) {
            // is this ever called?
            log('trying to apply something', propName);
            throw new Error(`unexpected/unimplemented proxied private env call to APPLY`); // see if this comes up anywhere
        },
    });

    // IN the template we use to wrap our srcCode:
    // - param 'window' passed will be our locally declared proxy and becomes the only non-proxied
    //   reference to outside world (i.e. will trap ALL var refs because its .has() always returnd true)
    const virtualEval = eval(`(async function(window){
        with(window) {
            ${srcCode}
        }
    })`);

    return await virtualEval(virtualPrivateEnv);
}

const myPrivateContext = {
    // define and require will be added to this object
    // unless already there?
    stashYourBitsPlugin(...args) { 
        return myPrivateContext.define(...args); // so basically an alias
    },

    // could add API here...
}

const modx = {
    id,
    srcCode, // or fcn
    baseUrl,
    scopes: [
        // 0 = lowest
        { name, props }
    ]
}

async function loadX(id, srcCode, privateCtx = {}, globalCtx = window) {
    try {
        return await asAMDCode(srcCode, privateCtx, globalCtx);
    }
    catch(err) {
        // but if 'module' missing? 
        if (err instanceof ModuleLoadError) { // try it as CJS
            try {
                // delete .define & .require first?
                return await asCJSCode(srcCode, privateCtx, globalCtx); 
            }
            catch(err) {
                return err;
            }
        }
        else {
            return err; // could be DownloadError or SyntaxError or anything else; either way, no way for us to recover
        }
    }
}

// really as AMD or NOT-CJS code: could just be basic code: returned result is module
async function asAMDCode(srcCode, privateCtx = {}, globalCtx = window, extractRequiresFirst = false) {
    return new Promise(async (resolve,reject) => {
        if (extractRequiresFirst) {
            // first, extract dependencies from source code
            const [deps, modCode, req, expt, modx] = extractRequireDependencies(srcCode);
            await loadModulesX(...deps); // loads them then makes them available to all
        }

        var usingDefineMethod = false; // result of define function (will be async since may need to load dependent modules first)

        const exportsUsed = () => Object.keys(exportedItems).length > 0;
        const exportedItems = {},
            moduleX = { exports: exportedItems };

        define.amd = {}; // ...use an object (truthy) NOT just 'true'
        async function define(...args) {

            // defineResult = new Promise((resolve,reject) => {

            // });

            // defineResult: either what's returned OR exports if using (require,exports,module) format
            usingDefineMethod = true;

            const moduleDefine = args.pop(); // always last param
            if (typeof moduleDefine !== 'function') 
                throw new ModuleLoadError(`expecting module definition to be a function (was ${typeof moduleDefine})`);

            const externals = []; // will become our execution parms

            if (args.length > 0) { // remaining args are deps to be pre-loaded

                // explicit dependencies
                // check for dependencies, then execute define [our current code]
                // SIMPLE STRATEGY 1: implement as AMD expects (a single array of dependencies)
                
                const depsArray = args.pop() || []; // expect an array or nothing
                if (Array.isArray(depsArray))
                    externals.push(...(await loadModulesX(...depsArray)));
                else
                    throw new ModuleLoadError(`expecting '[dependencies]' to be an array (was ${typeof externals})`);

                // BUT, if one of these is 'require' THEN need to extract anyway

                
            }

            // now look at function itself: is it based on its deps or using (require,exports,module)
            if (args.length === 0) {
                if (moduleDefine.length === 0) {
                    // all good
                }
                else {
                    // likely (require,exports,module): worth checking? would do what if not???
                    // only time would be an issue is if used other var names for some other purpose
                    const [deps, modCode, req, expt, modx] = extractRequireDependencies(srcCode);
                    await loadModulesX(...deps);
                    externals.push(getModuleByName, exportedItems, moduleX); // the (require,exports,module) parms when calling
                }
            }
        
            try {
                const result = await moduleDefine(...externals); // need to pass either (...deps) or (require,exports,module)
                resolve(exportsUsed ? exportedItems : result);
            }
            catch(err) {
                // just return?
                reject(err);
            }
        }

        function require(...args) {

            // could know if executing within 'define': does this matter?

            const req = args.pop(); // last or only parm
            if (args.length === 1 && typeof req === 'string') {
                // classic require: if loaded, return it
                // else: fail (later step can load req then try again)
                return getModuleByName(req); // may throw error if module not already loaded
            }
            else if (typeof req === 'function') {
                // treat like a define?
                //return 
                define(...args); // await???
            }
            else {
                // unexpected: treat as error?
                throw new Error(`unexpect parms for require`);
            }
        }

        try {
            // important: sets this context for AMD modules
            //Object.assign(privateCtx, { define, require, get module() { throw new ModuleLoadError(); } });
            const amdMethods = { define, require, get module() { throw new ModuleLoadError(); } };

            const result = await executeCode(srcCode, amdMethods, privateCtx, globalEnv); // have additional context in between...
            // no errors
            usingDefineMethod || resolve(result); // else, expects it to be resolved/rejected from define function
        }
        catch(err) {
            if (err instanceof ModuleNotLoaded) {
                // try to extract requires first, then try again?
                // OR try immediately as CJS or try again after extract/load it and others?
                if (extractRequiresFirst) {
                    // already tried, so fail now
                }
                else {
                    return await asAMDCode(srcCode, privateCtx, globalCtx, true); // privateCtx now has define/require: delete them 1st?
                }
            }
        }

    })
}


// await onModule('name') // promise
// onModule('name', onload(){}); // also promise but executed AFTER onload completes; OR returns result ofonload?


/* 
    attempt #1: pass define & require (no module, exports)
        - execute code as is (hope for the best)
        - within define:
            - look for require as dep 
            OR 
            - (require,exports, module)  as parms for fcn
                IF no other DEPS
                - if so, scan source code for requires
            - if not,
                - execute code as is (no code handling)
        - within require:
            - if [deps] and fcn
                - process as if define above
                - any result? ignored?
            - if 'string-only'
                - if module loaded, return it (all is well)
                - if no such module:
                    - likely meant to be CJS: 
                        - fail here and wait for attempt#2
                        - or go back, scan code for requires, load deps
                            - then try again
            - else error
        - if define/require NOT called
            - if no error, result is module (or module is now undefined)
            - if error
                - fail and wait for attempt #2

    attempt #2: pass module, module.exports, require
        - first scan for 'requires'
            - load those deps
        - then execute code
            - if require called with unknown string:
                - would have needed 'await require'
        - if success,
            result is module.exports
        - if error
            module load error


*/