Use Effection for Ursa async (WIP)

Also use Effection in some of Ark: evalArkFs becomes a generator. Make launch an Exp, not a Statement, as we need to be able to await it. ArkPromise becomes ArkOperation. Add NativeOperation, and bind Effection's sleep() using it. Use Effection in the JavaScript back-end too, using the "obvious" translation (basically, the Effection "Rosetta Stone")). Standalone scripts now require Effection. Change 'launch.ursa' test to await one launched expression but not the other, to test that the un-awaited one is terminated early.
ursalang · Jul 29, 2024 · d7f53ae · d7f53ae
1 parent 5fb6486
commit d7f53ae
Show file tree

Hide file tree

Showing 14 changed files with 99 additions and 70 deletions.
diff --git a/package.json b/package.json
@@ -12,6 +12,7 @@
     "@prettier/sync": "^0.5.2",
     "@sc3d/stacktracey": "^2.1.9",
     "argparse": "^2.0.1",
+    "effection": "^3.0.3",
     "env-paths": "^3.0.0",
     "fs-extra": "^11.2.0",
     "get-source": "^2.0.12",

diff --git a/src/ark/compiler/index.ts b/src/ark/compiler/index.ts
@@ -7,6 +7,8 @@ import fs from 'fs'
 import path from 'path'
 import {fileURLToPath} from 'url'
 import util from 'util'
+
+import {call} from 'effection'
 import getSource from 'get-source'
 import {
   CodeWithSourceMap, Position, SourceMapConsumer,
@@ -31,7 +33,7 @@ import {
   globals, jsGlobals,
   ArkBoolean, ArkBooleanVal, ArkList, ArkMap, ArkNull,
   ArkNumber, ArkNullVal, ArkNumberVal, ArkObject, ArkString,
-  ArkStringVal, ArkUndefined, ArkVal, NativeFn, ArkPromise,
+  ArkStringVal, ArkUndefined, ArkVal, NativeFn, ArkOperation,
 } from '../data.js'
 import {ArkExp} from '../code.js'
 import {debug} from '../util.js'
@@ -54,7 +56,7 @@ class UrsaStackTracey extends StackTracey {
 
 // Compile prelude and add it to globals
 export const preludeJs = fs.readFileSync(path.join(__dirname, 'prelude.js'), {encoding: 'utf-8'})
-const prelude = await evalArkJs(preludeJs) as ArkObject
+const prelude = evalArkJs(preludeJs).next().value as ArkObject
 prelude.properties.forEach((val, sym) => jsGlobals.set(sym, val))
 
 // runtimeContext records the values that are needed by JavaScript at
@@ -70,7 +72,7 @@ const runtimeContext: Record<string, unknown> = {
   ArkObject,
   ArkList,
   ArkMap,
-  ArkPromise,
+  ArkOperation,
   NativeFn,
   jsGlobals,
 }
@@ -121,13 +123,13 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
         return sourceNode(letAssign(inst.id, valToJs(inst.val)))
       } else if (inst instanceof ArkLaunchBlockCloseInst) {
         return sourceNode([
-          `return ${inst.id.description!}\n`,
-          '})())\n',
+          `yield* ${inst.blockId.description!}\n`,
+          '})\n',
         ])
       } else if (inst instanceof ArkFnBlockCloseInst) {
         env = env.popFrame()
         if (inst.matchingOpen instanceof ArkGeneratorBlockOpenInst) {
-          return sourceNode(['}()\nreturn new NativeFn([\'x\'], async (x) => {\nconst {value, done} = await gen.next(x)\nreturn done ? ArkNull() : value\n})\n})\n'])
+          return sourceNode(['}()\nreturn new NativeFn([\'x\'], (x) => {\nconst {value, done} = gen.next(x)\nreturn done ? ArkNull() : value\n})\n})\n'])
         }
         return sourceNode(['})\n'])
       } else if (inst instanceof ArkIfBlockOpenInst) {
@@ -142,20 +144,20 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
       } else if (inst instanceof ArkLoopBlockOpenInst) {
         return sourceNode([letAssign(inst.matchingClose.id, 'ArkNull()'), 'for (;;) {\n'])
       } else if (inst instanceof ArkLaunchBlockOpenInst) {
-        return sourceNode([letAssign(inst.matchingClose.id, 'new ArkPromise((async function() {')])
+        return sourceNode([letAssign(inst.matchingClose.id, 'yield* spawn(function* () {')])
       } else if (inst instanceof ArkGeneratorBlockOpenInst) {
         env = env.pushFrame(
           new Frame(inst.params.map((p) => new Location(p, false)), [], inst.name),
         )
         return sourceNode([
-          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], function (${inst.params.join(', ')}) {\nconst gen = async function*() {`),
+          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], function (${inst.params.join(', ')}) {\nconst gen = function* () {`),
         ])
       } else if (inst instanceof ArkFnBlockOpenInst) {
         env = env.pushFrame(
           new Frame(inst.params.map((p) => new Location(p, false)), [], inst.name),
         )
         return sourceNode([
-          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], async function(${inst.params.join(', ')}) {`),
+          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], function* (${inst.params.join(', ')}) {`),
         ])
       } else if (inst instanceof ArkLetBlockOpenInst) {
         return sourceNode([
@@ -166,7 +168,7 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
       } else if (inst instanceof ArkBlockOpenInst) {
         return sourceNode([`let ${inst.matchingClose.id.description!}\n`, '{\n'])
       } else if (inst instanceof ArkAwaitInst) {
-        return sourceNode(letAssign(inst.id, `await ${inst.argId.description}.promise`))
+        return sourceNode(letAssign(inst.id, `yield* ${inst.argId.description}`))
       } else if (inst instanceof ArkBreakInst) {
         return sourceNode([`${assign(inst.argId.description!, inst.loopInst.matchingClose.id.description!)}\n`, 'break\n'])
       } else if (inst instanceof ArkContinueInst) {
@@ -178,7 +180,7 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
       } else if (inst instanceof ArkLetCopyInst) {
         return sourceNode(letAssign(inst.id, inst.argId.description!))
       } else if (inst instanceof ArkCallInst) {
-        return sourceNode(letAssign(inst.id, `await ${inst.fnId.description}.body(${inst.argIds.map((id) => id.description).join(', ')})`))
+        return sourceNode(letAssign(inst.id, `${inst.fnId.description}.body(${inst.argIds.map((id) => id.description).join(', ')})`))
       } else if (inst instanceof ArkSetInst) {
         return sourceNode([
           `if (${inst.lexpId.description} !== ArkUndefined && ${inst.lexpId.description}.constructor !== ArkNullVal && ${inst.valId.description}.constructor !== ${inst.lexpId.description}.constructor) {\n`,
@@ -222,11 +224,10 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
   }
 
   const sourceNode = new SourceNode(1, 1, 'src/ursa/flat-to-js.ts', [
-    // FIXME: work out how to eval ESM, so we can use top-level await.
     '"use strict";\n',
-    '(async () => {\n',
+    '(function* () {\n',
     instsToJs(insts),
-    `return ${insts.id.description}\n})()`,
+    `return ${insts.id.description}\n})().next().value`,
   ])
   const jsCode = sourceNode.toStringWithSourceMap({file: file ?? undefined})
   if (process.env.DEBUG) {
@@ -240,7 +241,7 @@ export function arkToJs(exp: ArkExp, file: string | null = null): CodeWithSource
   return flatToJs(insts, file)
 }
 
-export async function evalArkJs(source: CodeWithSourceMap | string, file = '(Compiled Ark)'): Promise<ArkVal> {
+export function* evalArkJs(source: CodeWithSourceMap | string, file = '(Compiled Ark)') {
   let jsSource: string
   if (typeof source === 'string') {
     jsSource = source
@@ -253,7 +254,7 @@ export async function evalArkJs(source: CodeWithSourceMap | string, file = '(Com
   }
   try {
     // eslint-disable-next-line no-eval
-    return await (eval(jsSource) as Promise<ArkVal>)
+    return eval(jsSource) as ArkObject
   } catch (e) {
     assert(e instanceof Error)
     const dirtyStack = new UrsaStackTracey(e).withSources()
@@ -293,7 +294,7 @@ export async function evalArkJs(source: CodeWithSourceMap | string, file = '(Com
       newError.message = `${prefix}\n${message}`
       let consumer
       if (typeof source !== 'string') {
-        consumer = await new SourceMapConsumer(source.map.toJSON())
+        consumer = yield* call(new SourceMapConsumer(source.map.toJSON()))
       }
       for (const [i, frame] of stack.items.slice(1).entries()) {
         let fnLocation

diff --git a/src/ark/data.ts b/src/ark/data.ts
@@ -4,6 +4,10 @@
 
 import assert from 'assert'
 
+import {
+  action, Operation, Reject, Resolve, sleep,
+} from 'effection'
+
 import {FsMap} from './fsmap.js'
 import programVersion from '../version.js'
 import {debug} from './util.js'
@@ -147,8 +151,8 @@ export function ArkString(s: string) {
   return ConcreteInterned.value<ArkStringVal, string>(ArkStringVal, s)
 }
 
-export class ArkPromise extends ArkVal {
-  constructor(public promise: Promise<ArkVal>) {
+export class ArkOperation extends ArkVal {
+  constructor(public operation: Operation<ArkVal>) {
     super()
   }
 }
@@ -175,6 +179,12 @@ export class NativeFn extends ArkCallable {
   }
 }
 
+export class NativeOperation extends ArkCallable {
+  constructor(params: string[], public body: (...args: ArkVal[]) => Operation<ArkVal>) {
+    super(params)
+  }
+}
+
 // ts-unused-exports:disable-next-line
 export class NativeAsyncFn extends ArkCallable {
   constructor(params: string[], public body: (...args: ArkVal[]) => Promise<ArkVal>) {
@@ -366,17 +376,21 @@ export const globals = new ArkObject(new Map<string, ArkVal>([
     debug(obj)
     return ArkNull()
   })],
-  ['fs', new NativeFn(['path'], (path: ArkVal) => new NativeObject(new FsMap(toJs(path) as string)))],
-
-  ['Promise', new NativeAsyncFn(
+  ['fs', new NativeFn(['path'], (path: ArkVal) => new NativeObject(new FsMap((path as ArkStringVal).val)))],
+  ['sleep', new NativeOperation(['ms'], function* gen(ms: ArkVal) {
+    yield* sleep((ms as ArkNumberVal).val)
+    return ArkNull()
+  })],
+  ['action', new NativeFn(
     ['resolve', 'reject'],
-    (fn: ArkVal) => Promise.resolve(new ArkPromise(
-      new Promise(
-        toJs(fn) as
-        (resolve: (value: unknown) => void, reject: (reason?: unknown) => void) => void,
-      ).then((x) => fromJs(x)),
-    )),
+    function* gen(fn: ArkVal) {
+      const result = yield* action(
+        toJs(fn) as (resolve: Resolve<unknown>, reject: Reject) => Operation<void>,
+      )
+      return fromJs(result)
+    },
   )],
+  // FIXME: should be able to use js.fetch directly.
   ['fetch', new NativeAsyncFn(
     ['url', 'options'],
     async (url: ArkVal, options: ArkVal) => new NativeObject(

diff --git a/src/ark/interpreter.ts b/src/ark/interpreter.ts
@@ -2,6 +2,9 @@
 // © Reuben Thomas 2023-2024
 // Released under the MIT license.
 
+import {
+  call, Operation, run, spawn,
+} from 'effection'
 import {Interval} from 'ohm-js'
 
 import {
@@ -15,8 +18,8 @@ import {
 } from './flatten.js'
 import {
   ArkAbstractObjectBase, ArkBoolean, ArkList, ArkMap, ArkNull, ArkNullVal,
-  ArkObject, ArkPromise, ArkUndefined, ArkVal, NativeAsyncFn, NativeFn,
-  ArkRef, ArkValRef,
+  ArkObject, ArkOperation, ArkUndefined, ArkVal, NativeAsyncFn, NativeFn,
+  NativeOperation, ArkRef, ArkValRef,
 } from './data.js'
 import {
   ArkCapture, ArkContinuation, ArkFlatClosure, ArkFlatGeneratorClosure,
@@ -101,6 +104,10 @@ function makeLocals(names: string[], vals: ArkVal[]): ArkRef[] {
 }
 
 async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
+  return run(() => doEvalFlat(outerArk))
+}
+
+function* doEvalFlat(outerArk: ArkState): Operation<ArkVal> {
   let ark: ArkState | undefined = outerArk
   let inst = ark.inst
   let prevInst
@@ -157,9 +164,10 @@ async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
         ),
         ark.outerState,
       )
-      const result = Promise.resolve(new ArkPromise(evalFlat(innerArk)))
+      const operation = yield* spawn(() => doEvalFlat(innerArk))
+      const result = new ArkOperation(operation)
       mem.set(inst.id, result)
-      // The Promise becomes the result of the entire block.
+      // The ArkOperation becomes the result of the entire block.
       mem.set(inst.matchingClose.id, result)
       inst = inst.matchingClose.next
     } else if (inst instanceof ArkCallableBlockOpenInst) {
@@ -184,8 +192,9 @@ async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
     } else if (inst instanceof ArkBlockOpenInst) {
       inst = inst.next
     } else if (inst instanceof ArkAwaitInst) {
-      const promise = (mem.get(inst.argId)! as ArkPromise).promise
-      mem.set(inst.id, await promise)
+      const operation = (mem.get(inst.argId)! as ArkOperation).operation
+      const result = yield* operation
+      mem.set(inst.id, result)
       inst = inst.next
     } else if (inst instanceof ArkBreakInst) {
       const result = mem.get(inst.argId)!
@@ -277,7 +286,10 @@ async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
         mem.set(inst.id, callable.body(...args))
         inst = inst.next
       } else if (callable instanceof NativeAsyncFn) {
-        mem.set(inst.id, await callable.body(...args))
+        mem.set(inst.id, yield* call(callable.body(...args)))
+        inst = inst.next
+      } else if (callable instanceof NativeOperation) {
+        mem.set(inst.id, yield* callable.body(...args))
         inst = inst.next
       } else {
         throw new ArkRuntimeError(ark, 'Invalid call', inst.sourceLoc)

diff --git a/src/ark/serialize.ts b/src/ark/serialize.ts
@@ -4,7 +4,7 @@
 
 import {
   globals, ArkVal,
-  ArkConcreteVal, ArkNull, ArkPromise, ArkList, ArkMap, ArkObject,
+  ArkConcreteVal, ArkNull, ArkOperation, ArkList, ArkMap, ArkObject,
   ArkUndefined, NativeObject,
 } from './data.js'
 import {
@@ -87,7 +87,7 @@ export function valToJs(val: ArkVal | ArkExp, externalSyms = globals) {
       return ['yield', doValToJs(val.exp)]
     } else if (val instanceof ArkReturn) {
       return ['return', doValToJs(val.exp)]
-    } else if (val instanceof ArkPromise) {
+    } else if (val instanceof ArkOperation) {
       // FIXME: Can we properly serialize a promise?
       return ['promise']
     } else if (val === ArkNull()) {

diff --git a/src/grammar/ursa.ohm b/src/grammar/ursa.ohm
@@ -108,6 +108,7 @@ Ursa {
     | For
     | await Exp       -- await
     | yield Exp?      -- yield
+    | launch Exp      -- launch
     | LogicExp
 
   Assignment
@@ -119,7 +120,6 @@ Ursa {
     | Use
     | break Exp?      -- break
     | continue        -- continue
-    | launch Exp      -- launch
     | return Exp?     -- return
     | Exp
 

diff --git a/src/grammar/ursa.ohm-bundle.d.ts.diff b/src/grammar/ursa.ohm-bundle.d.ts.diff
@@ -1,5 +1,5 @@
---- src/grammar/ursa.ohm-bundle.d.ts	2024-07-25 19:26:08.992907449 +0100
-+++ src/grammar/ursa.ohm-bundle.d.part-patched.ts	2024-07-25 19:26:09.128908027 +0100
+--- src/grammar/ursa.ohm-bundle.d.ts	2024-07-27 22:31:25.130617483 +0100
++++ src/grammar/ursa.ohm-bundle.d.part-patched.ts	2024-07-27 22:31:25.270617985 +0100
 @@ -4,14 +4,33 @@
  import {
    BaseActionDict,

diff --git a/src/testutil.ts b/src/testutil.ts
@@ -52,7 +52,7 @@ function doTestGroup(
       const flat = expToInsts(compiled)
       const jsSource = flatToJs(flat)
       const resArk = await new ArkState(flat.insts[0]).run()
-      const resJs = await evalArkJs(jsSource, title)
+      const resJs = evalArkJs(jsSource, title)
       if (resArk instanceof ArkObject) {
         assert(typeof expected === 'object')
         // Remove methods of ArkObject

diff --git a/src/ursa/cli.ts b/src/ursa/cli.ts
@@ -251,7 +251,7 @@ async function runCode(source: string, args: Args) {
         result = await ark.run()
       }
     } else {
-      result = await evalArkJs(arkToJs(exp, prog), prog)
+      result = evalArkJs(arkToJs(exp, prog), prog)
     }
   }
   if (source === undefined || args.interactive) {
@@ -305,18 +305,20 @@ async function compileCommand(args: Args) {
       output += `#!/usr/bin/env -S ${process.argv0} --no-warnings ${process.argv[1]} --syntax=json run\n`
     }
     output += serializeVal(exp)
-  } else {
-    if (args.executable) {
-      output += '#!/usr/bin/env -S node --experimental-default-type=module --\n'
-      output += await buildRuntime()
-      // Read prelude but elide the "use strict" line.
-      const prelude = preludeJs.slice(preludeJs.indexOf('\n') + 1)
-      output += `let prelude = await${prelude}
+  } else if (args.executable) {
+    output += '#!/usr/bin/env -S node --experimental-default-type=module --\n'
+    output += await buildRuntime()
+    // Read prelude but elide the "use strict" line.
+    const prelude = preludeJs.slice(preludeJs.indexOf('\n') + 1)
+    output += `import {spawn, main} from 'effection'\nlet prelude = ${prelude}
 prelude.properties.forEach((val, sym) => jsGlobals.set(sym, val))\n`
-      output += `jsGlobals.set('argv', new ArkList(
+    output += `jsGlobals.set('argv', new ArkList(
   [ArkString(process.argv[1]), ...process.argv.slice(2).map((s) => ArkString(s))],
-))\n`
-    }
+));\n`
+    const code = arkToJs(exp, prog).code
+    const codeTail = code.slice(code.indexOf('\n') + 1, -('().next().value'.length + 1))
+    output += `await main${codeTail})\n`
+  } else {
     output += arkToJs(exp, prog).code
   }
   fs.writeFileSync(outputFile, output)

diff --git a/src/ursa/compiler.ts b/src/ursa/compiler.ts
@@ -549,6 +549,10 @@ semantics.addOperation<ArkExp>('toExp(a)', {
     return addLoc(new ArkYield(maybeVal(this.args.a, exp)), this)
   },
 
+  Exp_launch(_launch, exp) {
+    return addLoc(new ArkLaunch(exp.toExp(this.args.a)), this)
+  },
+
   Statement_break(_break, exp) {
     if (!this.args.a.inLoop) {
       throw new UrsaCompilerError(_break.source, 'break used outside a loop')
@@ -561,9 +565,6 @@ semantics.addOperation<ArkExp>('toExp(a)', {
     }
     return addLoc(new ArkContinue(), this)
   },
-  Statement_launch(_await, exp) {
-    return addLoc(new ArkLaunch(exp.toExp(this.args.a)), this)
-  },
   Statement_return(return_, exp) {
     if (!this.args.a.inFn) {
       throw new UrsaCompilerError(return_.source, 'return used outside a function')