Use Effection for Ursa async

Also use Effection in some of Ark: evalArkFs becomes a generator. Make launch an Exp, not a Statement, as we need to be able to await it. ArkPromise becomes ArkOperation. Add NativeOperation, and bind Effection's sleep() using it. Use Effection in the JavaScript back-end too, using the "obvious" translation (basically, the Effection "Rosetta Stone")). Standalone scripts now require Effection. Change 'launch.ursa' test to await one launched expression but not the other, to test that the un-awaited one is terminated early. It remains to make structured concurrency scopes explicit. At present there is only on, top-level scope.
ursalang · Aug 4, 2024 · 80fadd5 · 80fadd5
1 parent 5fb6486
commit 80fadd5
Show file tree

Hide file tree

Showing 15 changed files with 158 additions and 93 deletions.
diff --git a/TODO.md b/TODO.md
@@ -4,7 +4,7 @@
 
 * `Map` interface to internet.
 * Classes: properties, names for List and Map
-* Structured concurrency.
+* Structured concurrency: allow the declaration of new scopes, and to run a task in a new scope.
 * A generator in a comprehension gives a lazy list/map.
 
 ## Ark improvements

diff --git a/package.json b/package.json
@@ -10,15 +10,18 @@
   },
   "dependencies": {
     "@prettier/sync": "^0.5.2",
+    "@rollup/plugin-node-resolve": "^15.2.3",
     "@sc3d/stacktracey": "^2.1.9",
     "argparse": "^2.0.1",
+    "effection": "^3.0.3",
     "env-paths": "^3.0.0",
     "fs-extra": "^11.2.0",
     "get-source": "^2.0.12",
     "ohm-js": "^17.1.0",
     "rollup": "^4.19.0",
     "source-map": "^0.7.4",
     "tildify": "^3.0.0",
+    "tmp": "^0.2.3",
     "tslib": "^2.6.3"
   },
   "devDependencies": {
@@ -39,7 +42,6 @@
     "eslint-config-airbnb-typescript": "^18.0.0",
     "execa": "^9.3.0",
     "pre-push": "^0.1.4",
-    "tmp": "^0.2.3",
     "tree-kill": "^1.2.2",
     "ts-node": "^10.9.2",
     "ts-unused-exports": "^10.1.0",
@@ -64,7 +66,7 @@
     "patch-diff-ohm": "patch -p0 --output=src/grammar/ursa.ohm-bundle.d.part-patched.ts < src/grammar/ursa.ohm-bundle.d.ts.diff",
     "patch-ohm": "npm run patch-diff-ohm && sed -e 's/this: NonterminalNode/this: ThisNode/' < src/grammar/ursa.ohm-bundle.d.part-patched.ts > src/grammar/ursa.ohm-bundle.d.ts",
     "generate": "npm run generate-only && npm run patch-ohm && mkdir -p lib/grammar/ && cp src/grammar/*ohm-bundle.js src/grammar/*ohm-bundle.d.ts lib/grammar/",
-    "pre-compile-prelude": "echo null > src/ark/prelude.json && echo 'new ArkObject(new Map())' > src/ark/compiler/prelude.js",
+    "pre-compile-prelude": "echo null > src/ark/prelude.json && echo '(function* gen() { return new ArkObject(new Map()) })' > src/ark/compiler/prelude.js",
     "update-ohm-patch": "npm run generate-only && npm run patch-diff-ohm && diff -u src/grammar/ursa.ohm-bundle.d.ts src/grammar/ursa.ohm-bundle.d.part-patched.ts > src/grammar/ursa.ohm-bundle.d.ts.diff; npm run patch-ohm",
     "compile-prelude": "npm run pre-compile-prelude && ./bin/test-run.sh compile --output=src/ark/prelude.json src/ursa/prelude.ursa && ./bin/test-run.sh --target=js compile --output=src/ark/compiler/prelude.js src/ursa/prelude.ursa",
     "test": "npm run generate && npm run build && ava 'src/**/basics.test.ts' 'src/**/examples.test.ts' 'src/**/cli.test.ts' 'src/**/fsmap.test.ts'",

diff --git a/src/ark/compiler/index.ts b/src/ark/compiler/index.ts
@@ -7,6 +7,8 @@ import fs from 'fs'
 import path from 'path'
 import {fileURLToPath} from 'url'
 import util from 'util'
+
+import {Operation, run, spawn} from 'effection'
 import getSource from 'get-source'
 import {
   CodeWithSourceMap, Position, SourceMapConsumer,
@@ -31,7 +33,7 @@ import {
   globals, jsGlobals,
   ArkBoolean, ArkBooleanVal, ArkList, ArkMap, ArkNull,
   ArkNumber, ArkNullVal, ArkNumberVal, ArkObject, ArkString,
-  ArkStringVal, ArkUndefined, ArkVal, NativeFn, ArkPromise,
+  ArkStringVal, ArkUndefined, ArkVal, NativeFn, ArkOperation,
 } from '../data.js'
 import {ArkExp} from '../code.js'
 import {debug} from '../util.js'
@@ -44,11 +46,15 @@ class JsRuntimeError extends Error {}
 
 class UrsaStackTracey extends StackTracey {
   isThirdParty(path: string) {
-    return super.isThirdParty(path) || path.includes('ark/') || path.includes('ursa/') || path.includes('node:')
+    return super.isThirdParty(path)
+      || path.includes('ark/') || path.includes('ursa/')
+      || path.includes('effection/') || path.includes('deno.land/x/continuation@')
+      || path.includes('node:')
   }
 
   isClean(entry: Entry, index: number) {
-    return super.isClean(entry, index) && !entry.file.includes('node:')
+    return super.isClean(entry, index)
+      && !entry.file.includes('node:') && !(entry.callee === 'Generator.next')
   }
 }
 
@@ -57,24 +63,30 @@ export const preludeJs = fs.readFileSync(path.join(__dirname, 'prelude.js'), {en
 const prelude = await evalArkJs(preludeJs) as ArkObject
 prelude.properties.forEach((val, sym) => jsGlobals.set(sym, val))
 
-// runtimeContext records the values that are needed by JavaScript at
-// runtime, and prevents the TypeScript compiler throwing away their
-// imports.
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-const runtimeContext: Record<string, unknown> = {
+// Record internal values that are needed by JavaScript at runtime, and
+// prevent the TypeScript compiler throwing away their imports.
+export const runtimeContext: Record<string, unknown> = {
   ArkUndefined,
   ArkNull,
+  ArkNullVal,
   ArkBoolean,
   ArkNumber,
   ArkString,
   ArkObject,
   ArkList,
   ArkMap,
-  ArkPromise,
+  ArkOperation,
   NativeFn,
   jsGlobals,
 }
 
+// Record internal values that are needed by JavaScript at runtime, and
+// prevent the TypeScript compiler throwing away their imports.
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+const externalRuntimeContext: Record<string, unknown> = {
+  spawn,
+}
+
 function valToJs(val: ArkVal): string {
   if (val instanceof ArkNullVal) {
     return 'ArkNull()'
@@ -121,13 +133,13 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
         return sourceNode(letAssign(inst.id, valToJs(inst.val)))
       } else if (inst instanceof ArkLaunchBlockCloseInst) {
         return sourceNode([
-          `return ${inst.id.description!}\n`,
-          '})())\n',
+          `return ${inst.blockId.description!}\n`,
+          '})\n',
         ])
       } else if (inst instanceof ArkFnBlockCloseInst) {
         env = env.popFrame()
         if (inst.matchingOpen instanceof ArkGeneratorBlockOpenInst) {
-          return sourceNode(['}()\nreturn new NativeFn([\'x\'], async (x) => {\nconst {value, done} = await gen.next(x)\nreturn done ? ArkNull() : value\n})\n})\n'])
+          return sourceNode(['}()\nreturn new NativeFn([\'x\'], (x) => {\nconst {value, done} = gen.next(x)\nreturn done ? ArkNull() : value\n})\n})\n'])
         }
         return sourceNode(['})\n'])
       } else if (inst instanceof ArkIfBlockOpenInst) {
@@ -142,20 +154,20 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
       } else if (inst instanceof ArkLoopBlockOpenInst) {
         return sourceNode([letAssign(inst.matchingClose.id, 'ArkNull()'), 'for (;;) {\n'])
       } else if (inst instanceof ArkLaunchBlockOpenInst) {
-        return sourceNode([letAssign(inst.matchingClose.id, 'new ArkPromise((async function() {')])
+        return sourceNode([letAssign(inst.matchingClose.id, 'yield* spawn(function* () {')])
       } else if (inst instanceof ArkGeneratorBlockOpenInst) {
         env = env.pushFrame(
           new Frame(inst.params.map((p) => new Location(p, false)), [], inst.name),
         )
         return sourceNode([
-          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], function (${inst.params.join(', ')}) {\nconst gen = async function*() {`),
+          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], function (${inst.params.join(', ')}) {\nconst gen = function* () {`),
         ])
       } else if (inst instanceof ArkFnBlockOpenInst) {
         env = env.pushFrame(
           new Frame(inst.params.map((p) => new Location(p, false)), [], inst.name),
         )
         return sourceNode([
-          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], async function(${inst.params.join(', ')}) {`),
+          letAssign(inst.matchingClose.id, `new NativeFn([${inst.params.map((p) => `'${p}'`).join(', ')}], function* (${inst.params.join(', ')}) {`),
         ])
       } else if (inst instanceof ArkLetBlockOpenInst) {
         return sourceNode([
@@ -166,7 +178,7 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
       } else if (inst instanceof ArkBlockOpenInst) {
         return sourceNode([`let ${inst.matchingClose.id.description!}\n`, '{\n'])
       } else if (inst instanceof ArkAwaitInst) {
-        return sourceNode(letAssign(inst.id, `await ${inst.argId.description}.promise`))
+        return sourceNode(letAssign(inst.id, `yield* ${inst.argId.description}`))
       } else if (inst instanceof ArkBreakInst) {
         return sourceNode([`${assign(inst.argId.description!, inst.loopInst.matchingClose.id.description!)}\n`, 'break\n'])
       } else if (inst instanceof ArkContinueInst) {
@@ -178,7 +190,7 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
       } else if (inst instanceof ArkLetCopyInst) {
         return sourceNode(letAssign(inst.id, inst.argId.description!))
       } else if (inst instanceof ArkCallInst) {
-        return sourceNode(letAssign(inst.id, `await ${inst.fnId.description}.body(${inst.argIds.map((id) => id.description).join(', ')})`))
+        return sourceNode(letAssign(inst.id, `yield* ${inst.fnId.description}.body(${inst.argIds.map((id) => id.description).join(', ')})`))
       } else if (inst instanceof ArkSetInst) {
         return sourceNode([
           `if (${inst.lexpId.description} !== ArkUndefined && ${inst.lexpId.description}.constructor !== ArkNullVal && ${inst.valId.description}.constructor !== ${inst.lexpId.description}.constructor) {\n`,
@@ -222,11 +234,10 @@ export function flatToJs(insts: ArkInsts, file: string | null = null): CodeWithS
   }
 
   const sourceNode = new SourceNode(1, 1, 'src/ursa/flat-to-js.ts', [
-    // FIXME: work out how to eval ESM, so we can use top-level await.
     '"use strict";\n',
-    '(async () => {\n',
+    '(function* () {\n',
     instsToJs(insts),
-    `return ${insts.id.description}\n})()`,
+    `return ${insts.id.description}\n})`,
   ])
   const jsCode = sourceNode.toStringWithSourceMap({file: file ?? undefined})
   if (process.env.DEBUG) {
@@ -253,7 +264,8 @@ export async function evalArkJs(source: CodeWithSourceMap | string, file = '(Com
   }
   try {
     // eslint-disable-next-line no-eval
-    return await (eval(jsSource) as Promise<ArkVal>)
+    const gen = eval(jsSource) as () => Operation<ArkVal>
+    return await run<ArkVal>(gen)
   } catch (e) {
     assert(e instanceof Error)
     const dirtyStack = new UrsaStackTracey(e).withSources()
@@ -267,7 +279,7 @@ export async function evalArkJs(source: CodeWithSourceMap | string, file = '(Com
       const curFrame = stack.items[0]
       let prefix: string
       if (curFrame.line !== undefined) {
-        if (message.match('is not a function')) {
+        if (message.match('yield\\* \\(intermediate value\\)')) {
           const index = curFrame.column! - 1
           if (curFrame.sourceLine !== undefined && index < curFrame.sourceLine.length) {
             if (curFrame.sourceLine[index + 1] === '(') {

diff --git a/src/ark/data.ts b/src/ark/data.ts
@@ -3,6 +3,11 @@
 // Released under the MIT license.
 
 import assert from 'assert'
+import {isGeneratorFunction} from 'util/types'
+
+import {
+  action, call, Operation, Reject, Resolve, sleep,
+} from 'effection'
 
 import {FsMap} from './fsmap.js'
 import programVersion from '../version.js'
@@ -147,8 +152,8 @@ export function ArkString(s: string) {
   return ConcreteInterned.value<ArkStringVal, string>(ArkStringVal, s)
 }
 
-export class ArkPromise extends ArkVal {
-  constructor(public promise: Promise<ArkVal>) {
+export class ArkOperation extends ArkVal {
+  constructor(public operation: Operation<ArkVal>) {
     super()
   }
 }
@@ -170,15 +175,35 @@ export abstract class ArkClosure extends ArkCallable {
 export abstract class ArkGeneratorClosure extends ArkClosure {}
 
 export class NativeFn extends ArkCallable {
-  constructor(params: string[], public body: (...args: ArkVal[]) => ArkVal) {
+  public body: (...args: ArkVal[]) => Operation<ArkVal>
+
+  constructor(
+    params: string[],
+    innerBody: (...args: ArkVal[]) => ArkVal | Operation<ArkVal>,
+  ) {
+    super(params)
+    if (isGeneratorFunction(innerBody)) {
+      this.body = innerBody as (...args: ArkVal[]) => Operation<ArkVal>
+    } else {
+      // eslint-disable-next-line require-yield
+      this.body = function* gen(...args: ArkVal[]) { return innerBody(...args) }
+    }
+  }
+}
+
+export class NativeOperation extends ArkCallable {
+  constructor(params: string[], public body: (...args: ArkVal[]) => Operation<ArkVal>) {
     super(params)
   }
 }
 
 // ts-unused-exports:disable-next-line
 export class NativeAsyncFn extends ArkCallable {
-  constructor(params: string[], public body: (...args: ArkVal[]) => Promise<ArkVal>) {
+  public body: (...args: ArkVal[]) => Operation<ArkVal>
+
+  constructor(params: string[], innerBody: (...args: ArkVal[]) => Promise<ArkVal>) {
     super(params)
+    this.body = (...args: ArkVal[]) => call(() => innerBody(...args))
   }
 }
 
@@ -366,17 +391,21 @@ export const globals = new ArkObject(new Map<string, ArkVal>([
     debug(obj)
     return ArkNull()
   })],
-  ['fs', new NativeFn(['path'], (path: ArkVal) => new NativeObject(new FsMap(toJs(path) as string)))],
-
-  ['Promise', new NativeAsyncFn(
+  ['fs', new NativeFn(['path'], (path: ArkVal) => new NativeObject(new FsMap((path as ArkStringVal).val)))],
+  ['sleep', new NativeOperation(['ms'], function* gen(ms: ArkVal) {
+    yield* sleep((ms as ArkNumberVal).val)
+    return ArkNull()
+  })],
+  ['action', new NativeFn(
     ['resolve', 'reject'],
-    (fn: ArkVal) => Promise.resolve(new ArkPromise(
-      new Promise(
-        toJs(fn) as
-        (resolve: (value: unknown) => void, reject: (reason?: unknown) => void) => void,
-      ).then((x) => fromJs(x)),
-    )),
+    function* gen(fn: ArkVal) {
+      const result = yield* action(
+        toJs(fn) as (resolve: Resolve<unknown>, reject: Reject) => Operation<void>,
+      )
+      return fromJs(result)
+    },
   )],
+  // FIXME: should be able to use js.fetch directly.
   ['fetch', new NativeAsyncFn(
     ['url', 'options'],
     async (url: ArkVal, options: ArkVal) => new NativeObject(

diff --git a/src/ark/interpreter.ts b/src/ark/interpreter.ts
@@ -2,6 +2,9 @@
 // © Reuben Thomas 2023-2024
 // Released under the MIT license.
 
+import {
+  Operation, run, spawn,
+} from 'effection'
 import {Interval} from 'ohm-js'
 
 import {
@@ -15,8 +18,8 @@ import {
 } from './flatten.js'
 import {
   ArkAbstractObjectBase, ArkBoolean, ArkList, ArkMap, ArkNull, ArkNullVal,
-  ArkObject, ArkPromise, ArkUndefined, ArkVal, NativeAsyncFn, NativeFn,
-  ArkRef, ArkValRef,
+  ArkObject, ArkOperation, ArkUndefined, ArkVal, NativeAsyncFn, NativeFn,
+  NativeOperation, ArkRef, ArkValRef,
 } from './data.js'
 import {
   ArkCapture, ArkContinuation, ArkFlatClosure, ArkFlatGeneratorClosure,
@@ -101,6 +104,10 @@ function makeLocals(names: string[], vals: ArkVal[]): ArkRef[] {
 }
 
 async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
+  return run(() => doEvalFlat(outerArk))
+}
+
+function* doEvalFlat(outerArk: ArkState): Operation<ArkVal> {
   let ark: ArkState | undefined = outerArk
   let inst = ark.inst
   let prevInst
@@ -157,9 +164,10 @@ async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
         ),
         ark.outerState,
       )
-      const result = Promise.resolve(new ArkPromise(evalFlat(innerArk)))
+      const operation = yield* spawn(() => doEvalFlat(innerArk))
+      const result = new ArkOperation(operation)
       mem.set(inst.id, result)
-      // The Promise becomes the result of the entire block.
+      // The ArkOperation becomes the result of the entire block.
       mem.set(inst.matchingClose.id, result)
       inst = inst.matchingClose.next
     } else if (inst instanceof ArkCallableBlockOpenInst) {
@@ -184,8 +192,9 @@ async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
     } else if (inst instanceof ArkBlockOpenInst) {
       inst = inst.next
     } else if (inst instanceof ArkAwaitInst) {
-      const promise = (mem.get(inst.argId)! as ArkPromise).promise
-      mem.set(inst.id, await promise)
+      const operation = (mem.get(inst.argId)! as ArkOperation).operation
+      const result = yield* operation
+      mem.set(inst.id, result)
       inst = inst.next
     } else if (inst instanceof ArkBreakInst) {
       const result = mem.get(inst.argId)!
@@ -273,11 +282,10 @@ async function evalFlat(outerArk: ArkState): Promise<ArkVal> {
             inst = inst.next
           }
         }
-      } else if (callable instanceof NativeFn) {
-        mem.set(inst.id, callable.body(...args))
-        inst = inst.next
-      } else if (callable instanceof NativeAsyncFn) {
-        mem.set(inst.id, await callable.body(...args))
+      } else if (callable instanceof NativeFn
+        || callable instanceof NativeAsyncFn
+        || callable instanceof NativeOperation) {
+        mem.set(inst.id, yield* callable.body(...args))
         inst = inst.next
       } else {
         throw new ArkRuntimeError(ark, 'Invalid call', inst.sourceLoc)

diff --git a/src/ark/serialize.ts b/src/ark/serialize.ts
@@ -4,7 +4,7 @@
 
 import {
   globals, ArkVal,
-  ArkConcreteVal, ArkNull, ArkPromise, ArkList, ArkMap, ArkObject,
+  ArkConcreteVal, ArkNull, ArkOperation, ArkList, ArkMap, ArkObject,
   ArkUndefined, NativeObject,
 } from './data.js'
 import {
@@ -87,7 +87,7 @@ export function valToJs(val: ArkVal | ArkExp, externalSyms = globals) {
       return ['yield', doValToJs(val.exp)]
     } else if (val instanceof ArkReturn) {
       return ['return', doValToJs(val.exp)]
-    } else if (val instanceof ArkPromise) {
+    } else if (val instanceof ArkOperation) {
       // FIXME: Can we properly serialize a promise?
       return ['promise']
     } else if (val === ArkNull()) {