(feat) Add variable name extractor to expression runner (#1173)

packages/framework/esm-expression-evaluator/src/evaluator.test.ts

@@ -1,3 +1,4 @@
+import { describe, it, expect } from '@jest/globals';
 import { compile, evaluate, evaluateAsBoolean, evaluateAsNumber, evaluateAsType, evaluateAsync } from './evaluator';
 
 describe('OpenMRS Expression Evaluator', () => {

packages/framework/esm-expression-evaluator/src/evaluator.ts

@@ -6,6 +6,7 @@ import jsepNumbers from '@jsep-plugin/numbers';
 import jsepRegex from '@jsep-plugin/regex';
 import jsepTernary from '@jsep-plugin/ternary';
 import jsepTemplate, { type TemplateElement, type TemplateLiteral } from '@jsep-plugin/template';
+import { globals, globalsAsync } from './globals';
 
 jsep.plugins.register(jsepArrow);
 jsep.plugins.register(jsepNew);
@@ -19,6 +20,8 @@ jsep.plugins.register(jsepTemplate);
 jsep.addBinaryOp('in', 7);
 jsep.addBinaryOp('??', 1);
 
+export { jsep };
+
 /** An object containing the variable to use when evaluating this expression */
 export type VariablesMap = {
   [key: string]: string | number | boolean | Function | RegExp | object | null | VariablesMap | Array<VariablesMap>;
@@ -213,7 +216,7 @@ export function evaluateAsNumber(expression: string | jsep.Expression, variables
  * @returns The result of evaluating the expression
  */
 export function evaluateAsNumberAsync(expression: string | jsep.Expression, variables: VariablesMap = {}) {
-  return evaluateAsType(expression, variables, numberTypePredicate);
+  return evaluateAsTypeAsync(expression, variables, numberTypePredicate);
 }
 
 /**
@@ -244,6 +247,7 @@ export function evaluateAsType<T>(
 
   const context = createSynchronousContext(variables);
   const result = visitExpression(typeof expression === 'string' ? jsep(expression) : expression, context);
+
   if (typePredicate(result)) {
     return result;
   } else {
@@ -718,38 +722,3 @@ function isValidVariableType(val: unknown): val is VariablesMap['a'] {
 
   return false;
 }
-
-const globals = {
-  Array,
-  Boolean,
-  Symbol,
-  Infinity,
-  NaN,
-  Math,
-  Number,
-  BigInt,
-  String,
-  RegExp,
-  JSON,
-  isFinite,
-  isNaN,
-  parseFloat,
-  parseInt,
-  decodeURI,
-  encodeURI,
-  encodeURIComponent,
-  Object: {
-    __proto__: undefined,
-    assign: Object.assign.bind(null),
-    fromEntries: Object.fromEntries.bind(null),
-    hasOwn: Object.hasOwn.bind(null),
-    keys: Object.keys.bind(null),
-    is: Object.is.bind(null),
-    values: Object.values.bind(null),
-  },
-};
-
-const globalsAsync = {
-  ...globals,
-  Promise,
-};

packages/framework/esm-expression-evaluator/src/extractor.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from '@jest/globals';
+import { extractVariableNames } from './extractor';
+
+describe('OpenMRS Expression Extractor', () => {
+  it('Should return empty list for expression lacking variables', () => {
+    expect(extractVariableNames('1 + 1')).toEqual([]);
+  });
+
+  it('Should support basic variables', () => {
+    expect(extractVariableNames('1 + a')).toEqual(['a']);
+  });
+
+  it('Should extracting both variables from binary operators', () => {
+    expect(extractVariableNames('a ?? b')).toEqual(['a', 'b']);
+  });
+
+  it('Should support functions', () => {
+    expect(extractVariableNames('a(b)')).toEqual(['a', 'b']);
+  });
+
+  it('Should support built-in functions', () => {
+    expect(extractVariableNames('a.includes("v")')).toEqual(['a']);
+    expect(extractVariableNames('"value".includes(a)')).toEqual(['a']);
+    expect(extractVariableNames('(3.14159).toPrecision(a)')).toEqual(['a']);
+  });
+
+  it('Should support string templates', () => {
+    expect(extractVariableNames('`${a.b}`')).toEqual(['a']);
+  });
+
+  it('Should support RegExp', () => {
+    expect(extractVariableNames('/.*/.test(a)')).toEqual(['a']);
+  });
+
+  it('Should support global objects', () => {
+    expect(extractVariableNames('Math.min(a, b, c)')).toEqual(['a', 'b', 'c']);
+    expect(extractVariableNames('isNaN(a)')).toEqual(['a']);
+  });
+
+  it('Should support arrow functions inside expressions', () => {
+    expect(extractVariableNames('[1, 2, 3].find(v => v === a)')).toEqual(['a']);
+  });
+
+  it('Should support real-world use-cases', () => {
+    expect(extractVariableNames('!isEmpty(array)')).toEqual(['isEmpty', 'array']);
+
+    expect(
+      extractVariableNames(
+        "includes(referredToPreventionServices, '88cdde2b-753b-48ac-a51a-ae5e1ab24846') && !includes(referredToPreventionServices, '1691AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')",
+      ),
+    ).toEqual(['includes', 'referredToPreventionServices']);
+
+    expect(
+      extractVariableNames(
+        "(no_interest === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : no_interest === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : no_interest === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (depressed === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : depressed === '234259ec-5368-4488-8482-4f261cc76714' ? 2 :  depressed==='8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (bad_sleep === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : bad_sleep === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : bad_sleep === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (feeling_tired === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : feeling_tired === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : feeling_tired === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) +(poor_appetite === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : poor_appetite === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : poor_appetite === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (troubled === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : troubled === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : troubled === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (feeling_bad === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : feeling_bad === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : feeling_bad === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (speaking_slowly === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : speaking_slowly === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : speaking_slowly === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0) + (better_dead === 'b631d160-8d40-4cf7-92cd-67f628c889e8' ? 1 : better_dead === '234259ec-5368-4488-8482-4f261cc76714' ? 2 : better_dead === '8ff1f85c-4f04-4f5b-936a-5aa9320cb66e' ? 3 : 0)",
+      ),
+    ).toEqual([
+      'no_interest',
+      'depressed',
+      'bad_sleep',
+      'feeling_tired',
+      'poor_appetite',
+      'troubled',
+      'feeling_bad',
+      'speaking_slowly',
+      'better_dead',
+    ]);
+  });
+});

packages/framework/esm-expression-evaluator/src/extractor.ts

@@ -0,0 +1,193 @@
+/** @category Utility */
+import { type ArrowExpression } from '@jsep-plugin/arrow';
+import { type NewExpression } from '@jsep-plugin/new';
+import { type TemplateElement, type TemplateLiteral } from '@jsep-plugin/template';
+import { jsep } from './evaluator';
+import { globalsAsync } from './globals';
+
+/**
+ * `extractVariableNames()` is a companion function for `evaluate()` and `evaluateAsync()` which extracts the
+ * names of all unbound identifiers used in the expression. The idea is to be able to extract all of the names
+ * of variables that will need to be supplied in order to correctly process the expression.
+ *
+ * @example
+ * ```ts
+ * // variables will be ['isEmpty', 'array']
+ * const variables = extractVariableNames('!isEmpty(array)')
+ * ```
+ *
+ * An identifier is considered "unbound" if it is not a reference to the property of an object, is not defined
+ * as a parameter to an inline arrow function, and is not a global value. E.g.,
+ *
+ * @example
+ * ```ts
+ * // variables will be ['obj']
+ * const variables = extractVariableNames('obj.prop()')
+ * ```
+ *
+ * @example
+ * ```ts
+ * // variables will be ['arr', 'needle']
+ * const variables = extractVariableNames('arr.filter(v => v === needle)')
+ * ```
+ *
+ @example
+ * ```ts
+ * // variables will be ['myVar']
+ * const  variables = extractVariableNames('new String(myVar)')
+ *
+ * Note that because this expression evaluator uses a restricted definition of "global" there are some Javascript
+ * globals that will be reported as a unbound expression. This is expected because the evaluator will still fail
+ * on these expressions.
+ */
+export function extractVariableNames(expression: string | jsep.Expression) {
+  if (typeof expression !== 'string' && (typeof expression !== 'object' || !expression || !('type' in expression))) {
+    throw `Unknown expression type ${expression}. Expressions must either be a string or pre-compiled string.`;
+  }
+
+  const context = createAsynchronousContext();
+  visitExpression(typeof expression === 'string' ? jsep(expression) : expression, context);
+
+  return [...context.variables];
+}
+
+function visitExpression(expression: jsep.Expression, context: EvaluationContext) {
+  switch (expression.type) {
+    case 'UnaryExpression':
+      return visitUnaryExpression(expression as jsep.UnaryExpression, context);
+    case 'BinaryExpression':
+      return visitBinaryExpression(expression as jsep.BinaryExpression, context);
+    case 'ConditionalExpression':
+      return visitConditionalExpression(expression as jsep.ConditionalExpression, context);
+    case 'CallExpression':
+      return visitCallExpression(expression as jsep.CallExpression, context);
+    case 'ArrowFunctionExpression':
+      return visitArrowFunctionExpression(expression as ArrowExpression, context);
+    case 'MemberExpression':
+      return visitMemberExpression(expression as jsep.MemberExpression, context);
+    case 'ArrayExpression':
+      return visitArrayExpression(expression as jsep.ArrayExpression, context);
+    case 'SequenceExpression':
+      return visitSequenceExpression(expression as jsep.SequenceExpression, context);
+    case 'NewExpression':
+      return visitNewExpression(expression as NewExpression, context);
+    case 'Literal':
+      return visitLiteral(expression as jsep.Literal, context);
+    case 'Identifier':
+      return visitIdentifier(expression as jsep.Identifier, context);
+    case 'TemplateLiteral':
+      return visitTemplateLiteral(expression as TemplateLiteral, context);
+    case 'TemplateElement':
+      return visitTemplateElement(expression as TemplateElement, context);
+    default:
+      throw `Expression evaluator does not support expression of type '${expression.type}'`;
+  }
+}
+
+function visitUnaryExpression(expression: jsep.UnaryExpression, context: EvaluationContext) {
+  return visitExpression(expression.argument, context);
+}
+
+function visitBinaryExpression(expression: jsep.BinaryExpression, context: EvaluationContext) {
+  const left = visitExpression(expression.left, context);
+  const right = visitExpression(expression.right, context);
+  return [left, right].filter(Boolean);
+}
+
+function visitConditionalExpression(expression: jsep.ConditionalExpression, context: EvaluationContext) {
+  const consequent = visitExpression(expression.consequent, context);
+  const test = visitExpression(expression.test, context);
+  const alternate = visitExpression(expression.alternate, context);
+  return [consequent, test, alternate].filter(Boolean);
+}
+
+function visitCallExpression(expression: jsep.CallExpression, context: EvaluationContext) {
+  const fn = visitExpression(expression.callee, context);
+  expression.arguments?.map(handleNullableExpression(context));
+  return fn;
+}
+
+function visitArrowFunctionExpression(expression: ArrowExpression, context: EvaluationContext) {
+  const newContext = { ...context };
+  newContext.isLocalExpression = true;
+
+  const params = expression.params?.map(handleNullableExpression(newContext)) ?? [];
+  const bodyVariables = visitExpression(expression.body, newContext) ?? [];
+
+  if (bodyVariables && Array.isArray(bodyVariables)) {
+    for (const v of bodyVariables) {
+      if (!params.includes(v)) {
+        context.variables.add(v);
+      }
+    }
+  }
+}
+
+function visitMemberExpression(expression: jsep.MemberExpression, context: EvaluationContext) {
+  visitExpression(expression.object, context);
+  const newContext = { ...context };
+  newContext.isLocalExpression = true;
+  visitExpression(expression.property, newContext);
+}
+
+function visitArrayExpression(expression: jsep.ArrayExpression, context: EvaluationContext) {
+  expression.elements?.map(handleNullableExpression(context));
+}
+
+function visitSequenceExpression(expression: jsep.SequenceExpression, context: EvaluationContext) {
+  expression.expressions?.map(handleNullableExpression(context));
+}
+
+function visitNewExpression(expression: NewExpression, context: EvaluationContext) {
+  expression.arguments?.map(handleNullableExpression(context));
+}
+
+function visitTemplateLiteral(expression: TemplateLiteral, context: EvaluationContext) {
+  expression.expressions?.map(handleNullableExpression(context));
+  expression.quasis?.map(handleNullableExpression(context));
+}
+
+function visitTemplateElement(expression: TemplateElement, context: EvaluationContext) {}
+
+function visitIdentifier(expression: jsep.Identifier, context: EvaluationContext) {
+  if (!(expression.name in context.globals)) {
+    if (!context.isLocalExpression) {
+      context.variables.add(expression.name);
+    } else {
+      return expression.name;
+    }
+  }
+}
+
+function visitLiteral(expression: jsep.Literal, context: EvaluationContext) {}
+
+// Internals
+interface EvaluationContext {
+  globals: typeof globalsAsync;
+  isLocalExpression: boolean;
+  variables: Set<string>;
+}
+
+function createAsynchronousContext(): EvaluationContext {
+  return createContextInternal(globalsAsync);
+}
+
+function createContextInternal(globals_: typeof globalsAsync) {
+  const context = {
+    globals: { ...globals_ },
+    isLocalExpression: false,
+    variables: new Set<string>(),
+  };
+
+  return context;
+}
+
+function handleNullableExpression(context: EvaluationContext) {
+  return function handleNullableExpressionInner(expression: jsep.Expression | null) {
+    if (expression === null) {
+      return null;
+    }
+
+    return visitExpression(expression, context);
+  };
+}

packages/framework/esm-expression-evaluator/src/globals.ts

@@ -0,0 +1,34 @@
+export const globals = {
+  Array,
+  Boolean,
+  Symbol,
+  Infinity,
+  NaN,
+  Math,
+  Number,
+  BigInt,
+  String,
+  RegExp,
+  JSON,
+  isFinite,
+  isNaN,
+  parseFloat,
+  parseInt,
+  decodeURI,
+  encodeURI,
+  encodeURIComponent,
+  Object: {
+    __proto__: undefined,
+    assign: Object.assign.bind(null),
+    fromEntries: Object.fromEntries.bind(null),
+    hasOwn: Object.hasOwn.bind(null),
+    keys: Object.keys.bind(null),
+    is: Object.is.bind(null),
+    values: Object.values.bind(null),
+  },
+};
+
+export const globalsAsync = {
+  ...globals,
+  Promise,
+};

packages/framework/esm-expression-evaluator/src/index.ts

@@ -1 +1,2 @@
 export * from './evaluator';
+export * from './extractor';

packages/framework/esm-expression-evaluator/src/public.ts

@@ -11,3 +11,4 @@ export {
   type VariablesMap,
   type DefaultEvaluateReturnType,
 } from './evaluator';
+export { extractVariableNames } from './extractor';