Fix: Complex jexl issues

.pre-commit-config.yaml

@@ -2,19 +2,19 @@ repos:
 -   repo: local
     hooks:
     - id: ruff-format
-      stages: [commit]
+      stages: [pre-commit]
       name: format code
       language: system
       entry: ruff format .
       types: [python]
     - id: ruff-check
-      stages: [commit]
+      stages: [pre-commit]
       name: check format,import
       language: system
       entry: ruff check .
       types: [python]
     - id: reuse
-      stages: [commit]
+      stages: [pre-commit]
       name: reuse
       description: Check licenses
       entry: reuse lint

.reuse/dep5

@@ -33,6 +33,7 @@ License: CC0-1.0
 Files:
     caluma/*.py
     caluma/*.ambr
+    caluma/*.json
 Copyright: 2019 Adfinis AG <info@adfinis.com>
 License: GPL-3.0-or-later
 

caluma/caluma_core/jexl.py

@@ -167,6 +167,11 @@ def _length_transform(self, value, *options):
             return None
 
     def evaluate(self, expression, context=None):
+        # log.info(
+        #    "JEXL: evaluating expression <<< %s >>> in context: %s",
+        #    str(expression),
+        #    str(dict(context)),
+        # )
         self._expr_stack.append(expression)
         try:
             return super().evaluate(expression, context)

caluma/caluma_form/domain_logic.py

@@ -1,4 +1,5 @@
 from graphlib import TopologicalSorter
+from logging import getLogger
 from typing import Optional
 
 from django.db import transaction
@@ -12,6 +13,8 @@
 from caluma.caluma_user.models import BaseUser
 from caluma.utils import update_model
 
+log = getLogger(__name__)
+
 
 class BaseLogic:
     @staticmethod
@@ -156,13 +159,24 @@ def post_save(answer: models.Answer) -> models.Answer:
     def update_calc_dependents(answer):
         if not answer.question.calc_dependents:
             return
+        log.debug("update_calc_dependents(%s)", answer)
 
         root_doc = utils.prefetch_document(answer.document.family_id)
         struc = structure.FieldSet(root_doc, root_doc.form)
 
         for question in models.Question.objects.filter(
             pk__in=answer.question.calc_dependents
         ):
+            log.debug(
+                "update_calc_dependents(%s): updating question %s", answer, question.pk
+            )
+            # FIXME: update_or_create_calc_answer() does not properly
+            # deal with table rows: we start recalculating from the root doc,
+            # but if a recalculation was triggered inside a table row, we need to
+            # do *that* recalc properly as well (we search for dependents here, but
+            # if those dependents are in a row doc, we can't find all of them and)
+            # don't properly update them either (get_field() returns None, because the
+            # affected question is not in the root form)
             update_or_create_calc_answer(question, root_doc, struc)
 
     @classmethod

caluma/caluma_form/jexl2.py

@@ -0,0 +1,143 @@
+from collections import ChainMap
+from contextlib import contextmanager
+from functools import partial
+
+from pyjexl.analysis import ValidatingAnalyzer
+
+from caluma.caluma_form.jexl import QuestionMissing
+from caluma.caluma_form.structure2 import BaseField
+
+from ..caluma_core.jexl import (
+    JEXL,
+    ExtractTransformArgumentAnalyzer,
+    ExtractTransformSubjectAnalyzer,
+    ExtractTransformSubjectAndArgumentsAnalyzer,
+)
+from .structure import Field
+
+"""
+Rewrite of the JEXL handling code.
+
+Design principles:
+
+* The JEXL classes do not deal with context switching between questions anymore
+* The QuestionJexl class only sets up the "runtime", any context is used from the
+  structure2 code
+* We only deal with the *evaluation*, no transform/extraction is happening here - that code
+  is mostly fine and doesn't need a rewrite
+* Caching is done by the structure2 code, not here
+* JEXL evaluation happens lazily, but the results are cached.
+"""
+
+
+class QuestionValidatingAnalyzer(ValidatingAnalyzer):
+    def visit_Transform(self, transform):
+        if transform.name == "answer" and not isinstance(transform.subject.value, str):
+            yield f"{transform.subject.value} is not a valid question slug."
+
+        yield from super().visit_Transform(transform)
+
+
+class QuestionJexl2(JEXL):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.current_structure = []
+
+        self.add_transform("answer", self.answer_transform)
+
+    def get_structure(self):
+        return self.current_structure[-1]
+
+    @contextmanager
+    def use_structure(self, context):
+        self.current_structure.append(context)
+        try:
+            yield
+        finally:
+            self.current_structure.pop()
+
+    def answer_transform(self, question_slug, *args):
+        context = self.get_structure()
+        field = context.get_field(question_slug)
+
+        def _default_or_empty():
+            if len(args):
+                return args[0]
+            return field.question.empty_value()
+
+        if not field:
+            if args:
+                return args[0]
+            else:
+                # No default arg, so we must raise an exception
+                raise QuestionMissing(
+                    f"Question `{question_slug}` could not be found in form {context.get_form()}"
+                )
+
+        if field.is_hidden():
+            # Hidden fields *always* return the empty value, even if we have
+            # a default
+            return field.question.empty_value()
+        elif field.is_empty():
+            # not hidden, but empty
+            return _default_or_empty()
+
+        return field.get_value()
+
+    def validate(self, expression, **kwargs):
+        return super().validate(expression, QuestionValidatingAnalyzer)
+
+    def extract_referenced_questions(self, expr):
+        transforms = ["answer"]
+        yield from self.analyze(
+            expr, partial(ExtractTransformSubjectAnalyzer, transforms=transforms)
+        )
+
+    def extract_referenced_questions_with_arguments(self, expr):
+        transforms = ["answer"]
+        yield from self.analyze(
+            expr,
+            partial(ExtractTransformSubjectAndArgumentsAnalyzer, transforms=transforms),
+        )
+
+    def extract_referenced_mapby_questions(self, expr):
+        transforms = ["mapby"]
+        yield from self.analyze(
+            expr, partial(ExtractTransformArgumentAnalyzer, transforms=transforms)
+        )
+
+    def _get_referenced_fields(self, field: Field, expr: str):
+        deps = list(self.extract_referenced_questions_with_arguments(expr))
+        referenced_fields = [self._structure.get_field(slug) for slug, _ in deps]
+
+        referenced_slugs = [ref.question.slug for ref in referenced_fields if ref]
+
+        for slug, args in deps:
+            required = len(args) == 0
+            if slug not in referenced_slugs and required:
+                raise QuestionMissing(
+                    f"Question `{slug}` could not be found in form {field.form}"
+                )
+
+        return [field for field in referenced_fields if field]
+
+    def evaluate(self, expr, context: BaseField, raise_on_error=True):
+        try:
+            with self.use_structure(context):
+                # Sadly, some expressions (such as the answer transform)
+                # need the current context but don't regularly have it available.
+                # Therefore we use this context manager so it can do it's job
+                # Also, combine the global context (self.context) with
+                return super().evaluate(
+                    expr, ChainMap(self.context, context.get_context())
+                )
+        except (
+            TypeError,
+            ValueError,
+            ZeroDivisionError,
+            AttributeError,
+        ):
+            if raise_on_error:
+                raise
+            return None

caluma/caluma_form/structure.py

@@ -4,7 +4,7 @@
 from functools import singledispatch
 from typing import List, Optional
 
-from .models import Question
+from .models import FormQuestion, Question
 
 
 def object_local_memoise(method):
@@ -123,7 +123,7 @@ def children(self):
         # for the answerdocument_set. Sorting in DB would re-issue the query
         rows = sorted(
             self.answer.answerdocument_set.all(),
-            key=lambda answer_document: answer_document.sort,
+            key=lambda answer_document: -answer_document.sort,
         )
         return [
             FieldSet(
@@ -243,15 +243,17 @@ def get_field(
     @object_local_memoise
     def children(self):
         answers = {ans.question_id: ans for ans in self.document.answers.all()}
+        formquestions = FormQuestion.objects.filter(form=self.form).order_by("-sort")
+
         return [
             Field.factory(
                 document=self.document,
                 form=self.form,
-                question=question,
-                answer=answers.get(question.slug),
+                question=fq.question,
+                answer=answers.get(fq.question.slug),
                 parent=self,
             )
-            for question in self.form.questions.all()
+            for fq in formquestions
         ]
 
     def set_answer(self, question_slug, answer):