CodeEditor improvements in preparation for rendering uncommitted edits

- Auto-indent:
  - New line matches indentation of previous line
  - Backspace now deletes 4 spaces of indentation
- New `Ast`-based implementation of `ensoSyntax` extension
  - Stylesheet-defined syntax highlighting (fixes #11475)
  - Vue component implementation of code editor tooltips
  - Removes 2nd-to-last usage of `RawAst` (#10753)
- Switch to CM-compatible SourceRange so that we can use the same text
  algorithms for module sources and CM editor contents
- Update CodeMirror dependencies

app/gui/package.json

@@ -92,13 +92,13 @@
     "@ag-grid-enterprise/range-selection": "^32.3.3",
     "@babel/parser": "^7.24.7",
     "babel-plugin-react-compiler": "19.0.0-beta-a7bf2bd-20241110",
-    "@codemirror/commands": "^6.6.0",
-    "@codemirror/language": "^6.10.2",
+    "@codemirror/commands": "^6.7.1",
+    "@codemirror/language": "^6.10.6",
     "@codemirror/lang-markdown": "^v6.3.0",
-    "@codemirror/lint": "^6.8.1",
-    "@codemirror/search": "^6.5.6",
-    "@codemirror/state": "^6.4.1",
-    "@codemirror/view": "^6.35.0",
+    "@codemirror/lint": "^6.8.4",
+    "@codemirror/search": "^6.5.8",
+    "@codemirror/state": "^6.5.0",
+    "@codemirror/view": "^6.35.3",
     "@fast-check/vitest": "^0.0.8",
     "@floating-ui/vue": "^1.0.6",
     "@lezer/common": "^1.1.0",

app/gui/src/project-view/components/CodeEditor/CodeEditorImpl.vue

@@ -4,45 +4,58 @@ import { ensoSyntax } from '@/components/CodeEditor/ensoSyntax'
 import { useEnsoSourceSync } from '@/components/CodeEditor/sync'
 import { ensoHoverTooltip } from '@/components/CodeEditor/tooltips'
 import CodeMirrorRoot from '@/components/CodeMirrorRoot.vue'
+import VueComponentHost from '@/components/VueComponentHost.vue'
 import { useGraphStore } from '@/stores/graph'
 import { useProjectStore } from '@/stores/project'
 import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
 import { useAutoBlur } from '@/util/autoBlur'
 import { useCodeMirror } from '@/util/codemirror'
+import { highlightStyle } from '@/util/codemirror/highlight'
 import { testSupport } from '@/util/codemirror/testSupport'
-import { indentWithTab } from '@codemirror/commands'
-import {
-  bracketMatching,
-  defaultHighlightStyle,
-  foldGutter,
-  syntaxHighlighting,
-} from '@codemirror/language'
+import { indentWithTab, insertNewlineKeepIndent } from '@codemirror/commands'
+import { bracketMatching, foldGutter } from '@codemirror/language'
 import { lintGutter } from '@codemirror/lint'
 import { highlightSelectionMatches } from '@codemirror/search'
 import { keymap } from '@codemirror/view'
-import { type Highlighter } from '@lezer/highlight'
 import { minimalSetup } from 'codemirror'
-import { computed, onMounted, useTemplateRef, type ComponentInstance } from 'vue'
+import {
+  computed,
+  onMounted,
+  toRef,
+  useCssModule,
+  useTemplateRef,
+  type ComponentInstance,
+} from 'vue'
 
 const projectStore = useProjectStore()
 const graphStore = useGraphStore()
 const suggestionDbStore = useSuggestionDbStore()
+
+const vueComponentHost =
+  useTemplateRef<ComponentInstance<typeof VueComponentHost>>('vueComponentHost')
 const editorRoot = useTemplateRef<ComponentInstance<typeof CodeMirrorRoot>>('editorRoot')
 const rootElement = computed(() => editorRoot.value?.rootElement)
 useAutoBlur(rootElement)
 
+const autoindentOnEnter = {
+  key: 'Enter',
+  run: insertNewlineKeepIndent,
+}
+
+const vueHost = computed(() => vueComponentHost.value || undefined)
 const { editorView, setExtraExtensions } = useCodeMirror(editorRoot, {
   extensions: [
+    keymap.of([indentWithTab, autoindentOnEnter]),
     minimalSetup,
-    syntaxHighlighting(defaultHighlightStyle as Highlighter),
     bracketMatching(),
     foldGutter(),
     lintGutter(),
     highlightSelectionMatches(),
-    ensoSyntax(),
-    ensoHoverTooltip(graphStore, suggestionDbStore),
-    keymap.of([indentWithTab]),
+    ensoSyntax(toRef(graphStore, 'moduleRoot')),
+    highlightStyle(useCssModule()),
+    ensoHoverTooltip(graphStore, suggestionDbStore, vueHost),
   ],
+  vueHost,
 })
 ;(window as any).__codeEditorApi = testSupport(editorView)
 const { updateListener, connectModuleListener } = useEnsoSourceSync(
@@ -61,6 +74,7 @@ onMounted(() => {
 
 <template>
   <CodeMirrorRoot ref="editorRoot" class="CodeEditor" @keydown.tab.stop.prevent />
+  <VueComponentHost ref="vueComponentHost" />
 </template>
 
 <style scoped>
@@ -73,7 +87,6 @@ onMounted(() => {
 }
 
 :deep(.cm-scroller) {
-  font-family: var(--font-mono);
   /* Prevent touchpad back gesture, which can be triggered while panning. */
   overscroll-behavior: none;
 }
@@ -113,3 +126,32 @@ onMounted(() => {
   min-width: 32px;
 }
 </style>
+
+<!--suppress CssUnusedSymbol -->
+<style module>
+.keyword,
+.moduleKeyword,
+.modifier {
+  color: #708;
+}
+.number {
+  color: #164;
+}
+.string {
+  color: #a11;
+}
+.escape {
+  color: #e40;
+}
+.variableName,
+.definition-variableName {
+  color: #00f;
+}
+.lineComment,
+.docComment {
+  color: #940;
+}
+.invalid {
+  color: #f00;
+}
+</style>

app/gui/src/project-view/components/CodeEditor/CodeEditorTooltip.vue

@@ -0,0 +1,47 @@
+<script setup lang="ts">
+import { type NodeId } from '@/stores/graph'
+import { type GraphDb } from '@/stores/graph/graphDatabase'
+import { type SuggestionDbStore } from '@/stores/suggestionDatabase'
+import { computed } from 'vue'
+
+const { nodeId, syntax, graphDb, suggestionDbStore } = defineProps<{
+  nodeId: NodeId | undefined
+  syntax: string
+  graphDb: GraphDb
+  suggestionDbStore: SuggestionDbStore
+}>()
+
+const expressionInfo = computed(() => nodeId && graphDb.getExpressionInfo(nodeId))
+const typeName = computed(
+  () => expressionInfo.value && (expressionInfo.value.typename ?? 'Unknown'),
+)
+const executionTimeMs = computed(
+  () =>
+    expressionInfo.value?.profilingInfo[0] &&
+    (expressionInfo.value.profilingInfo[0].ExecutionTime.nanoTime / 1_000_000).toFixed(3),
+)
+const method = computed(() => expressionInfo.value?.methodCall?.methodPointer)
+const group = computed(() => {
+  const id = method.value && suggestionDbStore.entries.findByMethodPointer(method.value)
+  if (id == null) return
+  const suggestionEntry = suggestionDbStore.entries.get(id)
+  if (!suggestionEntry) return
+  const groupIndex = suggestionEntry.groupIndex
+  if (groupIndex == null) return
+  const group = suggestionDbStore.groups[groupIndex]
+  if (!group) return
+  return {
+    name: `${group.project}.${group.name}`,
+    color: group.color,
+  }
+})
+</script>
+
+<template>
+  <div v-if="nodeId">AST ID: {{ nodeId }}</div>
+  <div v-if="typeName">Type: {{ typeName }}</div>
+  <div v-if="executionTimeMs != null">Execution Time: {{ executionTimeMs }}ms</div>
+  <div>Syntax: {{ syntax }}</div>
+  <div v-if="method">Method: {{ method.module }}.{{ method.name }}</div>
+  <div v-if="group" :style="{ color: group.color }">Group: {{ group.name }}</div>
+</template>

app/gui/src/project-view/components/CodeEditor/diagnostics.ts

@@ -1,74 +1,20 @@
 import { type GraphStore } from '@/stores/graph'
 import { type ProjectStore } from '@/stores/project'
+import { valueExt } from '@/util/codemirror/stateEffect'
 import { type Diagnostic, forceLinting, linter } from '@codemirror/lint'
-import { type Extension, StateEffect, StateField } from '@codemirror/state'
+import { type Extension } from '@codemirror/state'
 import { type EditorView } from '@codemirror/view'
 import * as iter from 'enso-common/src/utilities/data/iter'
-import { computed, watch } from 'vue'
-import { type Diagnostic as LSDiagnostic, type Position } from 'ydoc-shared/languageServerTypes'
+import { computed, watchEffect } from 'vue'
+import { type SourceRange } from 'ydoc-shared/util/data/text'
+import { type ExternalId } from 'ydoc-shared/yjsModel'
 
-// Effect that can be applied to the document to invalidate the linter state.
-const diagnosticsUpdated = StateEffect.define()
-// State value that is perturbed by any `diagnosticsUpdated` effect.
-const diagnosticsVersion = StateField.define({
-  create: (_state) => 0,
-  update: (value, transaction) => {
-    for (const effect of transaction.effects) {
-      if (effect.is(diagnosticsUpdated)) value += 1
-    }
-    return value
-  },
-})
-
-/** Given a text, indexes it and returns a function for converting between different ways of identifying positions. */
-function stringPosConverter(text: string) {
-  let pos = 0
-  const lineStartIndex: number[] = []
-  for (const line of text.split('\n')) {
-    lineStartIndex.push(pos)
-    pos += line.length + 1
-  }
-  const length = text.length
-
-  function lineColToIndex({
-    line,
-    character,
-  }: {
-    line: number
-    character: number
-  }): number | undefined {
-    const startIx = lineStartIndex[line]
-    if (startIx == null) return
-    const ix = startIx + character
-    if (ix > length) return
-    return ix
-  }
-
-  return { lineColToIndex }
-}
-
-/** Convert the Language Server's diagnostics to CodeMirror diagnostics. */
-function lsDiagnosticsToCMDiagnostics(
-  diagnostics: LSDiagnostic[],
-  lineColToIndex: (lineCol: Position) => number | undefined,
-) {
-  const results: Diagnostic[] = []
-  for (const diagnostic of diagnostics) {
-    if (!diagnostic.location) continue
-    const from = lineColToIndex(diagnostic.location.start)
-    const to = lineColToIndex(diagnostic.location.end)
-    if (to == null || from == null) {
-      // Suppress temporary errors if the source is not the version of the document the LS is reporting diagnostics for.
-      continue
-    }
-    const severity =
-      diagnostic.kind === 'Error' ? 'error'
-      : diagnostic.kind === 'Warning' ? 'warning'
-      : 'info'
-    results.push({ from, to, message: diagnostic.message, severity })
-  }
-  return results
-}
+const {
+  set: setDiagnostics,
+  get: getDiagnostics,
+  changed: diagnosticsChanged,
+  extension: stateExt,
+} = valueExt<Diagnostic[], Diagnostic[] | undefined>(undefined)
 
 /**
  * CodeMirror extension providing diagnostics for an Enso module. Provides CodeMirror diagnostics based on dataflow
@@ -79,6 +25,13 @@ export function useEnsoDiagnostics(
   graphStore: Pick<GraphStore, 'moduleSource' | 'db'>,
   editorView: EditorView,
 ): Extension {
+  function spanOfExternalId(externalId: ExternalId): SourceRange | undefined {
+    const astId = graphStore.db.idFromExternal(externalId)
+    if (!astId) return
+    const span = graphStore.moduleSource.getSpan(astId)
+    if (!span) return
+    return span
+  }
   const expressionUpdatesDiagnostics = computed(() => {
     const updates = projectStore.computedValueRegistry.db
     const panics = updates.type.reverseLookup('Panic')
@@ -87,11 +40,9 @@ export function useEnsoDiagnostics(
     for (const externalId of iter.chain(panics, errors)) {
       const update = updates.get(externalId)
       if (!update) continue
-      const astId = graphStore.db.idFromExternal(externalId)
-      if (!astId) continue
-      const span = graphStore.moduleSource.getSpan(astId)
+      const span = spanOfExternalId(externalId)
       if (!span) continue
-      const [from, to] = span
+      const { from, to } = span
       switch (update.payload.type) {
         case 'Panic': {
           diagnostics.push({ from, to, message: update.payload.message, severity: 'error' })
@@ -108,27 +59,30 @@ export function useEnsoDiagnostics(
     }
     return diagnostics
   })
-  // The LS protocol doesn't identify what version of the file updates are in reference to. When diagnostics are
-  // received from the LS, we map them to the text assuming that they are applicable to the current version of the
-  // module. This will be correct if there is no one else editing, and we aren't editing faster than the LS can send
-  // updates. Typing too quickly can result in incorrect ranges, but at idle it should correct itself when we receive
-  // new diagnostics.
-  const executionContextDiagnostics = computed(() => {
-    const { lineColToIndex } = stringPosConverter(graphStore.moduleSource.text)
-    return lsDiagnosticsToCMDiagnostics(projectStore.diagnostics, lineColToIndex)
-  })
-  watch([executionContextDiagnostics, expressionUpdatesDiagnostics], () => {
-    editorView.dispatch({ effects: diagnosticsUpdated.of(null) })
+  const executionContextDiagnostics = computed<Diagnostic[]>(() =>
+    projectStore.diagnostics.flatMap((diagnostic) => {
+      const span = diagnostic.expressionId && spanOfExternalId(diagnostic.expressionId)
+      if (!span) return []
+      const { from, to } = span
+      const severity =
+        diagnostic.kind === 'Error' ? 'error'
+        : diagnostic.kind === 'Warning' ? 'warning'
+        : 'info'
+      return [{ from, to, message: diagnostic.message, severity }]
+    }),
+  )
+  watchEffect(() => {
+    const diagnostics = [
+      ...expressionUpdatesDiagnostics.value,
+      ...executionContextDiagnostics.value,
+    ]
+    editorView.dispatch({ effects: setDiagnostics.of(diagnostics) })
     forceLinting(editorView)
   })
   return [
-    diagnosticsVersion,
-    linter(() => [...executionContextDiagnostics.value, ...expressionUpdatesDiagnostics.value], {
-      needsRefresh(update) {
-        return (
-          update.state.field(diagnosticsVersion) !== update.startState.field(diagnosticsVersion)
-        )
-      },
+    stateExt,
+    linter((view) => view.state.facet(getDiagnostics) ?? [], {
+      needsRefresh: diagnosticsChanged,
     }),
   ]
 }