diff --git a/Cargo.lock b/Cargo.lock index 85bf6e97..cd423068 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,6 +1085,7 @@ dependencies = [ "lsp-types", "naga", "parking_lot", + "regex", "ropey", "serde", "serde_json", diff --git a/packages/server/Cargo.toml b/packages/server/Cargo.toml index 9ebb824e..4c779d67 100644 --- a/packages/server/Cargo.toml +++ b/packages/server/Cargo.toml @@ -18,6 +18,7 @@ itertools = "0.12" lsp-server = "0.7" lsp-types = "0.95" naga = { version = "0.19", features = ["wgsl-in"] } +regex = "1.10.4" ropey = "1.5" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/packages/server/src/documents/intake.rs b/packages/server/src/documents/intake.rs index 2d8a164d..672af805 100644 --- a/packages/server/src/documents/intake.rs +++ b/packages/server/src/documents/intake.rs @@ -21,8 +21,9 @@ use crate::{ use super::{ DocumentUri, DocumentsMap, ImportedSymbols, InactiveRanges, ModulePathsMap, ParseErrors, - PendingPreprocessing, PendingReads, PruneErrors, WgslAst, WgslComments, WgslPrunedSource, - WgslRope, WgslScopes, WgslSource, WgslSourceMap, WgslTokens, + PendingPreprocessing, PendingReads, PruneErrors, ReplacementErrors, SourceReplacements, + WgslAst, WgslComments, WgslPrunedSource, WgslRope, WgslScopes, WgslSource, WgslSourceMap, + WgslTokens, }; #[derive(Bundle)] @@ -178,6 +179,8 @@ fn spawn_preprocessed( source_map, inactive_spans, prune_errors, + replacements, + replacement_errors, } = pre::prune(&uri, source, defs); let mut parser = ParseStream::from(&pruned); @@ -208,6 +211,8 @@ fn spawn_preprocessed( WgslComments(comments), ParseErrors(parse_errors), PruneErrors(prune_errors), + SourceReplacements(replacements), + ReplacementErrors(replacement_errors), WgslAst(tree), WgslScopes(scopes), TokenReferences::default(), diff --git a/packages/server/src/documents/mod.rs b/packages/server/src/documents/mod.rs index 899c9849..b7156ff8 100644 --- a/packages/server/src/documents/mod.rs +++ b/packages/server/src/documents/mod.rs @@ -26,7 +26,7 @@ use crate::{ config::Config, ipc::{notify, Ipc}, lsp_extensions::{UnreadDependency, UnreadDependencyParams}, - pre::{self, PruneResult, SourceMap}, + pre::{self, PruneResult, SourceMap, SourceReplacement}, utils, workspace::Workspace, }; @@ -104,6 +104,12 @@ pub struct ParseErrors(pub Vec); #[derive(Component, Debug, Deref, DerefMut)] pub struct PruneErrors(pub Vec); +#[derive(Component, Debug, Deref, DerefMut)] +pub struct SourceReplacements(pub Vec); + +#[derive(Component, Debug, Deref, DerefMut)] +pub struct ReplacementErrors(pub Vec); + #[derive(Component, Clone, Debug, Deref, DerefMut)] pub struct WgslRope(pub Rope); @@ -210,6 +216,8 @@ fn update_documents( source_map, inactive_spans, prune_errors, + replacements, + replacement_errors, } = pre::prune(uri, source, defs); let was_pruned = pruned != original; @@ -263,6 +271,8 @@ fn update_documents( WgslSourceMap(source_map), InactiveRanges(inactive_spans), PruneErrors(prune_errors), + SourceReplacements(replacements), + ReplacementErrors(replacement_errors), )); } } diff --git a/packages/server/src/pre/interpolation.rs b/packages/server/src/pre/interpolation.rs new file mode 100644 index 00000000..90bf57d9 --- /dev/null +++ b/packages/server/src/pre/interpolation.rs @@ -0,0 +1,191 @@ +use std::{collections::VecDeque, sync::OnceLock}; + +use bevy_utils::HashMap; +use gramatika::{ArcStr, Position, SpannedError, Substr}; +use parser::Span; +use regex::{CaptureLocations, Regex}; + +static BRACED_PAT: OnceLock = OnceLock::new(); +static UNBRACED_PAT: OnceLock = OnceLock::new(); + +fn braced_pat() -> &'static Regex { + BRACED_PAT.get_or_init(|| Regex::new(r"#\{([a-zA-Z_][a-zA-Z0-9_]*)\}").unwrap()) +} + +fn unbraced_pat() -> &'static Regex { + UNBRACED_PAT.get_or_init(|| Regex::new(r"#([a-zA-Z_][a-zA-Z0-9_]*)").unwrap()) +} + +#[derive(Clone, Debug)] +pub enum SourceSlice { + Verbatim(Substr, Span), + Replacement(SourceReplacement), +} + +#[derive(Clone, Debug)] +pub struct SourceReplacement { + pub original: (Substr, Span), + pub replacement: String, +} + +pub(super) struct InterpolationStream<'a> { + full_source: ArcStr, + source: &'a [(Substr, Span)], + defs: &'a HashMap, + index: usize, + buffer: VecDeque, + braced_locs: CaptureLocations, + unbraced_locs: CaptureLocations, + errors: Vec, +} + +impl<'a> InterpolationStream<'a> { + pub fn new( + full_source: ArcStr, + source: &'a [(Substr, Span)], + defs: &'a HashMap, + ) -> Self { + Self { + full_source, + source, + defs, + index: 0, + buffer: VecDeque::new(), + braced_locs: braced_pat().capture_locations(), + unbraced_locs: unbraced_pat().capture_locations(), + errors: vec![], + } + } + + pub fn into_inner(self) -> Vec { + self.errors + } +} + +impl<'a> Iterator for InterpolationStream<'a> { + type Item = SourceSlice; + + fn next(&mut self) -> Option { + use SourceSlice::*; + + if !self.buffer.is_empty() { + return self.buffer.pop_front(); + } + + if self.index >= self.source.len() { + return None; + } + + let (mut substr, mut span) = self.source[self.index].clone(); + + while let Some(m) = None + .or_else(|| braced_pat().captures_read(&mut self.braced_locs, &substr)) + .or_else(|| unbraced_pat().captures_read(&mut self.unbraced_locs, &substr)) + { + // Gather the data we need about the match before going any further + // so we don't keep the `substr` borrowed for longer than necessary + let start = m.start(); + let (cap_start, cap_end) = self + .braced_locs + .get(1) + .or_else(|| self.unbraced_locs.get(1)) + .unwrap(); + + let symbol = substr.substr(cap_start..cap_end); + let match_len = m.len(); + + // Slice off the text before the match as verbatim + if let Some((before, before_span)) = split_at(&substr, span, start) { + self.buffer.push_back(Verbatim(before, before_span)); + + // Buffalo buffalo Buffalo buffalo... + substr = substr.substr(start..); + span.start = before_span.end; + } + + // Substr and Span for the match range + let replaced_slice = substr.substr(..match_len); + let replaced_span = Span { + start: span.start, + end: Position { + line: span.start.line, + character: span.start.character + match_len, + }, + }; + + // Ignore other directives that may be part of the pruned text + if symbol == "import" || symbol == "define_import_path" { + self.buffer + .push_back(Verbatim(replaced_slice, replaced_span)); + } + // Emit an error if the symbol is undefined and push the slice as verbatim + else if !self.defs.contains_key(symbol.as_str()) { + self.errors.push(SpannedError { + message: format!("`{symbol}` is undefined"), + source: self.full_source.clone(), + span: Some(replaced_span), + }); + + self.buffer + .push_back(Verbatim(replaced_slice.clone(), replaced_span)); + } + // Push the #define replaement + else { + self.buffer.push_back(Replacement(SourceReplacement { + original: (replaced_slice, replaced_span), + replacement: self.defs[symbol.as_str()].clone(), + })); + } + + // Advance the "cursor" to the position just past the match + substr = substr.substr(match_len..); + span = Span { + start: replaced_span.end, + end: span.end, + }; + } + + // We're finished processing interpolation matches in this slice + self.index += 1; + + // Handle the remainder of the slice after the last match + if !substr.is_empty() { + if self.buffer.is_empty() { + return Some(Verbatim(substr, span)); + } else { + self.buffer.push_back(Verbatim(substr, span)); + } + } + + self.buffer.pop_front() + } +} + +fn split_at(substr: &Substr, span: Span, idx: usize) -> Option<(Substr, Span)> { + if idx == 0 { + return None; + } + + let before = substr.substr(..idx); + + let mut line_inc = 0_usize; + let mut remaining = before.as_str(); + while let Some(i) = remaining.find('\n') { + line_inc += 1; + remaining = &remaining[i + 1..]; + } + let char_inc = remaining.len(); + + let start = span.start; + let mut end = start; + + end.line += line_inc; + + if line_inc > 0 { + end.character = char_inc; + } else { + end.character += char_inc; + } + + Some((before, Span { start, end })) +} diff --git a/packages/server/src/pre/mapping.rs b/packages/server/src/pre/mapping.rs index 2dd8caad..91a0fbff 100644 --- a/packages/server/src/pre/mapping.rs +++ b/packages/server/src/pre/mapping.rs @@ -16,17 +16,17 @@ impl SourceMap { return Some(needle); } - let offset = compute_offset(dest.start, src.start); + let offset = compute_offset(needle, dest.start, src.start); Some(apply_offset(needle, offset)) } None => { let start_idx = self.find_index_by_dest_pos(needle.start)?; let (dest_start, src_start) = self.0[start_idx]; - let start_offset = compute_offset(dest_start.start, src_start.start); + let start_offset = compute_offset(needle, dest_start.start, src_start.start); let end_idx = self.find_index_by_dest_pos(needle.end)?; let (dest_end, src_end) = self.0[end_idx]; - let end_offset = compute_offset(dest_end.end, src_end.end); + let end_offset = compute_offset(needle, dest_end.end, src_end.end); Some(apply_offsets(needle, start_offset, end_offset)) } @@ -41,17 +41,17 @@ impl SourceMap { return Some(needle); } - let offset = compute_offset(src.start, dest.start); + let offset = compute_offset(needle, src.start, dest.start); Some(apply_offset(needle, offset)) } None => { let start_idx = self.find_index_by_src_pos(needle.start)?; let (dest_start, src_start) = self.0[start_idx]; - let start_offset = compute_offset(src_start.start, dest_start.start); + let start_offset = compute_offset(needle, src_start.start, dest_start.start); let end_idx = self.find_index_by_src_pos(needle.end)?; let (dest_end, src_end) = self.0[end_idx]; - let end_offset = compute_offset(src_end.end, dest_end.end); + let end_offset = compute_offset(needle, src_end.end, dest_end.end); Some(apply_offsets(needle, start_offset, end_offset)) } @@ -103,11 +103,12 @@ impl SourceMap { } } -fn compute_offset(from: Position, to: Position) -> (isize, isize) { +fn compute_offset(needle: Span, from: Position, to: Position) -> (isize, isize) { let line = (to.line as isize) - (from.line as isize); - let character = match line { - 0 => (to.character as isize) - (from.character as isize), - _ => 0, + let character = if line == 0 && needle.start.line == from.line { + (to.character as isize) - (from.character as isize) + } else { + 0 }; (line, character) diff --git a/packages/server/src/pre/mod.rs b/packages/server/src/pre/mod.rs index 670781fd..248afd18 100644 --- a/packages/server/src/pre/mod.rs +++ b/packages/server/src/pre/mod.rs @@ -8,8 +8,9 @@ use parser::{ use crate::pre::pruner::Pruner; -pub use self::mapping::SourceMap; +pub use self::{interpolation::SourceReplacement, mapping::SourceMap}; +mod interpolation; mod interpreter; mod mapping; mod pruner; @@ -20,6 +21,8 @@ pub struct PruneResult { pub source_map: SourceMap, pub inactive_spans: Vec, pub prune_errors: Vec, + pub replacements: Vec, + pub replacement_errors: Vec, } pub fn prune(uri: &Url, source: ArcStr, defs: &HashMap) -> PruneResult { @@ -30,7 +33,7 @@ pub fn prune(uri: &Url, source: ArcStr, defs: &HashMap) -> Prune let mut pruner = Pruner::new(source, defs.clone()); parsed.walk(&mut pruner); - let (pruned, source_map) = pruner.write_output_mapped(); + let (pruned, source_map, replacements, replacement_errors) = pruner.write_output(); eprintln!("Pruned source for {uri}:"); for (idx, line) in pruned.lines().enumerate() { @@ -46,5 +49,7 @@ pub fn prune(uri: &Url, source: ArcStr, defs: &HashMap) -> Prune source_map, inactive_spans: pruner.inactive_spans, prune_errors: pruner.errors, + replacements, + replacement_errors, } } diff --git a/packages/server/src/pre/pruner.rs b/packages/server/src/pre/pruner.rs index d22a2619..fe669e14 100644 --- a/packages/server/src/pre/pruner.rs +++ b/packages/server/src/pre/pruner.rs @@ -14,7 +14,10 @@ use parser::{ use crate::pre::interpreter::{Interpreter, Value}; -use super::mapping::SourceMap; +use super::{ + interpolation::{InterpolationStream, SourceReplacement, SourceSlice}, + mapping::SourceMap, +}; pub struct Pruner { defs: HashMap, @@ -35,73 +38,61 @@ impl Pruner { } } - pub fn _write_output(&self) -> String { + pub fn write_output(&self) -> (String, SourceMap, Vec, Vec) { let mut text = String::new(); - for (idx, (substr, span)) in self.pruned.iter().enumerate() { - let prev = if idx > 0 { - self.pruned[idx - 1].1 - } else { - Span::default() - }; - - if prev.end.line < span.start.line { - for _ in prev.end.line..span.start.line { - text.push('\n'); - } - for _ in 0..span.start.character { - text.push(' '); - } - } else { - for _ in prev.end.character..span.start.character { - text.push(' '); - } - } + let mut map = SourceMap::default(); + let mut cursor = Span::default(); + let mut prev = Span::default(); + let mut replacements = vec![]; + + let mut stream = InterpolationStream::new(self.source.clone(), &self.pruned, &self.defs); + for slice in &mut stream { + match slice { + SourceSlice::Verbatim(substr, span) => { + if span.start.line > prev.end.line { + cursor.start.line += 1; + cursor.start.character = 0; + cursor.end = cursor.start; + writeln!(&mut text).unwrap(); + } - write!(&mut text, "{}", substr).unwrap(); - } + let line_inc = span.end.line - span.start.line; + let char_inc = match line_inc { + 0 => span.end.character - span.start.character, + _ => span.end.character, + }; - text - } + cursor.end.line += line_inc; + if line_inc > 0 { + cursor.end.character = char_inc; + } else { + cursor.end.character += char_inc; + } - pub fn write_output_mapped(&self) -> (String, SourceMap) { - let mut text = String::new(); - let mut map = SourceMap::default(); - let mut cursor = Span::default(); + map.define(cursor, span); + write!(&mut text, "{substr}").unwrap(); - for (idx, (substr, span)) in self.pruned.iter().enumerate() { - let prev = if idx > 0 { - self.pruned[idx - 1].1 - } else { - Span::default() - }; - - if span.start.line > prev.end.line { - cursor.start.line += 1; - cursor.start.character = 0; - cursor.end = cursor.start; - writeln!(&mut text).unwrap(); - } + cursor.start = cursor.end; + prev = span; + } + SourceSlice::Replacement(replacement) => { + replacements.push(replacement.clone()); - let line_inc = span.end.line - span.start.line; - let char_inc = match line_inc { - 0 => span.end.character - span.start.character, - _ => span.end.character, - }; + let (_, span) = replacement.original; + let replacement = replacement.replacement; - cursor.end.line += line_inc; - if line_inc > 0 { - cursor.end.character = char_inc; - } else { - cursor.end.character += char_inc; - } + cursor.end.character += replacement.len(); - map.define(cursor, *span); - write!(&mut text, "{substr}").unwrap(); + map.define(cursor, span); + write!(&mut text, "{replacement}").unwrap(); - cursor.start = cursor.end; + cursor.start = cursor.end; + prev = span; + } + } } - (text, map) + (text, map, replacements, stream.into_inner()) } } diff --git a/packages/server/src/semantic_tokens/mod.rs b/packages/server/src/semantic_tokens/mod.rs index fee9017c..3c81f5cb 100644 --- a/packages/server/src/semantic_tokens/mod.rs +++ b/packages/server/src/semantic_tokens/mod.rs @@ -65,11 +65,9 @@ fn handle_requests( get_semantic_tokens(ast, scopes, source_map) } request::SemanticTokens::Delta(request) => { - let Some(&entity) = r_docs.get(&request.text_document.uri) else { - bail!( - "Failed to get document entity for URI: {}", - &request.text_document.uri - ); + let uri = &request.text_document.uri; + let Some(&entity) = r_docs.get(uri) else { + bail!("Failed to get document entity for URI: {uri}"); }; let (ast, scopes) = q_docs.get(entity)?; let source_map = q_source_maps.get(entity).ok();