Simplify document processing flow

packages/server/src/declarations/mod.rs

@@ -23,7 +23,7 @@ use crate::{
 	dependencies::{CollectRecursive, Dependencies, DependencyMappingSystem, Dependents},
 	documents::{
 		DocumentIntakeSystem, DocumentUri, DocumentsMap, ImportedSymbols, ModulePathsMap,
-		PendingPreprocessing, PendingReads, WgslAst, WgslScopes, WgslSource, WgslTokens,
+		PendingReads, WgslAst, WgslScopes, WgslSource, WgslTokens,
 	},
 	references::TokenReferences,
 	utils::SrcLocation,
@@ -63,15 +63,14 @@ pub struct SymbolResolutionSystem;
 fn initialize_token_maps(
 	mut cmd: Commands,
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	r_documents: Res<DocumentsMap>,
 	r_module_paths: Res<ModulePathsMap>,
 	q_docs: Query<(Entity, &DocumentUri, &WgslTokens), Without<Dependencies>>,
 	q_trees: Query<(&WgslAst, &WgslScopes)>,
 	q_imports: Query<&ImportedSymbols>,
 	mut q_token_refs: Query<&mut TokenReferences>,
 ) {
-	if !r_pending_reads.is_empty() || !r_pending_prepro.is_empty() {
+	if !r_pending_reads.is_empty() {
 		return;
 	}
 
@@ -103,7 +102,6 @@ fn initialize_token_maps(
 fn update_token_maps(
 	mut cmd: Commands,
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	r_documents: Res<DocumentsMap>,
 	r_module_paths: Res<ModulePathsMap>,
 	q_src: Query<Ref<WgslSource>>,
@@ -114,7 +112,7 @@ fn update_token_maps(
 	mut q_token_refs: Query<&mut TokenReferences>,
 	mut l_changed_entities: Local<EntityHashSet>,
 ) {
-	if !r_pending_reads.is_empty() || !r_pending_prepro.is_empty() {
+	if !r_pending_reads.is_empty() {
 		return;
 	}
 

packages/server/src/definition.rs

@@ -14,10 +14,7 @@ use parser::{utils::ToRange, Span};
 
 use crate::{
 	declarations::TokenDeclarations,
-	documents::{
-		DocumentIntakeSystem, DocumentsMap, PendingPreprocessing, PendingReads, WgslSourceMap,
-		WgslTokens,
-	},
+	documents::{DocumentIntakeSystem, DocumentsMap, PendingReads, WgslSourceMap, WgslTokens},
 	ipc::{request, HasRequestId, Ipc},
 	utils::{self, FindBySpan},
 };
@@ -40,11 +37,10 @@ struct PendingEvents(HashMap<RequestId, request::GotoDefinition>);
 
 fn queue_events(
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	mut er_requests: EventReader<request::GotoDefinition>,
 	mut r_pending_events: ResMut<PendingEvents>,
 ) {
-	if !r_pending_reads.is_empty() || !r_pending_prepro.is_empty() {
+	if !r_pending_reads.is_empty() {
 		for request in er_requests.read() {
 			r_pending_events.insert(request.id(), request.clone());
 		}
@@ -54,11 +50,10 @@ fn queue_events(
 
 fn dequeue_events(
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	mut ew_requests: EventWriter<request::GotoDefinition>,
 	mut r_pending_events: ResMut<PendingEvents>,
 ) {
-	if r_pending_reads.is_empty() && r_pending_prepro.is_empty() && !r_pending_events.is_empty() {
+	if r_pending_reads.is_empty() && !r_pending_events.is_empty() {
 		let pending_ids = r_pending_events.keys().cloned().collect_vec();
 		for id in pending_ids {
 			let request = r_pending_events.remove(&id).unwrap();

packages/server/src/dependencies.rs

@@ -14,7 +14,7 @@ use lsp_types::Url;
 
 use crate::{
 	declarations::{SymbolResolutionSystem, TokenDeclarations},
-	documents::{DocumentUri, DocumentsMap, PendingPreprocessing, PendingReads},
+	documents::{DocumentUri, DocumentsMap, PendingReads},
 };
 
 pub struct DependenciesPlugin;
@@ -113,11 +113,10 @@ fn collect_recursive(
 fn map_dependencies(
 	mut cmd: Commands,
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	r_documents: Res<DocumentsMap>,
 	q_docs: Query<(Entity, &DocumentUri, &TokenDeclarations), Changed<TokenDeclarations>>,
 ) {
-	if !r_pending_reads.is_empty() || !r_pending_prepro.is_empty() {
+	if !r_pending_reads.is_empty() {
 		return;
 	}
 
@@ -149,11 +148,10 @@ fn map_dependencies(
 
 fn map_dependents(
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	q_dependencies: Query<(Entity, &Dependencies)>,
 	mut q_dependents: Query<&mut Dependents>,
 ) {
-	if !r_pending_reads.is_empty() || !r_pending_prepro.is_empty() {
+	if !r_pending_reads.is_empty() {
 		return;
 	}
 

packages/server/src/document_symbols.rs

@@ -16,10 +16,7 @@ use parser::{
 };
 
 use crate::{
-	documents::{
-		DocumentIntakeSystem, DocumentsMap, PendingPreprocessing, PendingReads, WgslAst,
-		WgslSourceMap,
-	},
+	documents::{DocumentIntakeSystem, DocumentsMap, PendingReads, WgslAst, WgslSourceMap},
 	ipc::{request, HasRequestId, Ipc},
 	utils,
 };
@@ -42,11 +39,10 @@ struct PendingEvents(HashMap<RequestId, request::DocumentSymbols>);
 
 fn queue_events(
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	mut er_requests: ResMut<Events<request::DocumentSymbols>>,
 	mut r_pending_events: ResMut<PendingEvents>,
 ) {
-	if !r_pending_reads.is_empty() || !r_pending_prepro.is_empty() {
+	if !r_pending_reads.is_empty() {
 		for request in er_requests.drain() {
 			r_pending_events.insert(request.id(), request);
 		}
@@ -55,11 +51,10 @@ fn queue_events(
 
 fn dequeue_events(
 	r_pending_reads: Res<PendingReads>,
-	r_pending_prepro: Res<PendingPreprocessing>,
 	mut ew_requests: EventWriter<request::DocumentSymbols>,
 	mut r_pending_events: ResMut<PendingEvents>,
 ) {
-	if r_pending_reads.is_empty() && r_pending_prepro.is_empty() && !r_pending_events.is_empty() {
+	if r_pending_reads.is_empty() && !r_pending_events.is_empty() {
 		let pending_ids = r_pending_events.keys().cloned().collect_vec();
 		for id in pending_ids {
 			let request = r_pending_events.remove(&id).unwrap();

packages/server/src/documents/ecs_types.rs

@@ -0,0 +1,79 @@
+use std::sync::Arc;
+
+use bevy_derive::{Deref, DerefMut};
+use bevy_ecs::{component::Component, entity::Entity, schedule::SystemSet, system::Resource};
+use bevy_utils::{HashMap, HashSet};
+use gramatika::{SourceStr, SpannedError, Substr};
+use lsp_types::Url;
+use parser::{comment::Comment, scopes::Scope, Span, SyntaxTree, Token};
+use ropey::Rope;
+
+use crate::pre::{SourceMap, SourceReplacement};
+
+// --- Resources ---------------------------------------------------------------
+
+#[derive(Resource, Clone, Debug, Default, Deref, DerefMut)]
+pub struct DocumentsMap(pub HashMap<Url, Entity>);
+
+#[derive(Resource, Clone, Debug, Default, Deref, DerefMut)]
+pub struct ModulePathsMap(pub HashMap<Arc<[Substr]>, Url>);
+
+/// Document URIs that have not yet been read.
+#[derive(Resource, Clone, Debug, Default, Deref, DerefMut)]
+pub struct PendingReads(pub HashSet<Url>);
+
+/// Documents that have been read or updated and require processing
+#[derive(Resource, Clone, Debug, Default, Deref, DerefMut)]
+pub struct PendingDocs(pub HashMap<Url, SourceStr>);
+
+// --- Components --------------------------------------------------------------
+
+#[derive(Component, Clone, Debug, Deref, DerefMut, Hash)]
+pub struct DocumentUri(pub Url);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct WgslSource(pub Substr);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct WgslPrunedSource(pub Substr);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct WgslSourceMap(pub SourceMap);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct WgslTokens(pub Vec<Token>);
+
+#[derive(Component, Debug, Deref, DerefMut)]
+pub struct WgslAst(pub SyntaxTree);
+
+#[derive(Component, Debug, Deref, DerefMut)]
+pub struct WgslComments(pub Vec<Comment>);
+
+#[derive(Component, Debug, Deref, DerefMut)]
+pub struct ParseErrors(pub Vec<SpannedError>);
+
+#[derive(Component, Debug, Deref, DerefMut)]
+pub struct PruneErrors(pub Vec<SpannedError>);
+
+#[derive(Component, Debug, Deref, DerefMut)]
+pub struct SourceReplacements(pub Vec<SourceReplacement>);
+
+#[derive(Component, Debug, Deref, DerefMut)]
+pub struct ReplacementErrors(pub Vec<SpannedError>);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct WgslRope(pub Rope);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct WgslScopes(pub Arc<Scope>);
+
+#[derive(Component, Clone, Debug, Deref, DerefMut)]
+pub struct ImportedSymbols(pub HashMap<Substr, Arc<[Substr]>>);
+
+#[derive(Component, Clone, Debug, Default, Deref, DerefMut)]
+pub struct InactiveRanges(pub Vec<Span>);
+
+// --- Systems -----------------------------------------------------------------
+
+#[derive(SystemSet, Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct DocumentIntakeSystem;