From 405e9dcf85134d05bfdca9e0765bdb4b2180db95 Mon Sep 17 00:00:00 2001 From: Ori Ziv Date: Sun, 18 Aug 2024 15:54:20 +0300 Subject: [PATCH] Moved `defs` plugins files to be based on external files. commit-id:9e8383ea --- crates/cairo-lang-compiler/src/db.rs | 10 +++-- crates/cairo-lang-defs/src/db.rs | 44 ++++++++++++++----- crates/cairo-lang-defs/src/ids.rs | 5 +++ crates/cairo-lang-defs/src/test.rs | 10 +++-- crates/cairo-lang-filesystem/src/db.rs | 38 ++++++++-------- crates/cairo-lang-filesystem/src/ids.rs | 6 +-- .../src/lang/db/mod.rs | 9 +++- .../src/lang/lsp/ls_proto_group.rs | 25 +++++------ crates/cairo-lang-lowering/src/test_utils.rs | 9 +++- crates/cairo-lang-plugins/src/test.rs | 10 +++-- crates/cairo-lang-semantic/src/test_utils.rs | 8 +++- .../src/test_utils.rs | 10 +++-- 12 files changed, 120 insertions(+), 64 deletions(-) diff --git a/crates/cairo-lang-compiler/src/db.rs b/crates/cairo-lang-compiler/src/db.rs index 1be808ec563..50adf1cb268 100644 --- a/crates/cairo-lang-compiler/src/db.rs +++ b/crates/cairo-lang-compiler/src/db.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::{anyhow, bail, Result}; -use cairo_lang_defs::db::{DefsDatabase, DefsGroup}; +use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use cairo_lang_defs::plugin::{InlineMacroExprPlugin, MacroPlugin}; use cairo_lang_filesystem::cfg::CfgSet; use cairo_lang_filesystem::db::{ @@ -10,7 +10,7 @@ use cairo_lang_filesystem::db::{ }; use cairo_lang_filesystem::detect::detect_corelib; use cairo_lang_filesystem::flag::Flag; -use cairo_lang_filesystem::ids::{CrateLongId, FlagId}; +use cairo_lang_filesystem::ids::{CrateLongId, FlagId, VirtualFile}; use cairo_lang_lowering::db::{init_lowering_group, LoweringDatabase, LoweringGroup}; use cairo_lang_parser::db::{ParserDatabase, ParserGroup}; use cairo_lang_project::ProjectConfig; @@ -38,7 +38,11 @@ pub struct RootDatabase { storage: salsa::Storage, } impl salsa::Database for RootDatabase {} -impl ExternalFiles for RootDatabase {} +impl ExternalFiles for RootDatabase { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl salsa::ParallelDatabase for RootDatabase { fn snapshot(&self) -> salsa::Snapshot { salsa::Snapshot::new(RootDatabase { storage: self.storage.snapshot() }) diff --git a/crates/cairo-lang-defs/src/db.rs b/crates/cairo-lang-defs/src/db.rs index 2a9b10a54e3..079d24a027c 100644 --- a/crates/cairo-lang-defs/src/db.rs +++ b/crates/cairo-lang-defs/src/db.rs @@ -18,8 +18,9 @@ use cairo_lang_syntax::node::ids::SyntaxStablePtrId; use cairo_lang_syntax::node::{ast, Terminal, TypedStablePtr, TypedSyntaxNode}; use cairo_lang_utils::ordered_hash_map::OrderedHashMap; use cairo_lang_utils::ordered_hash_set::OrderedHashSet; -use cairo_lang_utils::{Intern, Upcast}; +use cairo_lang_utils::{Intern, LookupIntern, Upcast}; use itertools::{chain, Itertools}; +use salsa::InternKey; use crate::ids::*; use crate::plugin::{ @@ -83,6 +84,8 @@ pub trait DefsGroup: fn intern_generic_param(&self, id: GenericParamLongId) -> GenericParamId; #[salsa::interned] fn intern_local_var(&self, id: LocalVarLongId) -> LocalVarId; + #[salsa::interned] + fn intern_ext_file(&self, id: ExtFileLongId) -> ExtFileId; // Plugins. // ======== @@ -389,13 +392,13 @@ pub struct ModuleData { #[derive(Clone, Debug, Eq, PartialEq)] pub struct PrivModuleFileData { /// The files generated by plugins running on items. - pub files: Vec, + files: OrderedHashMap, /// The aux data per such file. - pub aux_data: Vec>, + aux_data: Vec>, /// The items not filtered out by plugins. - pub items: Vec, + items: Vec, /// The diagnostics generated by the plugins. - pub plugin_diagnostics: Vec, + plugin_diagnostics: Vec, } fn priv_module_data(db: &dyn DefsGroup, module_id: ModuleId) -> Maybe { @@ -448,7 +451,7 @@ fn priv_module_data(db: &dyn DefsGroup, module_id: ModuleId) -> Maybe Maybe VirtualFile { + let ExtFileLongId(module_id, stable_ptr, _name) = + ExtFileId::from_intern_id(external_id).lookup_intern(db); + let file_id = FileLongId::External(external_id).intern(db); + let data = db.priv_module_sub_files(module_id, stable_ptr.file_id(db.upcast())).unwrap(); + data.files[&file_id].clone() +} + fn priv_module_sub_files( db: &dyn DefsGroup, module_id: ModuleId, @@ -604,7 +616,7 @@ fn priv_module_sub_files( edition, }; - let mut files = Vec::new(); + let mut files = OrderedHashMap::<_, _>::default(); let mut aux_data = Vec::new(); let mut items = Vec::new(); let mut plugin_diagnostics = Vec::new(); @@ -621,15 +633,25 @@ fn priv_module_sub_files( } if let Some(generated) = result.code { - files.push( - FileLongId::Virtual(VirtualFile { + let generated_file_id = FileLongId::External( + ExtFileLongId( + module_id, + item_ast.stable_ptr().untyped(), + generated.name.clone(), + ) + .intern(db) + .as_intern_id(), + ) + .intern(db); + files.insert( + generated_file_id, + VirtualFile { parent: Some(file_id), name: generated.name, content: generated.content.into(), code_mappings: generated.code_mappings.into(), kind: FileKind::Module, - }) - .intern(db), + }, ); aux_data.push(generated.aux_data); } diff --git a/crates/cairo-lang-defs/src/ids.rs b/crates/cairo-lang-defs/src/ids.rs index 93164efa35a..fee54e73791 100644 --- a/crates/cairo-lang-defs/src/ids.rs +++ b/crates/cairo-lang-defs/src/ids.rs @@ -303,6 +303,11 @@ impl ModuleFileId { } } +/// An id for a file defined out of the filesystem crate. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct ExtFileLongId(pub ModuleId, pub SyntaxStablePtrId, pub SmolStr); +define_short_id!(ExtFileId, ExtFileLongId, DefsGroup, lookup_intern_ext_file, intern_ext_file); + define_language_element_id_as_enum! { #[toplevel] /// Id for direct children of a module. diff --git a/crates/cairo-lang-defs/src/test.rs b/crates/cairo-lang-defs/src/test.rs index 09ad4be70b4..6889de76a51 100644 --- a/crates/cairo-lang-defs/src/test.rs +++ b/crates/cairo-lang-defs/src/test.rs @@ -6,7 +6,7 @@ use cairo_lang_filesystem::db::{ init_files_group, AsFilesGroupMut, CrateConfiguration, ExternalFiles, FilesDatabase, FilesGroup, FilesGroupEx, }; -use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId}; +use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId, VirtualFile}; use cairo_lang_parser::db::{ParserDatabase, ParserGroup}; use cairo_lang_syntax::node::db::{SyntaxDatabase, SyntaxGroup}; use cairo_lang_syntax::node::helpers::QueryAttrs; @@ -17,7 +17,7 @@ use cairo_lang_utils::ordered_hash_map::OrderedHashMap; use cairo_lang_utils::{extract_matches, try_extract_matches, Intern, LookupIntern, Upcast}; use indoc::indoc; -use crate::db::{DefsDatabase, DefsGroup}; +use crate::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use crate::ids::{ FileIndex, GenericParamLongId, ModuleFileId, ModuleId, ModuleItemId, NamedLanguageElementId, SubmoduleLongId, @@ -31,7 +31,11 @@ pub struct DatabaseForTesting { storage: salsa::Storage, } impl salsa::Database for DatabaseForTesting {} -impl ExternalFiles for DatabaseForTesting {} +impl ExternalFiles for DatabaseForTesting { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl Default for DatabaseForTesting { fn default() -> Self { let mut res = Self { storage: Default::default() }; diff --git a/crates/cairo-lang-filesystem/src/db.rs b/crates/cairo-lang-filesystem/src/db.rs index ea602884124..c75a0984aeb 100644 --- a/crates/cairo-lang-filesystem/src/db.rs +++ b/crates/cairo-lang-filesystem/src/db.rs @@ -10,7 +10,8 @@ use serde::{Deserialize, Serialize}; use crate::cfg::CfgSet; use crate::flag::Flag; use crate::ids::{ - CrateId, CrateLongId, Directory, FileId, FileLongId, FlagId, FlagLongId, VirtualFile, + CodeMapping, CrateId, CrateLongId, Directory, FileId, FileLongId, FlagId, FlagLongId, + VirtualFile, }; use crate::span::{FileSummary, TextOffset, TextSpan, TextWidth}; @@ -109,19 +110,9 @@ pub struct ExperimentalFeaturesConfig { /// A trait for defining files external to the `filesystem` crate. pub trait ExternalFiles { - /// Returns the content of an external file. - fn ext_file_content(&self, _external_id: u32) -> Option> { - None - } - - /// Returns the name of an external file. - fn ext_file_name(&self, _external_id: u32) -> String { - "".to_string() - } - - /// Returns the full path of an external file. - fn ext_file_full_path(&self, _external_id: u32) -> String { - "".to_string() + /// Returns the virtual file matching the external id. + fn ext_as_virtual(&self, _external_id: salsa::InternId) -> VirtualFile { + panic!("Should not be called, unless specifically implemented!"); } } @@ -262,7 +253,7 @@ fn priv_raw_file_content(db: &dyn FilesGroup, file: FileId) -> Option> Err(_) => None, }, FileLongId::Virtual(virt) => Some(virt.content), - FileLongId::External(external_id) => db.ext_file_content(external_id), + FileLongId::External(external_id) => Some(db.ext_as_virtual(external_id).content), } } fn file_content(db: &dyn FilesGroup, file: FileId) -> Option> { @@ -291,9 +282,7 @@ pub fn get_originating_location( mut file_id: FileId, mut span: TextSpan, ) -> (FileId, TextSpan) { - while let FileLongId::Virtual(VirtualFile { parent: Some(parent), code_mappings, .. }) = - file_id.lookup_intern(db) - { + while let Some((parent, code_mappings)) = get_parent_and_mapping(db, file_id) { if let Some(origin) = code_mappings.iter().find_map(|mapping| mapping.translate(span)) { span = origin; file_id = parent; @@ -303,3 +292,16 @@ pub fn get_originating_location( } (file_id, span) } + +/// Returns the parent file and the code mappings of the file. +fn get_parent_and_mapping( + db: &dyn FilesGroup, + file_id: FileId, +) -> Option<(FileId, Arc<[CodeMapping]>)> { + let vf = match file_id.lookup_intern(db) { + FileLongId::OnDisk(_) => return None, + FileLongId::Virtual(vf) => vf, + FileLongId::External(id) => db.ext_as_virtual(id), + }; + Some((vf.parent?, vf.code_mappings)) +} diff --git a/crates/cairo-lang-filesystem/src/ids.rs b/crates/cairo-lang-filesystem/src/ids.rs index 746099c59d3..fb02a8d9f5b 100644 --- a/crates/cairo-lang-filesystem/src/ids.rs +++ b/crates/cairo-lang-filesystem/src/ids.rs @@ -62,7 +62,7 @@ impl FlagId { pub enum FileLongId { OnDisk(PathBuf), Virtual(VirtualFile), - External(u32), + External(salsa::InternId), } /// Whether the file holds syntax for a module or for an expression. #[derive(Clone, Debug, Hash, PartialEq, Eq)] @@ -132,14 +132,14 @@ impl<'b> FileId { path.file_name().and_then(|x| x.to_str()).unwrap_or("").to_string() } FileLongId::Virtual(vf) => vf.name.to_string(), - FileLongId::External(external_id) => db.ext_file_name(external_id), + FileLongId::External(external_id) => db.ext_as_virtual(external_id).name.to_string(), } } pub fn full_path(self, db: &dyn FilesGroup) -> String { match self.lookup_intern(db) { FileLongId::OnDisk(path) => path.to_str().unwrap_or("").to_string(), FileLongId::Virtual(vf) => vf.full_path(db), - FileLongId::External(external_id) => db.ext_file_full_path(external_id), + FileLongId::External(external_id) => db.ext_as_virtual(external_id).full_path(db), } } pub fn kind(self, db: &dyn FilesGroup) -> FileKind { diff --git a/crates/cairo-lang-language-server/src/lang/db/mod.rs b/crates/cairo-lang-language-server/src/lang/db/mod.rs index 9fafd1c010a..2a546309062 100644 --- a/crates/cairo-lang-language-server/src/lang/db/mod.rs +++ b/crates/cairo-lang-language-server/src/lang/db/mod.rs @@ -1,9 +1,10 @@ -use cairo_lang_defs::db::{DefsDatabase, DefsGroup}; +use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use cairo_lang_doc::db::DocDatabase; use cairo_lang_filesystem::cfg::{Cfg, CfgSet}; use cairo_lang_filesystem::db::{ init_files_group, AsFilesGroupMut, ExternalFiles, FilesDatabase, FilesGroup, }; +use cairo_lang_filesystem::ids::VirtualFile; use cairo_lang_lowering::db::{init_lowering_group, LoweringDatabase, LoweringGroup}; use cairo_lang_lowering::utils::InliningStrategy; use cairo_lang_parser::db::{ParserDatabase, ParserGroup}; @@ -73,7 +74,11 @@ impl AnalysisDatabase { } impl salsa::Database for AnalysisDatabase {} -impl ExternalFiles for AnalysisDatabase {} +impl ExternalFiles for AnalysisDatabase { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl salsa::ParallelDatabase for AnalysisDatabase { fn snapshot(&self) -> salsa::Snapshot { diff --git a/crates/cairo-lang-language-server/src/lang/lsp/ls_proto_group.rs b/crates/cairo-lang-language-server/src/lang/lsp/ls_proto_group.rs index 7d56d5d1fd0..abd17e18d4f 100644 --- a/crates/cairo-lang-language-server/src/lang/lsp/ls_proto_group.rs +++ b/crates/cairo-lang-language-server/src/lang/lsp/ls_proto_group.rs @@ -42,20 +42,17 @@ pub trait LsProtoGroup: Upcast { /// Get the canonical [`Url`] for a [`FileId`]. fn url_for_file(&self, file_id: FileId) -> Url { - match self.upcast().lookup_intern_file(file_id) { - FileLongId::OnDisk(path) => Url::from_file_path(path).unwrap(), - FileLongId::Virtual(virtual_file) => { - // NOTE: The URL is constructed using setters and path segments in order to - // url-encode any funky characters in parts that LS is not controlling. - let mut url = Url::parse("vfs://").unwrap(); - url.set_host(Some(&file_id.as_intern_id().to_string())).unwrap(); - url.path_segments_mut().unwrap().push(&format!("{}.cairo", virtual_file.name)); - url - } - FileLongId::External(id) => { - unimplemented!("External files are not supported yet: {id}") - } - } + let vf = match self.upcast().lookup_intern_file(file_id) { + FileLongId::OnDisk(path) => return Url::from_file_path(path).unwrap(), + FileLongId::Virtual(vf) => vf, + FileLongId::External(id) => self.upcast().ext_as_virtual(id), + }; + // NOTE: The URL is constructed using setters and path segments in order to + // url-encode any funky characters in parts that LS is not controlling. + let mut url = Url::parse("vfs://").unwrap(); + url.set_host(Some(&file_id.as_intern_id().to_string())).unwrap(); + url.path_segments_mut().unwrap().push(&format!("{}.cairo", vf.name)); + url } } diff --git a/crates/cairo-lang-lowering/src/test_utils.rs b/crates/cairo-lang-lowering/src/test_utils.rs index bcdccb36385..147966f4465 100644 --- a/crates/cairo-lang-lowering/src/test_utils.rs +++ b/crates/cairo-lang-lowering/src/test_utils.rs @@ -1,10 +1,11 @@ use std::sync::{LazyLock, Mutex}; -use cairo_lang_defs::db::{DefsDatabase, DefsGroup}; +use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use cairo_lang_filesystem::db::{ init_dev_corelib, init_files_group, AsFilesGroupMut, ExternalFiles, FilesDatabase, FilesGroup, }; use cairo_lang_filesystem::detect::detect_corelib; +use cairo_lang_filesystem::ids::VirtualFile; use cairo_lang_parser::db::{ParserDatabase, ParserGroup}; use cairo_lang_semantic::db::{SemanticDatabase, SemanticGroup}; use cairo_lang_semantic::inline_macros::get_default_plugin_suite; @@ -26,7 +27,11 @@ pub struct LoweringDatabaseForTesting { storage: salsa::Storage, } impl salsa::Database for LoweringDatabaseForTesting {} -impl ExternalFiles for LoweringDatabaseForTesting {} +impl ExternalFiles for LoweringDatabaseForTesting { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl salsa::ParallelDatabase for LoweringDatabaseForTesting { fn snapshot(&self) -> salsa::Snapshot { salsa::Snapshot::new(LoweringDatabaseForTesting { storage: self.storage.snapshot() }) diff --git a/crates/cairo-lang-plugins/src/test.rs b/crates/cairo-lang-plugins/src/test.rs index 85d7ff2a261..113137eef83 100644 --- a/crates/cairo-lang-plugins/src/test.rs +++ b/crates/cairo-lang-plugins/src/test.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use cairo_lang_defs::db::{DefsDatabase, DefsGroup}; +use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use cairo_lang_defs::ids::ModuleId; use cairo_lang_defs::plugin::{ MacroPlugin, MacroPluginMetadata, PluginDiagnostic, PluginGeneratedFile, PluginResult, @@ -10,7 +10,7 @@ use cairo_lang_filesystem::db::{ init_files_group, AsFilesGroupMut, CrateConfiguration, ExternalFiles, FilesDatabase, FilesGroup, FilesGroupEx, }; -use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId}; +use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId, VirtualFile}; use cairo_lang_parser::db::ParserDatabase; use cairo_lang_syntax::node::ast; use cairo_lang_syntax::node::db::{SyntaxDatabase, SyntaxGroup}; @@ -50,7 +50,11 @@ pub struct DatabaseForTesting { storage: salsa::Storage, } impl salsa::Database for DatabaseForTesting {} -impl ExternalFiles for DatabaseForTesting {} +impl ExternalFiles for DatabaseForTesting { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl Default for DatabaseForTesting { fn default() -> Self { let mut res = Self { storage: Default::default() }; diff --git a/crates/cairo-lang-semantic/src/test_utils.rs b/crates/cairo-lang-semantic/src/test_utils.rs index 1a7cf997713..abbeb7d6514 100644 --- a/crates/cairo-lang-semantic/src/test_utils.rs +++ b/crates/cairo-lang-semantic/src/test_utils.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::sync::{LazyLock, Mutex}; -use cairo_lang_defs::db::{DefsDatabase, DefsGroup}; +use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use cairo_lang_defs::ids::{FunctionWithBodyId, ModuleId, SubmoduleId, SubmoduleLongId}; use cairo_lang_diagnostics::{Diagnostics, DiagnosticsBuilder}; use cairo_lang_filesystem::db::{ @@ -30,7 +30,11 @@ pub struct SemanticDatabaseForTesting { storage: salsa::Storage, } impl salsa::Database for SemanticDatabaseForTesting {} -impl ExternalFiles for SemanticDatabaseForTesting {} +impl ExternalFiles for SemanticDatabaseForTesting { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl salsa::ParallelDatabase for SemanticDatabaseForTesting { fn snapshot(&self) -> salsa::Snapshot { salsa::Snapshot::new(SemanticDatabaseForTesting { storage: self.storage.snapshot() }) diff --git a/crates/cairo-lang-sierra-generator/src/test_utils.rs b/crates/cairo-lang-sierra-generator/src/test_utils.rs index 2fb0b989610..2646507326c 100644 --- a/crates/cairo-lang-sierra-generator/src/test_utils.rs +++ b/crates/cairo-lang-sierra-generator/src/test_utils.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, LazyLock, Mutex}; -use cairo_lang_defs::db::{DefsDatabase, DefsGroup}; +use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup}; use cairo_lang_defs::ids::ModuleId; use cairo_lang_filesystem::db::{ init_dev_corelib, init_files_group, AsFilesGroupMut, ExternalFiles, FilesDatabase, FilesGroup, @@ -8,7 +8,7 @@ use cairo_lang_filesystem::db::{ }; use cairo_lang_filesystem::detect::detect_corelib; use cairo_lang_filesystem::flag::Flag; -use cairo_lang_filesystem::ids::FlagId; +use cairo_lang_filesystem::ids::{FlagId, VirtualFile}; use cairo_lang_lowering::db::{LoweringDatabase, LoweringGroup}; use cairo_lang_parser::db::{ParserDatabase, ParserGroup}; use cairo_lang_semantic::db::{SemanticDatabase, SemanticGroup}; @@ -42,7 +42,11 @@ pub struct SierraGenDatabaseForTesting { storage: salsa::Storage, } impl salsa::Database for SierraGenDatabaseForTesting {} -impl ExternalFiles for SierraGenDatabaseForTesting {} +impl ExternalFiles for SierraGenDatabaseForTesting { + fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile { + ext_as_virtual_impl(self.upcast(), external_id) + } +} impl salsa::ParallelDatabase for SierraGenDatabaseForTesting { fn snapshot(&self) -> salsa::Snapshot { salsa::Snapshot::new(SierraGenDatabaseForTesting { storage: self.storage.snapshot() })