Moved defs plugins files to be based on external files.

commit-id:9e8383ea

crates/cairo-lang-compiler/src/db.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use anyhow::{anyhow, bail, Result};
-use cairo_lang_defs::db::{DefsDatabase, DefsGroup};
+use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup};
 use cairo_lang_defs::plugin::{InlineMacroExprPlugin, MacroPlugin};
 use cairo_lang_filesystem::cfg::CfgSet;
 use cairo_lang_filesystem::db::{
@@ -10,7 +10,7 @@ use cairo_lang_filesystem::db::{
 };
 use cairo_lang_filesystem::detect::detect_corelib;
 use cairo_lang_filesystem::flag::Flag;
-use cairo_lang_filesystem::ids::{CrateLongId, FlagId};
+use cairo_lang_filesystem::ids::{CrateLongId, FlagId, VirtualFile};
 use cairo_lang_lowering::db::{init_lowering_group, LoweringDatabase, LoweringGroup};
 use cairo_lang_parser::db::{ParserDatabase, ParserGroup};
 use cairo_lang_project::ProjectConfig;
@@ -38,7 +38,11 @@ pub struct RootDatabase {
     storage: salsa::Storage<RootDatabase>,
 }
 impl salsa::Database for RootDatabase {}
-impl ExternalFiles for RootDatabase {}
+impl ExternalFiles for RootDatabase {
+    fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile {
+        ext_as_virtual_impl(self.upcast(), external_id)
+    }
+}
 impl salsa::ParallelDatabase for RootDatabase {
     fn snapshot(&self) -> salsa::Snapshot<RootDatabase> {
         salsa::Snapshot::new(RootDatabase { storage: self.storage.snapshot() })

crates/cairo-lang-defs/src/db.rs

@@ -18,8 +18,9 @@ use cairo_lang_syntax::node::ids::SyntaxStablePtrId;
 use cairo_lang_syntax::node::{ast, Terminal, TypedStablePtr, TypedSyntaxNode};
 use cairo_lang_utils::ordered_hash_map::OrderedHashMap;
 use cairo_lang_utils::ordered_hash_set::OrderedHashSet;
-use cairo_lang_utils::{Intern, Upcast};
+use cairo_lang_utils::{Intern, LookupIntern, Upcast};
 use itertools::{chain, Itertools};
+use salsa::InternKey;
 
 use crate::ids::*;
 use crate::plugin::{
@@ -83,6 +84,8 @@ pub trait DefsGroup:
     fn intern_generic_param(&self, id: GenericParamLongId) -> GenericParamId;
     #[salsa::interned]
     fn intern_local_var(&self, id: LocalVarLongId) -> LocalVarId;
+    #[salsa::interned]
+    fn intern_ext_file(&self, id: ExtFileLongId) -> ExtFileId;
 
     // Plugins.
     // ========
@@ -389,13 +392,13 @@ pub struct ModuleData {
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct PrivModuleFileData {
     /// The files generated by plugins running on items.
-    pub files: Vec<FileId>,
+    files: OrderedHashMap<FileId, VirtualFile>,
     /// The aux data per such file.
-    pub aux_data: Vec<Option<DynGeneratedFileAuxData>>,
+    aux_data: Vec<Option<DynGeneratedFileAuxData>>,
     /// The items not filtered out by plugins.
-    pub items: Vec<ast::ModuleItem>,
+    items: Vec<ast::ModuleItem>,
     /// The diagnostics generated by the plugins.
-    pub plugin_diagnostics: Vec<PluginDiagnostic>,
+    plugin_diagnostics: Vec<PluginDiagnostic>,
 }
 
 fn priv_module_data(db: &dyn DefsGroup, module_id: ModuleId) -> Maybe<ModuleData> {
@@ -448,7 +451,7 @@ fn priv_module_data(db: &dyn DefsGroup, module_id: ModuleId) -> Maybe<ModuleData
         files.push(file_id);
 
         let priv_module_data = db.priv_module_sub_files(module_id, file_id)?;
-        file_queue.extend(priv_module_data.files.iter().copied());
+        file_queue.extend(priv_module_data.files.keys().copied());
         for diag in &priv_module_data.plugin_diagnostics {
             plugin_diagnostics.push((module_file_id, diag.clone()));
         }
@@ -560,6 +563,15 @@ fn priv_module_data(db: &dyn DefsGroup, module_id: ModuleId) -> Maybe<ModuleData
     Ok(res)
 }
 
+/// Returns the `VirtualFile` matching the given external id.
+pub fn ext_as_virtual_impl(db: &dyn DefsGroup, external_id: salsa::InternId) -> VirtualFile {
+    let ExtFileLongId(module_id, stable_ptr, _name) =
+        ExtFileId::from_intern_id(external_id).lookup_intern(db);
+    let file_id = FileLongId::External(external_id).intern(db);
+    let data = db.priv_module_sub_files(module_id, stable_ptr.file_id(db.upcast())).unwrap();
+    data.files[&file_id].clone()
+}
+
 fn priv_module_sub_files(
     db: &dyn DefsGroup,
     module_id: ModuleId,
@@ -604,7 +616,7 @@ fn priv_module_sub_files(
         edition,
     };
 
-    let mut files = Vec::new();
+    let mut files = OrderedHashMap::<_, _>::default();
     let mut aux_data = Vec::new();
     let mut items = Vec::new();
     let mut plugin_diagnostics = Vec::new();
@@ -621,15 +633,25 @@ fn priv_module_sub_files(
             }
 
             if let Some(generated) = result.code {
-                files.push(
-                    FileLongId::Virtual(VirtualFile {
+                let generated_file_id = FileLongId::External(
+                    ExtFileLongId(
+                        module_id,
+                        item_ast.stable_ptr().untyped(),
+                        generated.name.clone(),
+                    )
+                    .intern(db)
+                    .as_intern_id(),
+                )
+                .intern(db);
+                files.insert(
+                    generated_file_id,
+                    VirtualFile {
                         parent: Some(file_id),
                         name: generated.name,
                         content: generated.content.into(),
                         code_mappings: generated.code_mappings.into(),
                         kind: FileKind::Module,
-                    })
-                    .intern(db),
+                    },
                 );
                 aux_data.push(generated.aux_data);
             }

crates/cairo-lang-defs/src/ids.rs

@@ -303,6 +303,11 @@ impl ModuleFileId {
     }
 }
 
+/// An id for a file defined out of the filesystem crate.
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct ExtFileLongId(pub ModuleId, pub SyntaxStablePtrId, pub SmolStr);
+define_short_id!(ExtFileId, ExtFileLongId, DefsGroup, lookup_intern_ext_file, intern_ext_file);
+
 define_language_element_id_as_enum! {
     #[toplevel]
     /// Id for direct children of a module.

crates/cairo-lang-defs/src/test.rs

@@ -6,7 +6,7 @@ use cairo_lang_filesystem::db::{
     init_files_group, AsFilesGroupMut, CrateConfiguration, ExternalFiles, FilesDatabase,
     FilesGroup, FilesGroupEx,
 };
-use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId};
+use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId, VirtualFile};
 use cairo_lang_parser::db::{ParserDatabase, ParserGroup};
 use cairo_lang_syntax::node::db::{SyntaxDatabase, SyntaxGroup};
 use cairo_lang_syntax::node::helpers::QueryAttrs;
@@ -17,7 +17,7 @@ use cairo_lang_utils::ordered_hash_map::OrderedHashMap;
 use cairo_lang_utils::{extract_matches, try_extract_matches, Intern, LookupIntern, Upcast};
 use indoc::indoc;
 
-use crate::db::{DefsDatabase, DefsGroup};
+use crate::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup};
 use crate::ids::{
     FileIndex, GenericParamLongId, ModuleFileId, ModuleId, ModuleItemId, NamedLanguageElementId,
     SubmoduleLongId,
@@ -31,7 +31,11 @@ pub struct DatabaseForTesting {
     storage: salsa::Storage<DatabaseForTesting>,
 }
 impl salsa::Database for DatabaseForTesting {}
-impl ExternalFiles for DatabaseForTesting {}
+impl ExternalFiles for DatabaseForTesting {
+    fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile {
+        ext_as_virtual_impl(self.upcast(), external_id)
+    }
+}
 impl Default for DatabaseForTesting {
     fn default() -> Self {
         let mut res = Self { storage: Default::default() };

crates/cairo-lang-filesystem/src/db.rs

@@ -10,7 +10,8 @@ use serde::{Deserialize, Serialize};
 use crate::cfg::CfgSet;
 use crate::flag::Flag;
 use crate::ids::{
-    CrateId, CrateLongId, Directory, FileId, FileLongId, FlagId, FlagLongId, VirtualFile,
+    CodeMapping, CrateId, CrateLongId, Directory, FileId, FileLongId, FlagId, FlagLongId,
+    VirtualFile,
 };
 use crate::span::{FileSummary, TextOffset, TextSpan, TextWidth};
 
@@ -109,19 +110,9 @@ pub struct ExperimentalFeaturesConfig {
 
 /// A trait for defining files external to the `filesystem` crate.
 pub trait ExternalFiles {
-    /// Returns the content of an external file.
-    fn ext_file_content(&self, _external_id: u32) -> Option<Arc<str>> {
-        None
-    }
-
-    /// Returns the name of an external file.
-    fn ext_file_name(&self, _external_id: u32) -> String {
-        "<external>".to_string()
-    }
-
-    /// Returns the full path of an external file.
-    fn ext_file_full_path(&self, _external_id: u32) -> String {
-        "<external>".to_string()
+    /// Returns the virtual file matching the external id.
+    fn ext_as_virtual(&self, _external_id: salsa::InternId) -> VirtualFile {
+        panic!("Should not be called, unless specifically implemented!");
     }
 }
 
@@ -262,7 +253,7 @@ fn priv_raw_file_content(db: &dyn FilesGroup, file: FileId) -> Option<Arc<str>>
             Err(_) => None,
         },
         FileLongId::Virtual(virt) => Some(virt.content),
-        FileLongId::External(external_id) => db.ext_file_content(external_id),
+        FileLongId::External(external_id) => Some(db.ext_as_virtual(external_id).content),
     }
 }
 fn file_content(db: &dyn FilesGroup, file: FileId) -> Option<Arc<str>> {
@@ -291,9 +282,7 @@ pub fn get_originating_location(
     mut file_id: FileId,
     mut span: TextSpan,
 ) -> (FileId, TextSpan) {
-    while let FileLongId::Virtual(VirtualFile { parent: Some(parent), code_mappings, .. }) =
-        file_id.lookup_intern(db)
-    {
+    while let Some((parent, code_mappings)) = get_parent_and_mapping(db, file_id) {
         if let Some(origin) = code_mappings.iter().find_map(|mapping| mapping.translate(span)) {
             span = origin;
             file_id = parent;
@@ -303,3 +292,16 @@ pub fn get_originating_location(
     }
     (file_id, span)
 }
+
+/// Returns the parent file and the code mappings of the file.
+fn get_parent_and_mapping(
+    db: &dyn FilesGroup,
+    file_id: FileId,
+) -> Option<(FileId, Arc<[CodeMapping]>)> {
+    let vf = match file_id.lookup_intern(db) {
+        FileLongId::OnDisk(_) => return None,
+        FileLongId::Virtual(vf) => vf,
+        FileLongId::External(id) => db.ext_as_virtual(id),
+    };
+    Some((vf.parent?, vf.code_mappings))
+}

crates/cairo-lang-filesystem/src/ids.rs

@@ -62,7 +62,7 @@ impl FlagId {
 pub enum FileLongId {
     OnDisk(PathBuf),
     Virtual(VirtualFile),
-    External(u32),
+    External(salsa::InternId),
 }
 /// Whether the file holds syntax for a module or for an expression.
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -132,14 +132,14 @@ impl<'b> FileId {
                 path.file_name().and_then(|x| x.to_str()).unwrap_or("<unknown>").to_string()
             }
             FileLongId::Virtual(vf) => vf.name.to_string(),
-            FileLongId::External(external_id) => db.ext_file_name(external_id),
+            FileLongId::External(external_id) => db.ext_as_virtual(external_id).name.to_string(),
         }
     }
     pub fn full_path(self, db: &dyn FilesGroup) -> String {
         match self.lookup_intern(db) {
             FileLongId::OnDisk(path) => path.to_str().unwrap_or("<unknown>").to_string(),
             FileLongId::Virtual(vf) => vf.full_path(db),
-            FileLongId::External(external_id) => db.ext_file_full_path(external_id),
+            FileLongId::External(external_id) => db.ext_as_virtual(external_id).full_path(db),
         }
     }
     pub fn kind(self, db: &dyn FilesGroup) -> FileKind {

crates/cairo-lang-language-server/src/lang/db/mod.rs

@@ -1,9 +1,10 @@
-use cairo_lang_defs::db::{DefsDatabase, DefsGroup};
+use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup};
 use cairo_lang_doc::db::DocDatabase;
 use cairo_lang_filesystem::cfg::{Cfg, CfgSet};
 use cairo_lang_filesystem::db::{
     init_files_group, AsFilesGroupMut, ExternalFiles, FilesDatabase, FilesGroup,
 };
+use cairo_lang_filesystem::ids::VirtualFile;
 use cairo_lang_lowering::db::{init_lowering_group, LoweringDatabase, LoweringGroup};
 use cairo_lang_lowering::utils::InliningStrategy;
 use cairo_lang_parser::db::{ParserDatabase, ParserGroup};
@@ -73,7 +74,11 @@ impl AnalysisDatabase {
 }
 
 impl salsa::Database for AnalysisDatabase {}
-impl ExternalFiles for AnalysisDatabase {}
+impl ExternalFiles for AnalysisDatabase {
+    fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile {
+        ext_as_virtual_impl(self.upcast(), external_id)
+    }
+}
 
 impl salsa::ParallelDatabase for AnalysisDatabase {
     fn snapshot(&self) -> salsa::Snapshot<Self> {

crates/cairo-lang-language-server/src/lang/lsp/ls_proto_group.rs

@@ -42,20 +42,17 @@ pub trait LsProtoGroup: Upcast<dyn FilesGroup> {
 
     /// Get the canonical [`Url`] for a [`FileId`].
     fn url_for_file(&self, file_id: FileId) -> Url {
-        match self.upcast().lookup_intern_file(file_id) {
-            FileLongId::OnDisk(path) => Url::from_file_path(path).unwrap(),
-            FileLongId::Virtual(virtual_file) => {
-                // NOTE: The URL is constructed using setters and path segments in order to
-                //   url-encode any funky characters in parts that LS is not controlling.
-                let mut url = Url::parse("vfs://").unwrap();
-                url.set_host(Some(&file_id.as_intern_id().to_string())).unwrap();
-                url.path_segments_mut().unwrap().push(&format!("{}.cairo", virtual_file.name));
-                url
-            }
-            FileLongId::External(id) => {
-                unimplemented!("External files are not supported yet: {id}")
-            }
-        }
+        let vf = match self.upcast().lookup_intern_file(file_id) {
+            FileLongId::OnDisk(path) => return Url::from_file_path(path).unwrap(),
+            FileLongId::Virtual(vf) => vf,
+            FileLongId::External(id) => self.upcast().ext_as_virtual(id),
+        };
+        // NOTE: The URL is constructed using setters and path segments in order to
+        //   url-encode any funky characters in parts that LS is not controlling.
+        let mut url = Url::parse("vfs://").unwrap();
+        url.set_host(Some(&file_id.as_intern_id().to_string())).unwrap();
+        url.path_segments_mut().unwrap().push(&format!("{}.cairo", vf.name));
+        url
     }
 }
 

crates/cairo-lang-lowering/src/test_utils.rs

@@ -1,10 +1,11 @@
 use std::sync::{LazyLock, Mutex};
 
-use cairo_lang_defs::db::{DefsDatabase, DefsGroup};
+use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup};
 use cairo_lang_filesystem::db::{
     init_dev_corelib, init_files_group, AsFilesGroupMut, ExternalFiles, FilesDatabase, FilesGroup,
 };
 use cairo_lang_filesystem::detect::detect_corelib;
+use cairo_lang_filesystem::ids::VirtualFile;
 use cairo_lang_parser::db::{ParserDatabase, ParserGroup};
 use cairo_lang_semantic::db::{SemanticDatabase, SemanticGroup};
 use cairo_lang_semantic::inline_macros::get_default_plugin_suite;
@@ -26,7 +27,11 @@ pub struct LoweringDatabaseForTesting {
     storage: salsa::Storage<LoweringDatabaseForTesting>,
 }
 impl salsa::Database for LoweringDatabaseForTesting {}
-impl ExternalFiles for LoweringDatabaseForTesting {}
+impl ExternalFiles for LoweringDatabaseForTesting {
+    fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile {
+        ext_as_virtual_impl(self.upcast(), external_id)
+    }
+}
 impl salsa::ParallelDatabase for LoweringDatabaseForTesting {
     fn snapshot(&self) -> salsa::Snapshot<LoweringDatabaseForTesting> {
         salsa::Snapshot::new(LoweringDatabaseForTesting { storage: self.storage.snapshot() })

crates/cairo-lang-plugins/src/test.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use cairo_lang_defs::db::{DefsDatabase, DefsGroup};
+use cairo_lang_defs::db::{ext_as_virtual_impl, DefsDatabase, DefsGroup};
 use cairo_lang_defs::ids::ModuleId;
 use cairo_lang_defs::plugin::{
     MacroPlugin, MacroPluginMetadata, PluginDiagnostic, PluginGeneratedFile, PluginResult,
@@ -10,7 +10,7 @@ use cairo_lang_filesystem::db::{
     init_files_group, AsFilesGroupMut, CrateConfiguration, ExternalFiles, FilesDatabase,
     FilesGroup, FilesGroupEx,
 };
-use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId};
+use cairo_lang_filesystem::ids::{CrateLongId, Directory, FileLongId, VirtualFile};
 use cairo_lang_parser::db::ParserDatabase;
 use cairo_lang_syntax::node::ast;
 use cairo_lang_syntax::node::db::{SyntaxDatabase, SyntaxGroup};
@@ -50,7 +50,11 @@ pub struct DatabaseForTesting {
     storage: salsa::Storage<DatabaseForTesting>,
 }
 impl salsa::Database for DatabaseForTesting {}
-impl ExternalFiles for DatabaseForTesting {}
+impl ExternalFiles for DatabaseForTesting {
+    fn ext_as_virtual(&self, external_id: salsa::InternId) -> VirtualFile {
+        ext_as_virtual_impl(self.upcast(), external_id)
+    }
+}
 impl Default for DatabaseForTesting {
     fn default() -> Self {
         let mut res = Self { storage: Default::default() };