Skip to content

Commit

Permalink
Auto merge of rust-lang#17900 - darichey:exclude-vendored-libraries, …
Browse files Browse the repository at this point in the history 
…r=davidbarsky

Add scip/lsif flag to exclude vendored libaries

rust-lang#17809 changed StaticIndex to include vendored libraries. This PR adds a flag to disable that behavior.

At work, our monorepo has too many rust targets to index all at once, so we split them up into several shards. Since all of our libraries are vendored, if rust-analyzer includes them, sharding no longer has much benefit, because every shard will have to index the entire transitive dependency graphs of all of its targets. We get around the issue presented in rust-lang#17809 because some other shard will index the libraries directly.
  • Loading branch information
@bors
bors committed Aug 16, 2024
2 parents 6908451 + bf4d31d commit 995a014
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 19 deletions.
4 changes: 3 additions & 1 deletion src/tools/rust-analyzer/crates/ide/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,9 @@ pub use crate::{
rename::RenameError,
runnables::{Runnable, RunnableKind, TestId},
signature_help::SignatureHelp,
static_index::{StaticIndex, StaticIndexedFile, TokenId, TokenStaticData},
static_index::{
StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, VendoredLibrariesConfig,
},
syntax_highlighting::{
tags::{Highlight, HlMod, HlMods, HlOperator, HlPunct, HlTag},
HighlightConfig, HlRange,
Expand Down
65 changes: 55 additions & 10 deletions src/tools/rust-analyzer/crates/ide/src/static_index.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@ fn documentation_for_definition(
)
}

pub enum VendoredLibrariesConfig<'a> {
Included { workspace_root: &'a VfsPath },
Excluded,
}

impl StaticIndex<'_> {
fn add_file(&mut self, file_id: FileId) {
let current_crate = crates_for(self.db, file_id).pop().map(Into::into);
Expand Down Expand Up @@ -240,15 +245,22 @@ impl StaticIndex<'_> {
self.files.push(result);
}

pub fn compute<'a>(analysis: &'a Analysis, workspace_root: &VfsPath) -> StaticIndex<'a> {
pub fn compute<'a>(
analysis: &'a Analysis,
vendored_libs_config: VendoredLibrariesConfig<'_>,
) -> StaticIndex<'a> {
let db = &*analysis.db;
let work = all_modules(db).into_iter().filter(|module| {
let file_id = module.definition_source_file_id(db).original_file(db);
let source_root = db.file_source_root(file_id.into());
let source_root = db.source_root(source_root);
let is_vendored = source_root
.path_for_file(&file_id.into())
.is_some_and(|module_path| module_path.starts_with(workspace_root));
let is_vendored = match vendored_libs_config {
VendoredLibrariesConfig::Included { workspace_root } => source_root
.path_for_file(&file_id.into())
.is_some_and(|module_path| module_path.starts_with(workspace_root)),
VendoredLibrariesConfig::Excluded => false,
};

!source_root.is_library || is_vendored
});
let mut this = StaticIndex {
Expand Down Expand Up @@ -278,10 +290,11 @@ mod tests {
use ide_db::{base_db::VfsPath, FileRange, FxHashSet};
use syntax::TextSize;

fn check_all_ranges(ra_fixture: &str) {
use super::VendoredLibrariesConfig;

fn check_all_ranges(ra_fixture: &str, vendored_libs_config: VendoredLibrariesConfig<'_>) {
let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture);
let s =
StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
let s = StaticIndex::compute(&analysis, vendored_libs_config);
let mut range_set: FxHashSet<_> = ranges.iter().map(|it| it.0).collect();
for f in s.files {
for (range, _) in f.tokens {
Expand All @@ -298,10 +311,9 @@ mod tests {
}

#[track_caller]
fn check_definitions(ra_fixture: &str) {
fn check_definitions(ra_fixture: &str, vendored_libs_config: VendoredLibrariesConfig<'_>) {
let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture);
let s =
StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
let s = StaticIndex::compute(&analysis, vendored_libs_config);
let mut range_set: FxHashSet<_> = ranges.iter().map(|it| it.0).collect();
for (_, t) in s.tokens.iter() {
if let Some(t) = t.definition {
Expand Down Expand Up @@ -329,6 +341,9 @@ struct Foo;
enum E { X(Foo) }
//^ ^ ^^^
"#,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);
check_definitions(
r#"
Expand All @@ -337,6 +352,9 @@ struct Foo;
enum E { X(Foo) }
//^ ^
"#,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);
}

Expand All @@ -359,6 +377,9 @@ pub func() {
}
"#,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);
}

Expand All @@ -377,9 +398,30 @@ struct ExternalLibrary(i32);
struct VendoredLibrary(i32);
//^^^^^^^^^^^^^^^ ^^^
"#,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);
}

#[test]
fn vendored_crate_excluded() {
check_all_ranges(
r#"
//- /workspace/main.rs crate:main deps:external,vendored
struct Main(i32);
//^^^^ ^^^
//- /external/lib.rs new_source_root:library crate:external@0.1.0,https://a.b/foo.git library
struct ExternalLibrary(i32);
//- /workspace/vendored/lib.rs new_source_root:library crate:vendored@0.1.0,https://a.b/bar.git library
struct VendoredLibrary(i32);
"#,
VendoredLibrariesConfig::Excluded,
)
}

#[test]
fn derives() {
check_all_ranges(
Expand All @@ -394,6 +436,9 @@ pub macro Copy {}
struct Hello(i32);
//^^^^^ ^^^
"#,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);
}
}
9 changes: 9 additions & 0 deletions src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,9 @@ xflags::xflags! {

cmd lsif {
required path: PathBuf

/// Exclude code from vendored libraries from the resulting index.
optional --exclude-vendored-libraries
}

cmd scip {
Expand All @@ -148,6 +151,9 @@ xflags::xflags! {

/// A path to an json configuration file that can be used to customize cargo behavior.
optional --config-path config_path: PathBuf

/// Exclude code from vendored libraries from the resulting index.
optional --exclude-vendored-libraries
}
}
}
Expand Down Expand Up @@ -259,6 +265,8 @@ pub struct Search {
#[derive(Debug)]
pub struct Lsif {
pub path: PathBuf,

pub exclude_vendored_libraries: bool,
}

#[derive(Debug)]
Expand All @@ -267,6 +275,7 @@ pub struct Scip {

pub output: Option<PathBuf>,
pub config_path: Option<PathBuf>,
pub exclude_vendored_libraries: bool,
}

impl RustAnalyzer {
Expand Down
10 changes: 8 additions & 2 deletions src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::time::Instant;

use ide::{
Analysis, AnalysisHost, FileId, FileRange, MonikerKind, PackageInformation, RootDatabase,
StaticIndex, StaticIndexedFile, TokenId, TokenStaticData,
StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, VendoredLibrariesConfig,
};
use ide_db::{line_index::WideEncoding, LineIndexDatabase};
use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
Expand Down Expand Up @@ -296,7 +296,13 @@ impl flags::Lsif {
let db = host.raw_database();
let analysis = host.analysis();

let si = StaticIndex::compute(&analysis, &path.clone().into());
let vendored_libs_config = if self.exclude_vendored_libraries {
VendoredLibrariesConfig::Excluded
} else {
VendoredLibrariesConfig::Included { workspace_root: &path.clone().into() }
};

let si = StaticIndex::compute(&analysis, vendored_libs_config);

let mut lsif = LsifManager::new(&analysis, db, &vfs);
lsif.add_vertex(lsif::Vertex::MetaData(lsif::MetaData {
Expand Down
26 changes: 20 additions & 6 deletions src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::{path::PathBuf, time::Instant};

use ide::{
AnalysisHost, LineCol, MonikerDescriptorKind, MonikerResult, StaticIndex, StaticIndexedFile,
SymbolInformationKind, TextRange, TokenId,
SymbolInformationKind, TextRange, TokenId, VendoredLibrariesConfig,
};
use ide_db::LineIndexDatabase;
use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
Expand Down Expand Up @@ -63,7 +63,13 @@ impl flags::Scip {
let db = host.raw_database();
let analysis = host.analysis();

let si = StaticIndex::compute(&analysis, &root.clone().into());
let vendored_libs_config = if self.exclude_vendored_libraries {
VendoredLibrariesConfig::Excluded
} else {
VendoredLibrariesConfig::Included { workspace_root: &root.clone().into() }
};

let si = StaticIndex::compute(&analysis, vendored_libs_config);

let metadata = scip_types::Metadata {
version: scip_types::ProtocolVersion::UnspecifiedProtocolVersion.into(),
Expand Down Expand Up @@ -352,8 +358,12 @@ mod test {
let (host, position) = position(ra_fixture);

let analysis = host.analysis();
let si =
StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
let si = StaticIndex::compute(
&analysis,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);

let FilePosition { file_id, offset } = position;

Expand Down Expand Up @@ -617,8 +627,12 @@ pub mod example_mod {
host.raw_database_mut().apply_change(change_fixture.change);

let analysis = host.analysis();
let si =
StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
let si = StaticIndex::compute(
&analysis,
VendoredLibrariesConfig::Included {
workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
},
);

let file = si.files.first().unwrap();
let (_, token_id) = file.tokens.first().unwrap();
Expand Down

0 comments on commit 995a014

Please sign in to comment.