From bf135de3bc451573e9f84703cf9dda2e669bb2a7 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 7 Oct 2022 13:57:32 +1100 Subject: [PATCH 1/7] Clean up rustdoc startup. rustc's startup has several layers, including: - `interface::run_compiler` passes a closure, `f`, to `run_in_thread_pool_with_globals`, which creates a thread pool, sets up session globals, and passes `f` to `create_compiler_and_run`. - `create_compiler_and_run` creates a `Session`, a `Compiler`, sets the source map, and calls `f`. rustdoc is a bit different. - `main_args` calls `main_options` via `run_in_thread_pool_with_globals`, which (again) creates a thread pool (hardcoded to a single thread!) and sets up session globals. - `main_options` has four different paths. - The second one calls `interface::run_compiler`, which redoes the `run_in_thread_pool_with_globals`! This is bad. - The fourth one calls `interface::create_compiler_and_run`, which is reasonable. - The first and third ones don't do anything of note involving the above functions, except for some symbol interning which requires session globals. In other words, rustdoc calls into `rustc_interface` at three different levels. It's a bit confused, and feels like code where functionality has been added by different people at different times without fully understanding how the globally accessible stuff is set up. This commit tidies things up. It removes the `run_in_thread_pool_with_globals` call in `main_args`, and adjust the four paths in `main_options` as follows. - `markdown::test` calls `test::test_main`, which provides its own parallelism and so doesn't need a thread pool. It had one small use of symbol interning, which required session globals, but the commit removes this. - `doctest::run` already calls `interface::run_compiler`, so it doesn't need further adjustment. - `markdown::render` is simple but needs session globals for interning (which can't easily be removed), so it's now wrapped in `create_session_globals_then`. - The fourth path now uses `interface::run_compiler`, which is equivalent to the old `run_in_thread_pool_with_globals` + `create_compiler_and_run` pairing. --- src/librustdoc/doctest.rs | 9 ++++----- src/librustdoc/lib.rs | 14 +++++++------- src/librustdoc/markdown.rs | 5 +++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index f4ec60735a8dd..d9160e8abd4e3 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -18,7 +18,6 @@ use rustc_session::{lint, DiagnosticOutput, Session}; use rustc_span::edition::Edition; use rustc_span::source_map::SourceMap; use rustc_span::symbol::sym; -use rustc_span::Symbol; use rustc_span::{BytePos, FileName, Pos, Span, DUMMY_SP}; use rustc_target::spec::TargetTriple; use tempfile::Builder as TempFileBuilder; @@ -125,7 +124,7 @@ pub(crate) fn run(options: RustdocOptions) -> Result<(), ErrorGuaranteed> { let opts = scrape_test_config(crate_attrs); let enable_per_target_ignores = options.enable_per_target_ignores; let mut collector = Collector::new( - tcx.crate_name(LOCAL_CRATE), + tcx.crate_name(LOCAL_CRATE).to_string(), options, false, opts, @@ -909,7 +908,7 @@ pub(crate) struct Collector { rustdoc_options: RustdocOptions, use_headers: bool, enable_per_target_ignores: bool, - crate_name: Symbol, + crate_name: String, opts: GlobalTestOptions, position: Span, source_map: Option>, @@ -921,7 +920,7 @@ pub(crate) struct Collector { impl Collector { pub(crate) fn new( - crate_name: Symbol, + crate_name: String, rustdoc_options: RustdocOptions, use_headers: bool, opts: GlobalTestOptions, @@ -984,7 +983,7 @@ impl Tester for Collector { fn add_test(&mut self, test: String, config: LangString, line: usize) { let filename = self.get_filename(); let name = self.generate_name(line, &filename); - let crate_name = self.crate_name.to_string(); + let crate_name = self.crate_name.clone(); let opts = self.opts.clone(); let edition = config.edition.unwrap_or(self.rustdoc_options.edition); let rustdoc_options = self.rustdoc_options.clone(); diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index ef01b854e5a69..ce6f7e817c620 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -700,11 +700,7 @@ fn main_args(at_args: &[String]) -> MainResult { }; } }; - rustc_interface::util::run_in_thread_pool_with_globals( - options.edition, - 1, // this runs single-threaded, even in a parallel compiler - move || main_options(options), - ) + main_options(options) } fn wrap_return(diag: &rustc_errors::Handler, res: Result<(), String>) -> MainResult { @@ -749,9 +745,12 @@ fn main_options(options: config::Options) -> MainResult { (true, true) => return wrap_return(&diag, markdown::test(options)), (true, false) => return doctest::run(options), (false, true) => { + // Session globals are required for symbol interning. return wrap_return( &diag, - markdown::render(&options.input, options.render_options, options.edition), + rustc_span::create_session_globals_then(options.edition, || { + markdown::render(&options.input, options.render_options, options.edition) + }), ); } (false, false) => {} @@ -777,9 +776,10 @@ fn main_options(options: config::Options) -> MainResult { let render_options = options.render_options.clone(); let scrape_examples_options = options.scrape_examples_options.clone(); let document_private = options.render_options.document_private; + let config = core::create_config(options); - interface::create_compiler_and_run(config, |compiler| { + interface::run_compiler(config, |compiler| { let sess = compiler.session(); if sess.opts.describe_lints { diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs index 0b557ef244e94..eb64ac455dc4c 100644 --- a/src/librustdoc/markdown.rs +++ b/src/librustdoc/markdown.rs @@ -5,7 +5,6 @@ use std::path::Path; use rustc_span::edition::Edition; use rustc_span::source_map::DUMMY_SP; -use rustc_span::Symbol; use crate::config::{Options, RenderOptions}; use crate::doctest::{Collector, GlobalTestOptions}; @@ -36,6 +35,8 @@ fn extract_leading_metadata(s: &str) -> (Vec<&str>, &str) { /// Render `input` (e.g., "foo.md") into an HTML file in `output` /// (e.g., output = "bar" => "bar/foo.html"). +/// +/// Requires session globals to be available, for symbol interning. pub(crate) fn render>( input: P, options: RenderOptions, @@ -133,7 +134,7 @@ pub(crate) fn test(options: Options) -> Result<(), String> { let mut opts = GlobalTestOptions::default(); opts.no_crate_inject = true; let mut collector = Collector::new( - Symbol::intern(&options.input.display().to_string()), + options.input.display().to_string(), options.clone(), true, opts, From c461f3a760f4a7eeef82bd1a9389d68210abaf4a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 7 Oct 2022 15:36:44 +1100 Subject: [PATCH 2/7] Merge `main_options` into `main_args`. There is no longer any need for them to be separate. --- src/librustdoc/lib.rs | 57 ++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index ce6f7e817c620..7e0cc668d7b59 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -674,35 +674,6 @@ fn usage(argv0: &str) { /// A result type used by several functions under `main()`. type MainResult = Result<(), ErrorGuaranteed>; -fn main_args(at_args: &[String]) -> MainResult { - let args = rustc_driver::args::arg_expand_all(at_args); - - let mut options = getopts::Options::new(); - for option in opts() { - (option.apply)(&mut options); - } - let matches = match options.parse(&args[1..]) { - Ok(m) => m, - Err(err) => { - early_error(ErrorOutputType::default(), &err.to_string()); - } - }; - - // Note that we discard any distinction between different non-zero exit - // codes from `from_matches` here. - let options = match config::Options::from_matches(&matches, args) { - Ok(opts) => opts, - Err(code) => { - return if code == 0 { - Ok(()) - } else { - Err(ErrorGuaranteed::unchecked_claim_error_was_emitted()) - }; - } - }; - main_options(options) -} - fn wrap_return(diag: &rustc_errors::Handler, res: Result<(), String>) -> MainResult { match res { Ok(()) => Ok(()), @@ -733,7 +704,33 @@ fn run_renderer<'tcx, T: formats::FormatRenderer<'tcx>>( } } -fn main_options(options: config::Options) -> MainResult { +fn main_args(at_args: &[String]) -> MainResult { + let args = rustc_driver::args::arg_expand_all(at_args); + + let mut options = getopts::Options::new(); + for option in opts() { + (option.apply)(&mut options); + } + let matches = match options.parse(&args[1..]) { + Ok(m) => m, + Err(err) => { + early_error(ErrorOutputType::default(), &err.to_string()); + } + }; + + // Note that we discard any distinction between different non-zero exit + // codes from `from_matches` here. + let options = match config::Options::from_matches(&matches, args) { + Ok(opts) => opts, + Err(code) => { + return if code == 0 { + Ok(()) + } else { + Err(ErrorGuaranteed::unchecked_claim_error_was_emitted()) + }; + } + }; + let diag = core::new_handler( options.error_format, None, From d156a9092738b3f1fb5e665f532613ab06380162 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 7 Oct 2022 11:57:25 +1100 Subject: [PATCH 3/7] Inline and remove `scoped_thread`. It has a single call site, and removing it slightly improves the confusing tangle of nested closures present at startup. --- compiler/rustc_interface/src/util.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index f7e70d355cf86..c07bfb9c23e4a 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -132,33 +132,27 @@ fn get_stack_size() -> Option { env::var_os("RUST_MIN_STACK").is_none().then_some(STACK_SIZE) } -/// Like a `thread::Builder::spawn` followed by a `join()`, but avoids the need -/// for `'static` bounds. -#[cfg(not(parallel_compiler))] -fn scoped_thread R + Send, R: Send>(cfg: thread::Builder, f: F) -> R { - // SAFETY: join() is called immediately, so any closure captures are still - // alive. - match unsafe { cfg.spawn_unchecked(f) }.unwrap().join() { - Ok(v) => v, - Err(e) => panic::resume_unwind(e), - } -} - #[cfg(not(parallel_compiler))] pub fn run_in_thread_pool_with_globals R + Send, R: Send>( edition: Edition, _threads: usize, f: F, ) -> R { + // The thread pool is a single thread in the non-parallel compiler. let mut cfg = thread::Builder::new().name("rustc".to_string()); - if let Some(size) = get_stack_size() { cfg = cfg.stack_size(size); } - let main_handler = move || rustc_span::create_session_globals_then(edition, f); + let f = move || rustc_span::create_session_globals_then(edition, f); - scoped_thread(cfg, main_handler) + // This avoids the need for `'static` bounds. + // + // SAFETY: join() is called immediately, so any closure captures are still alive. + match unsafe { cfg.spawn_unchecked(f) }.unwrap().join() { + Ok(v) => v, + Err(e) => panic::resume_unwind(e), + } } /// Creates a new thread and forwards information in thread locals to it. From 226387a4039082b2df687c645acb4bde08f2939d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 7 Oct 2022 15:35:42 +1100 Subject: [PATCH 4/7] Reduce visibility of some functions. --- compiler/rustc_interface/src/interface.rs | 2 +- compiler/rustc_interface/src/util.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 949bd02ad6839..d0555f3db0c9b 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -276,7 +276,7 @@ pub struct Config { pub registry: Registry, } -pub fn create_compiler_and_run(config: Config, f: impl FnOnce(&Compiler) -> R) -> R { +fn create_compiler_and_run(config: Config, f: impl FnOnce(&Compiler) -> R) -> R { crate::callbacks::setup_callbacks(); let registry = &config.registry; diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index c07bfb9c23e4a..7ec0d4d73f7a4 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -133,7 +133,7 @@ fn get_stack_size() -> Option { } #[cfg(not(parallel_compiler))] -pub fn run_in_thread_pool_with_globals R + Send, R: Send>( +pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( edition: Edition, _threads: usize, f: F, @@ -171,7 +171,7 @@ unsafe fn handle_deadlock() { } #[cfg(parallel_compiler)] -pub fn run_in_thread_pool_with_globals R + Send, R: Send>( +pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( edition: Edition, threads: usize, f: F, From c00937f2d9b2ea994e5841c5f6047a603eeffe85 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 7 Oct 2022 16:17:57 +1100 Subject: [PATCH 5/7] Inline and remove `create_compiler_and_run`. It has a single call site. --- compiler/rustc_interface/src/interface.rs | 106 +++++++++++----------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index d0555f3db0c9b..bee2e91c74730 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -276,59 +276,6 @@ pub struct Config { pub registry: Registry, } -fn create_compiler_and_run(config: Config, f: impl FnOnce(&Compiler) -> R) -> R { - crate::callbacks::setup_callbacks(); - - let registry = &config.registry; - let (mut sess, codegen_backend) = util::create_session( - config.opts, - config.crate_cfg, - config.crate_check_cfg, - config.diagnostic_output, - config.file_loader, - config.input_path.clone(), - config.lint_caps, - config.make_codegen_backend, - registry.clone(), - ); - - if let Some(parse_sess_created) = config.parse_sess_created { - parse_sess_created( - &mut Lrc::get_mut(&mut sess) - .expect("create_session() should never share the returned session") - .parse_sess, - ); - } - - let temps_dir = sess.opts.unstable_opts.temps_dir.as_ref().map(|o| PathBuf::from(&o)); - - let compiler = Compiler { - sess, - codegen_backend, - input: config.input, - input_path: config.input_path, - output_dir: config.output_dir, - output_file: config.output_file, - temps_dir, - register_lints: config.register_lints, - override_queries: config.override_queries, - }; - - rustc_span::with_source_map(compiler.sess.parse_sess.clone_source_map(), move || { - let r = { - let _sess_abort_error = OnDrop(|| { - compiler.sess.finish_diagnostics(registry); - }); - - f(&compiler) - }; - - let prof = compiler.sess.prof.clone(); - prof.generic_activity("drop_compiler").run(move || drop(compiler)); - r - }) -} - // JUSTIFICATION: before session exists, only config #[allow(rustc::bad_opt_access)] pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Send) -> R { @@ -336,7 +283,58 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se util::run_in_thread_pool_with_globals( config.opts.edition, config.opts.unstable_opts.threads, - || create_compiler_and_run(config, f), + || { + crate::callbacks::setup_callbacks(); + + let registry = &config.registry; + let (mut sess, codegen_backend) = util::create_session( + config.opts, + config.crate_cfg, + config.crate_check_cfg, + config.diagnostic_output, + config.file_loader, + config.input_path.clone(), + config.lint_caps, + config.make_codegen_backend, + registry.clone(), + ); + + if let Some(parse_sess_created) = config.parse_sess_created { + parse_sess_created( + &mut Lrc::get_mut(&mut sess) + .expect("create_session() should never share the returned session") + .parse_sess, + ); + } + + let temps_dir = sess.opts.unstable_opts.temps_dir.as_ref().map(|o| PathBuf::from(&o)); + + let compiler = Compiler { + sess, + codegen_backend, + input: config.input, + input_path: config.input_path, + output_dir: config.output_dir, + output_file: config.output_file, + temps_dir, + register_lints: config.register_lints, + override_queries: config.override_queries, + }; + + rustc_span::with_source_map(compiler.sess.parse_sess.clone_source_map(), move || { + let r = { + let _sess_abort_error = OnDrop(|| { + compiler.sess.finish_diagnostics(registry); + }); + + f(&compiler) + }; + + let prof = compiler.sess.prof.clone(); + prof.generic_activity("drop_compiler").run(move || drop(compiler)); + r + }) + }, ) } From 806701607285b6a61c6169797b5e22ef9a18d5de Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 7 Oct 2022 16:20:20 +1100 Subject: [PATCH 6/7] Apply `Lrc` later to `sess` and `codegen_backend`. This avoids the need for a degenerate `Lrc::get_mut` call. --- compiler/rustc_interface/src/interface.rs | 10 +++------- compiler/rustc_interface/src/util.rs | 5 ++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index bee2e91c74730..a697f8071b4b3 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -300,18 +300,14 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se ); if let Some(parse_sess_created) = config.parse_sess_created { - parse_sess_created( - &mut Lrc::get_mut(&mut sess) - .expect("create_session() should never share the returned session") - .parse_sess, - ); + parse_sess_created(&mut sess.parse_sess); } let temps_dir = sess.opts.unstable_opts.temps_dir.as_ref().map(|o| PathBuf::from(&o)); let compiler = Compiler { - sess, - codegen_backend, + sess: Lrc::new(sess), + codegen_backend: Lrc::new(codegen_backend), input: config.input, input_path: config.input_path, output_dir: config.output_dir, diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 7ec0d4d73f7a4..ae7a240d81cb4 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -5,7 +5,6 @@ use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; #[cfg(parallel_compiler)] use rustc_data_structures::jobserver; -use rustc_data_structures::sync::Lrc; use rustc_errors::registry::Registry; #[cfg(parallel_compiler)] use rustc_middle::ty::tls; @@ -73,7 +72,7 @@ pub fn create_session( Box Box + Send>, >, descriptions: Registry, -) -> (Lrc, Lrc>) { +) -> (Session, Box) { let codegen_backend = if let Some(make_codegen_backend) = make_codegen_backend { make_codegen_backend(&sopts) } else { @@ -121,7 +120,7 @@ pub fn create_session( sess.parse_sess.config = cfg; sess.parse_sess.check_config = check_cfg; - (Lrc::new(sess), Lrc::new(codegen_backend)) + (sess, codegen_backend) } const STACK_SIZE: usize = 8 * 1024 * 1024; From 1f0463a7b5ed89cedf25842e7c2917efc9200939 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sat, 8 Oct 2022 07:43:15 +1100 Subject: [PATCH 7/7] Replace a `spawn_unchecked` with `spawn_scoped`. --- compiler/rustc_interface/src/util.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index ae7a240d81cb4..1bbfb9a415699 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -138,20 +138,24 @@ pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( f: F, ) -> R { // The thread pool is a single thread in the non-parallel compiler. - let mut cfg = thread::Builder::new().name("rustc".to_string()); - if let Some(size) = get_stack_size() { - cfg = cfg.stack_size(size); - } + thread::scope(|s| { + let mut builder = thread::Builder::new().name("rustc".to_string()); + if let Some(size) = get_stack_size() { + builder = builder.stack_size(size); + } - let f = move || rustc_span::create_session_globals_then(edition, f); + // `unwrap` is ok here because `spawn_scoped` only panics if the thread + // name contains null bytes. + let r = builder + .spawn_scoped(s, move || rustc_span::create_session_globals_then(edition, f)) + .unwrap() + .join(); - // This avoids the need for `'static` bounds. - // - // SAFETY: join() is called immediately, so any closure captures are still alive. - match unsafe { cfg.spawn_unchecked(f) }.unwrap().join() { - Ok(v) => v, - Err(e) => panic::resume_unwind(e), - } + match r { + Ok(v) => v, + Err(e) => panic::resume_unwind(e), + } + }) } /// Creates a new thread and forwards information in thread locals to it.