From 9a25c446e7391f65086934a877f75b43c502698c Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 14 Jun 2024 10:45:17 -0400 Subject: [PATCH] Move concurrency settings to top-level --- crates/uv-settings/src/settings.rs | 6 +- crates/uv/src/commands/venv.rs | 6 +- crates/uv/src/main.rs | 40 +++----- crates/uv/src/settings.rs | 45 +++----- crates/uv/tests/show_settings.rs | 160 ++++++++++++++--------------- uv.schema.json | 48 ++++----- 6 files changed, 141 insertions(+), 164 deletions(-) diff --git a/crates/uv-settings/src/settings.rs b/crates/uv-settings/src/settings.rs index f8c2e1224827..dc3e57301473 100644 --- a/crates/uv-settings/src/settings.rs +++ b/crates/uv-settings/src/settings.rs @@ -58,6 +58,9 @@ pub struct GlobalOptions { pub offline: Option, pub no_cache: Option, pub cache_dir: Option, + pub concurrent_downloads: Option, + pub concurrent_builds: Option, + pub concurrent_installs: Option, pub preview: Option, } @@ -189,7 +192,4 @@ pub struct PipOptions { pub upgrade_package: Option>, pub reinstall: Option, pub reinstall_package: Option>, - pub concurrent_downloads: Option, - pub concurrent_builds: Option, - pub concurrent_installs: Option, } diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index 42305b49b06a..4df010a8be41 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -49,6 +49,7 @@ pub(crate) async fn venv( seed: bool, allow_existing: bool, exclude_newer: Option, + concurrency: Concurrency, native_tls: bool, preview: PreviewMode, cache: &Cache, @@ -68,6 +69,7 @@ pub(crate) async fn venv( preview, allow_existing, exclude_newer, + concurrency, native_tls, cache, printer, @@ -117,6 +119,7 @@ async fn venv_impl( preview: PreviewMode, allow_existing: bool, exclude_newer: Option, + concurrency: Concurrency, native_tls: bool, cache: &Cache, printer: Printer, @@ -206,9 +209,8 @@ async fn venv_impl( // Track in-flight downloads, builds, etc., across resolutions. let in_flight = InFlight::default(); - // For seed packages, assume the default settings and concurrency is sufficient. + // For seed packages, assume the default settings is sufficient. let config_settings = ConfigSettings::default(); - let concurrency = Concurrency::default(); // Do not allow builds let build_options = BuildOptions::new(NoBinary::None, NoBuild::All); diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index a69fd47ef358..baf00db5198d 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -13,7 +13,6 @@ use tracing::{debug, instrument}; use cli::{ToolCommand, ToolNamespace, ToolchainCommand, ToolchainNamespace}; use uv_cache::Cache; -use uv_configuration::Concurrency; use uv_distribution::Workspace; use uv_requirements::RequirementsSource; use uv_settings::Combine; @@ -188,6 +187,11 @@ async fn run() -> Result { ) }))?; + rayon::ThreadPoolBuilder::new() + .num_threads(globals.concurrency.installs) + .build_global() + .expect("failed to initialize global rayon pool"); + debug!("uv {}", version::version()); // Write out any resolved settings. @@ -220,11 +224,6 @@ async fn run() -> Result { let args = PipCompileSettings::resolve(args, filesystem); show_settings!(args); - rayon::ThreadPoolBuilder::new() - .num_threads(args.settings.concurrency.installs) - .build_global() - .expect("failed to initialize global rayon pool"); - // Initialize the cache. let cache = cache.init()?.with_refresh(args.refresh); @@ -280,7 +279,7 @@ async fn run() -> Result { args.settings.link_mode, args.settings.python, args.settings.system, - args.settings.concurrency, + globals.concurrency, globals.native_tls, globals.quiet, globals.preview, @@ -298,11 +297,6 @@ async fn run() -> Result { let args = PipSyncSettings::resolve(args, filesystem); show_settings!(args); - rayon::ThreadPoolBuilder::new() - .num_threads(args.settings.concurrency.installs) - .build_global() - .expect("failed to initialize global rayon pool"); - // Initialize the cache. let cache = cache.init()?.with_refresh(args.refresh); @@ -341,7 +335,7 @@ async fn run() -> Result { args.settings.break_system_packages, args.settings.target, args.settings.prefix, - args.settings.concurrency, + globals.concurrency, globals.native_tls, globals.preview, cache, @@ -359,11 +353,6 @@ async fn run() -> Result { let args = PipInstallSettings::resolve(args, filesystem); show_settings!(args); - rayon::ThreadPoolBuilder::new() - .num_threads(args.settings.concurrency.installs) - .build_global() - .expect("failed to initialize global rayon pool"); - // Initialize the cache. let cache = cache.init()?.with_refresh(args.refresh); let requirements = args @@ -419,7 +408,7 @@ async fn run() -> Result { args.settings.break_system_packages, args.settings.target, args.settings.prefix, - args.settings.concurrency, + globals.concurrency, globals.native_tls, globals.preview, cache, @@ -595,6 +584,7 @@ async fn run() -> Result { args.seed, args.allow_existing, args.settings.exclude_newer, + globals.concurrency, globals.native_tls, globals.preview, &cache, @@ -628,7 +618,7 @@ async fn run() -> Result { globals.isolated, globals.preview, globals.connectivity, - Concurrency::default(), + globals.concurrency, globals.native_tls, &cache, printer, @@ -650,7 +640,7 @@ async fn run() -> Result { args.settings, globals.preview, globals.connectivity, - Concurrency::default(), + globals.concurrency, globals.native_tls, &cache, printer, @@ -670,7 +660,7 @@ async fn run() -> Result { args.settings, globals.preview, globals.connectivity, - Concurrency::default(), + globals.concurrency, globals.native_tls, &cache, printer, @@ -690,7 +680,7 @@ async fn run() -> Result { args.python, globals.preview, globals.connectivity, - Concurrency::default(), + globals.concurrency, globals.native_tls, &cache, printer, @@ -710,7 +700,7 @@ async fn run() -> Result { args.python, globals.preview, globals.connectivity, - Concurrency::default(), + globals.concurrency, globals.native_tls, &cache, printer, @@ -749,7 +739,7 @@ async fn run() -> Result { globals.isolated, globals.preview, globals.connectivity, - Concurrency::default(), + globals.concurrency, globals.native_tls, &cache, printer, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index 1a432c10b5b5..7c6ee9ee0dea 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -43,6 +43,7 @@ pub(crate) struct GlobalSettings { pub(crate) connectivity: Connectivity, pub(crate) isolated: bool, pub(crate) show_settings: bool, + pub(crate) concurrency: Concurrency, pub(crate) preview: PreviewMode, } @@ -82,6 +83,20 @@ impl GlobalSettings { }, isolated: args.isolated, show_settings: args.show_settings, + concurrency: Concurrency { + downloads: env(env::CONCURRENT_DOWNLOADS) + .combine(workspace.and_then(|workspace| workspace.globals.concurrent_downloads)) + .map(NonZeroUsize::get) + .unwrap_or(Concurrency::DEFAULT_DOWNLOADS), + builds: env(env::CONCURRENT_BUILDS) + .combine(workspace.and_then(|workspace| workspace.globals.concurrent_builds)) + .map(NonZeroUsize::get) + .unwrap_or_else(Concurrency::threads), + installs: env(env::CONCURRENT_INSTALLS) + .combine(workspace.and_then(|workspace| workspace.globals.concurrent_installs)) + .map(NonZeroUsize::get) + .unwrap_or_else(Concurrency::threads), + }, preview: PreviewMode::from( flag(args.preview, args.no_preview) .combine(workspace.and_then(|workspace| workspace.globals.preview)) @@ -500,9 +515,6 @@ impl PipCompileSettings { emit_marker_expression: flag(emit_marker_expression, no_emit_marker_expression), emit_index_annotation: flag(emit_index_annotation, no_emit_index_annotation), annotation_style, - concurrent_builds: env(env::CONCURRENT_BUILDS), - concurrent_downloads: env(env::CONCURRENT_DOWNLOADS), - concurrent_installs: env(env::CONCURRENT_INSTALLS), ..PipOptions::from(resolver) }, filesystem, @@ -581,9 +593,6 @@ impl PipSyncSettings { python_version, python_platform, require_hashes: flag(require_hashes, no_require_hashes), - concurrent_builds: env(env::CONCURRENT_BUILDS), - concurrent_downloads: env(env::CONCURRENT_DOWNLOADS), - concurrent_installs: env(env::CONCURRENT_INSTALLS), ..PipOptions::from(installer) }, filesystem, @@ -693,9 +702,6 @@ impl PipInstallSettings { python_version, python_platform, require_hashes: flag(require_hashes, no_require_hashes), - concurrent_builds: env(env::CONCURRENT_BUILDS), - concurrent_downloads: env(env::CONCURRENT_DOWNLOADS), - concurrent_installs: env(env::CONCURRENT_INSTALLS), ..PipOptions::from(installer) }, filesystem, @@ -1296,7 +1302,6 @@ pub(crate) struct PipSettings { pub(crate) require_hashes: bool, pub(crate) upgrade: Upgrade, pub(crate) reinstall: Reinstall, - pub(crate) concurrency: Concurrency, } impl PipSettings { @@ -1352,9 +1357,6 @@ impl PipSettings { upgrade_package, reinstall, reinstall_package, - concurrent_builds, - concurrent_downloads, - concurrent_installs, } = pip.unwrap_or_default(); let ResolverInstallerOptions { @@ -1515,23 +1517,6 @@ impl PipSettings { .combine(reinstall_package) .unwrap_or_default(), ), - concurrency: Concurrency { - downloads: args - .concurrent_downloads - .combine(concurrent_downloads) - .map(NonZeroUsize::get) - .unwrap_or(Concurrency::DEFAULT_DOWNLOADS), - builds: args - .concurrent_builds - .combine(concurrent_builds) - .map(NonZeroUsize::get) - .unwrap_or_else(Concurrency::threads), - installs: args - .concurrent_installs - .combine(concurrent_installs) - .map(NonZeroUsize::get) - .unwrap_or_else(Concurrency::threads), - }, build_options: BuildOptions::new( NoBinary::from_pip_args(args.no_binary.combine(no_binary).unwrap_or_default()) .combine(NoBinary::from_args( diff --git a/crates/uv/tests/show_settings.rs b/crates/uv/tests/show_settings.rs index 8e20e3e29dee..5660682cd521 100644 --- a/crates/uv/tests/show_settings.rs +++ b/crates/uv/tests/show_settings.rs @@ -56,6 +56,11 @@ fn resolve_uv_toml() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -154,11 +159,6 @@ fn resolve_uv_toml() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -182,6 +182,11 @@ fn resolve_uv_toml() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -280,11 +285,6 @@ fn resolve_uv_toml() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -309,6 +309,11 @@ fn resolve_uv_toml() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -407,11 +412,6 @@ fn resolve_uv_toml() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -468,6 +468,11 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -566,11 +571,6 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -596,6 +596,11 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -671,11 +676,6 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -710,6 +710,11 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -808,11 +813,6 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -861,6 +861,11 @@ fn resolve_index_url() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -982,11 +987,6 @@ fn resolve_index_url() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1012,6 +1012,11 @@ fn resolve_index_url() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1155,11 +1160,6 @@ fn resolve_index_url() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1208,6 +1208,11 @@ fn resolve_find_links() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1301,11 +1306,6 @@ fn resolve_find_links() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1353,6 +1353,11 @@ fn resolve_top_level() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1428,11 +1433,6 @@ fn resolve_top_level() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1473,6 +1473,11 @@ fn resolve_top_level() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1593,11 +1598,6 @@ fn resolve_top_level() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1621,6 +1621,11 @@ fn resolve_top_level() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1741,11 +1746,6 @@ fn resolve_top_level() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1793,6 +1793,11 @@ fn resolve_user_configuration() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1868,11 +1873,6 @@ fn resolve_user_configuration() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -1903,6 +1903,11 @@ fn resolve_user_configuration() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -1978,11 +1983,6 @@ fn resolve_user_configuration() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -2013,6 +2013,11 @@ fn resolve_user_configuration() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -2088,11 +2093,6 @@ fn resolve_user_configuration() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } @@ -2125,6 +2125,11 @@ fn resolve_user_configuration() -> anyhow::Result<()> { connectivity: Online, isolated: false, show_settings: true, + concurrency: Concurrency { + downloads: 50, + builds: 16, + installs: 8, + }, preview: Disabled, } CacheSettings { @@ -2200,11 +2205,6 @@ fn resolve_user_configuration() -> anyhow::Result<()> { require_hashes: false, upgrade: None, reinstall: None, - concurrency: Concurrency { - downloads: 50, - builds: 16, - installs: 8, - }, }, } diff --git a/uv.schema.json b/uv.schema.json index ccce817fe9e5..929936097b45 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -16,6 +16,30 @@ "null" ] }, + "concurrent-builds": { + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 1.0 + }, + "concurrent-downloads": { + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 1.0 + }, + "concurrent-installs": { + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 1.0 + }, "config-settings": { "anyOf": [ { @@ -443,30 +467,6 @@ "null" ] }, - "concurrent-builds": { - "type": [ - "integer", - "null" - ], - "format": "uint", - "minimum": 1.0 - }, - "concurrent-downloads": { - "type": [ - "integer", - "null" - ], - "format": "uint", - "minimum": 1.0 - }, - "concurrent-installs": { - "type": [ - "integer", - "null" - ], - "format": "uint", - "minimum": 1.0 - }, "config-settings": { "anyOf": [ {