Skip to content

Commit

Permalink
Move concurrency settings to top-level
Browse files Browse the repository at this point in the history
  • Loading branch information
@charliermarsh
charliermarsh committed Aug 19, 2024
1 parent 865e9e0 commit 62e753d
Show file tree
Hide file tree
Showing 7 changed files with 288 additions and 309 deletions.
66 changes: 33 additions & 33 deletions crates/uv-settings/src/settings.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,39 @@ pub struct GlobalOptions {
possible_values = true
)]
pub python_downloads: Option<PythonDownloads>,
/// The maximum number of in-flight concurrent downloads that uv will perform at any given
/// time.
#[option(
default = "50",
value_type = "int",
example = r#"
concurrent-downloads = 4
"#
)]
pub concurrent_downloads: Option<NonZeroUsize>,
/// The maximum number of source distributions that uv will build concurrently at any given
/// time.
///
/// Defaults to the number of available CPU cores.
#[option(
default = "None",
value_type = "int",
example = r#"
concurrent-builds = 4
"#
)]
pub concurrent_builds: Option<NonZeroUsize>,
/// The number of threads used when installing and unzipping packages.
///
/// Defaults to the number of available CPU cores.
#[option(
default = "None",
value_type = "int",
example = r#"
concurrent-installs = 4
"#
)]
pub concurrent_installs: Option<NonZeroUsize>,
}

/// Settings relevant to all installer operations.
Expand Down Expand Up @@ -1170,39 +1203,6 @@ pub struct PipOptions {
"#
)]
pub reinstall_package: Option<Vec<PackageName>>,
/// The maximum number of in-flight concurrent downloads that uv will perform at any given
/// time.
#[option(
default = "50",
value_type = "int",
example = r#"
concurrent-downloads = 4
"#
)]
pub concurrent_downloads: Option<NonZeroUsize>,
/// The maximum number of source distributions that uv will build concurrently at any given
/// time.
///
/// Defaults to the number of available CPU cores.
#[option(
default = "None",
value_type = "int",
example = r#"
concurrent-builds = 4
"#
)]
pub concurrent_builds: Option<NonZeroUsize>,
/// The number of threads used when installing and unzipping packages.
///
/// Defaults to the number of available CPU cores.
#[option(
default = "None",
value_type = "int",
example = r#"
concurrent-installs = 4
"#
)]
pub concurrent_installs: Option<NonZeroUsize>,
}

impl From<ResolverInstallerOptions> for ResolverOptions {
Expand Down
6 changes: 4 additions & 2 deletions crates/uv/src/commands/venv.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ pub(crate) async fn venv(
seed: bool,
allow_existing: bool,
exclude_newer: Option<ExcludeNewer>,
concurrency: Concurrency,
native_tls: bool,
preview: PreviewMode,
cache: &Cache,
Expand All @@ -77,6 +78,7 @@ pub(crate) async fn venv(
python_downloads,
allow_existing,
exclude_newer,
concurrency,
native_tls,
cache,
printer,
Expand Down Expand Up @@ -129,6 +131,7 @@ async fn venv_impl(
python_downloads: PythonDownloads,
allow_existing: bool,
exclude_newer: Option<ExcludeNewer>,
concurrency: Concurrency,
native_tls: bool,
cache: &Cache,
printer: Printer,
Expand Down Expand Up @@ -275,9 +278,8 @@ async fn venv_impl(
// Initialize any shared state.
let state = SharedState::default();

// For seed packages, assume a bunch of default settings and concurrency are sufficient.
// For seed packages, assume a bunch of default settings are sufficient.
let build_constraints = [];
let concurrency = Concurrency::default();
let config_settings = ConfigSettings::default();
let setup_py = SetupPyStrategy::default();
let sources = SourceStrategy::Disabled;
Expand Down
46 changes: 18 additions & 28 deletions crates/uv/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ use uv_cli::{
use uv_cli::{PythonCommand, PythonNamespace, ToolCommand, ToolNamespace};
#[cfg(feature = "self-update")]
use uv_cli::{SelfCommand, SelfNamespace};
use uv_configuration::Concurrency;
use uv_fs::CWD;
use uv_requirements::RequirementsSource;
use uv_scripts::Pep723Script;
Expand Down Expand Up @@ -209,6 +208,11 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
)
}))?;

rayon::ThreadPoolBuilder::new()
.num_threads(globals.concurrency.installs)
.build_global()
.expect("failed to initialize global rayon pool");

debug!("uv {}", version::version());

// Write out any resolved settings.
Expand Down Expand Up @@ -246,11 +250,6 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
let args = PipCompileSettings::resolve(args, filesystem);
show_settings!(args);

rayon::ThreadPoolBuilder::new()
.num_threads(args.settings.concurrency.installs)
.build_global()
.expect("failed to initialize global rayon pool");

// Initialize the cache.
let cache = cache.init()?.with_refresh(
args.refresh
Expand Down Expand Up @@ -323,7 +322,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
args.settings.python,
args.settings.system,
globals.python_preference,
args.settings.concurrency,
globals.concurrency,
globals.native_tls,
globals.quiet,
globals.preview,
Expand All @@ -341,11 +340,6 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
let args = PipSyncSettings::resolve(args, filesystem);
show_settings!(args);

rayon::ThreadPoolBuilder::new()
.num_threads(args.settings.concurrency.installs)
.build_global()
.expect("failed to initialize global rayon pool");

// Initialize the cache.
let cache = cache.init()?.with_refresh(
args.refresh
Expand Down Expand Up @@ -397,7 +391,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
args.settings.target,
args.settings.prefix,
args.settings.sources,
args.settings.concurrency,
globals.concurrency,
globals.native_tls,
globals.preview,
cache,
Expand All @@ -415,11 +409,6 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
let args = PipInstallSettings::resolve(args, filesystem);
show_settings!(args);

rayon::ThreadPoolBuilder::new()
.num_threads(args.settings.concurrency.installs)
.build_global()
.expect("failed to initialize global rayon pool");

// Initialize the cache.
let cache = cache.init()?.with_refresh(
args.refresh
Expand Down Expand Up @@ -490,7 +479,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
args.settings.break_system_packages,
args.settings.target,
args.settings.prefix,
args.settings.concurrency,
globals.concurrency,
globals.native_tls,
globals.preview,
cache,
Expand Down Expand Up @@ -701,6 +690,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
args.seed,
args.allow_existing,
args.settings.exclude_newer,
globals.concurrency,
globals.native_tls,
globals.preview,
&cache,
Expand Down Expand Up @@ -772,7 +762,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
globals.python_preference,
globals.python_downloads,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
cache,
printer,
Expand Down Expand Up @@ -817,7 +807,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
globals.python_preference,
globals.python_downloads,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand Down Expand Up @@ -851,7 +841,7 @@ async fn run(cli: Cli) -> Result<ExitStatus> {
globals.connectivity,
args.args,
args.filesystem,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
globals.preview,
Expand Down Expand Up @@ -1072,7 +1062,7 @@ async fn run_project(
globals.python_preference,
globals.python_downloads,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand Down Expand Up @@ -1104,7 +1094,7 @@ async fn run_project(
args.settings,
globals.preview,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand All @@ -1128,7 +1118,7 @@ async fn run_project(
globals.python_preference,
globals.python_downloads,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand Down Expand Up @@ -1188,7 +1178,7 @@ async fn run_project(
globals.python_downloads,
globals.preview,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand Down Expand Up @@ -1221,7 +1211,7 @@ async fn run_project(
globals.python_downloads,
globals.preview,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand Down Expand Up @@ -1253,7 +1243,7 @@ async fn run_project(
globals.python_downloads,
globals.preview,
globals.connectivity,
Concurrency::default(),
globals.concurrency,
globals.native_tls,
&cache,
printer,
Expand Down
45 changes: 15 additions & 30 deletions crates/uv/src/settings.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ pub(crate) struct GlobalSettings {
pub(crate) verbose: u8,
pub(crate) color: ColorChoice,
pub(crate) native_tls: bool,
pub(crate) concurrency: Concurrency,
pub(crate) connectivity: Connectivity,
pub(crate) show_settings: bool,
pub(crate) preview: PreviewMode,
Expand Down Expand Up @@ -103,6 +104,20 @@ impl GlobalSettings {
native_tls: flag(args.native_tls, args.no_native_tls)
.combine(workspace.and_then(|workspace| workspace.globals.native_tls))
.unwrap_or(false),
concurrency: Concurrency {
downloads: env(env::CONCURRENT_DOWNLOADS)
.combine(workspace.and_then(|workspace| workspace.globals.concurrent_downloads))
.map(NonZeroUsize::get)
.unwrap_or(Concurrency::DEFAULT_DOWNLOADS),
builds: env(env::CONCURRENT_BUILDS)
.combine(workspace.and_then(|workspace| workspace.globals.concurrent_builds))
.map(NonZeroUsize::get)
.unwrap_or_else(Concurrency::threads),
installs: env(env::CONCURRENT_INSTALLS)
.combine(workspace.and_then(|workspace| workspace.globals.concurrent_installs))
.map(NonZeroUsize::get)
.unwrap_or_else(Concurrency::threads),
},
connectivity: if flag(args.offline, args.no_offline)
.combine(workspace.and_then(|workspace| workspace.globals.offline))
.unwrap_or(false)
Expand Down Expand Up @@ -1036,9 +1051,6 @@ impl PipCompileSettings {
emit_marker_expression: flag(emit_marker_expression, no_emit_marker_expression),
emit_index_annotation: flag(emit_index_annotation, no_emit_index_annotation),
annotation_style,
concurrent_builds: env(env::CONCURRENT_BUILDS),
concurrent_downloads: env(env::CONCURRENT_DOWNLOADS),
concurrent_installs: env(env::CONCURRENT_INSTALLS),
..PipOptions::from(resolver)
},
filesystem,
Expand Down Expand Up @@ -1127,9 +1139,6 @@ impl PipSyncSettings {
python_version,
python_platform,
strict: flag(strict, no_strict),
concurrent_builds: env(env::CONCURRENT_BUILDS),
concurrent_downloads: env(env::CONCURRENT_DOWNLOADS),
concurrent_installs: env(env::CONCURRENT_INSTALLS),
..PipOptions::from(installer)
},
filesystem,
Expand Down Expand Up @@ -1264,9 +1273,6 @@ impl PipInstallSettings {
python_platform,
require_hashes: flag(require_hashes, no_require_hashes),
verify_hashes: flag(verify_hashes, no_verify_hashes),
concurrent_builds: env(env::CONCURRENT_BUILDS),
concurrent_downloads: env(env::CONCURRENT_DOWNLOADS),
concurrent_installs: env(env::CONCURRENT_INSTALLS),
..PipOptions::from(installer)
},
filesystem,
Expand Down Expand Up @@ -1869,7 +1875,6 @@ pub(crate) struct PipSettings {
pub(crate) hash_checking: Option<HashCheckingMode>,
pub(crate) upgrade: Upgrade,
pub(crate) reinstall: Reinstall,
pub(crate) concurrency: Concurrency,
}

impl PipSettings {
Expand Down Expand Up @@ -1932,9 +1937,6 @@ impl PipSettings {
upgrade_package,
reinstall,
reinstall_package,
concurrent_builds,
concurrent_downloads,
concurrent_installs,
} = pip.unwrap_or_default();

let ResolverInstallerOptions {
Expand Down Expand Up @@ -2129,23 +2131,6 @@ impl PipSettings {
.combine(reinstall_package)
.unwrap_or_default(),
),
concurrency: Concurrency {
downloads: args
.concurrent_downloads
.combine(concurrent_downloads)
.map(NonZeroUsize::get)
.unwrap_or(Concurrency::DEFAULT_DOWNLOADS),
builds: args
.concurrent_builds
.combine(concurrent_builds)
.map(NonZeroUsize::get)
.unwrap_or_else(Concurrency::threads),
installs: args
.concurrent_installs
.combine(concurrent_installs)
.map(NonZeroUsize::get)
.unwrap_or_else(Concurrency::threads),
},
build_options: BuildOptions::new(
NoBinary::from_pip_args(args.no_binary.combine(no_binary).unwrap_or_default())
.combine(NoBinary::from_args(
Expand Down
Loading

0 comments on commit 62e753d

Please sign in to comment.