diff --git a/build/generator.rs b/build/generator.rs index f5c66974..cf7737e9 100644 --- a/build/generator.rs +++ b/build/generator.rs @@ -1,16 +1,17 @@ -use std::ffi::OsStr; -use std::fs::{File, OpenOptions}; -use std::io::{BufRead, BufReader, BufWriter, Write}; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Instant; -use std::{env, fs, io, thread}; +use std::{env, fs, thread}; +use collector::Collector; use opencv_binding_generator::{Generator, IteratorExt}; use super::docs::transfer_bindings_to_docs; -use super::{files_with_extension, files_with_predicate, Library, Result, MODULES, OUT_DIR, SRC_CPP_DIR, SRC_DIR}; +use super::{files_with_predicate, Library, Result, MODULES, OUT_DIR, SRC_CPP_DIR, SRC_DIR}; + +#[path = "generator/collector.rs"] +mod collector; pub struct BindingGenerator { build_script_path: PathBuf, @@ -47,7 +48,8 @@ impl BindingGenerator { self.run(modules, opencv_header_dir, opencv)?; - collect_generated_bindings(modules, &target_module_dir, &manual_dir)?; + let collector = Collector::new(modules, &target_module_dir, &manual_dir, &OUT_DIR); + collector.collect_bindings()?; if let Some(target_docs_dir) = target_docs_dir { if !target_docs_dir.exists() { @@ -82,7 +84,7 @@ impl BindingGenerator { .into_iter() .map(|p| p.to_str().expect("Can't convert additional include dir to UTF-8 string")) .join(","); - let job_server = build_job_server()?; + let job_server = Jobserver::build()?; let start = Instant::now(); eprintln!("=== Generating {} modules", modules.len()); thread::scope(|scope| { @@ -123,233 +125,48 @@ impl BindingGenerator { } } -fn is_type_file(path: &Path, module: &str) -> bool { - path.file_stem().and_then(OsStr::to_str).map_or(false, |stem| { - let mut stem_chars = stem.chars(); - (&mut stem_chars).take(3).all(|c| c.is_ascii_digit()) && // first 3 chars are digits - matches!(stem_chars.next(), Some('-')) && // dash - module.chars().zip(&mut stem_chars).all(|(m, s)| m == s) && // module name - matches!(stem_chars.next(), Some('-')) && // dash - stem.ends_with(".type") // ends with ".type" - }) -} - -fn is_type_externs_file(path: &Path, module: &str) -> bool { - path.file_stem().and_then(OsStr::to_str).map_or(false, |stem| { - let mut stem_chars = stem.chars(); - (&mut stem_chars).take(3).all(|c| c.is_ascii_digit()) && // first 3 chars are digits - matches!(stem_chars.next(), Some('-')) && // dash - module.chars().zip(&mut stem_chars).all(|(m, s)| m == s) && // module name - matches!(stem_chars.next(), Some('-')) && // dash - stem.ends_with(".type.externs") // ends with ".type" - }) -} - -fn copy_indent(mut read: impl BufRead, mut write: impl Write, indent: &str) -> Result<()> { - let mut line = Vec::with_capacity(100); - while read.read_until(b'\n', &mut line)? != 0 { - write.write_all(indent.as_bytes())?; - write.write_all(&line)?; - line.clear(); - } - Ok(()) +pub struct Jobserver { + client: jobserver::Client, + reacquire_token_on_drop: bool, } -fn collect_generated_bindings(modules: &[String], target_module_dir: &Path, manual_dir: &Path) -> Result<()> { - if !target_module_dir.exists() { - fs::create_dir(target_module_dir)?; - } - for path in files_with_extension(target_module_dir, "rs")? { - let _ = fs::remove_file(path); - } - - fn write_has_module(mut write: impl Write, module: &str) -> Result<()> { - Ok(writeln!(write, "#[cfg(ocvrs_has_module_{module})]")?) - } - - fn write_module_include(write: &mut BufWriter, module: &str) -> Result<()> { - // Use include instead of #[path] attribute because rust-analyzer doesn't handle #[path] inside other include! too well: - // https://github.com/twistedfall/opencv-rust/issues/418 - // https://github.com/rust-lang/rust-analyzer/issues/11682 - Ok(writeln!( - write, - r#"include!(concat!(env!("OUT_DIR"), "/opencv/{module}.rs"));"# - )?) - } - - let add_manual = |file: &mut BufWriter, module: &str| -> Result { - if manual_dir.join(format!("{module}.rs")).exists() { - writeln!(file, "pub use crate::manual::{module}::*;")?; - Ok(true) - } else { - Ok(false) - } - }; - - let start = Instant::now(); - let mut hub_rs = BufWriter::new(File::create(target_module_dir.join("hub.rs"))?); - - let mut types_rs = BufWriter::new(File::create(target_module_dir.join("types.rs"))?); - writeln!(types_rs)?; - - let mut sys_rs = BufWriter::new(File::create(target_module_dir.join("sys.rs"))?); - writeln!(sys_rs, "use crate::{{mod_prelude_sys::*, core}};")?; - writeln!(sys_rs)?; - - for module in modules { - // merge multiple *-type.cpp files into a single module_types.hpp - let module_cpp = OUT_DIR.join(format!("{module}.cpp")); - if module_cpp.is_file() { - let module_types_cpp = OUT_DIR.join(format!("{module}_types.hpp")); - let mut module_types_file = BufWriter::new( - OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(module_types_cpp)?, - ); - let mut type_files = files_with_extension(&OUT_DIR, "cpp")? - .filter(|f| is_type_file(f, module)) - .collect::>(); - type_files.sort_unstable(); - for entry in type_files { - io::copy(&mut BufReader::new(File::open(&entry)?), &mut module_types_file)?; - let _ = fs::remove_file(entry); - } - } - - // add module entry to hub.rs and move the module file into opencv/ - write_has_module(&mut hub_rs, module)?; - write_module_include(&mut hub_rs, module)?; - let module_filename = format!("{module}.rs"); - let module_src_file = OUT_DIR.join(&module_filename); - let mut module_rs = BufWriter::new(File::create(target_module_dir.join(&module_filename))?); - // Need to wrap modules inside `mod { }` because they have top-level comments (//!) and those don't play well when - // module file is include!d (as opposed to connecting the module with `mod` from the parent module). - // The same doesn't apply to `sys` and `types` below because they don't contain top-level comments. - writeln!(module_rs, "pub mod {module} {{")?; - copy_indent(BufReader::new(File::open(&module_src_file)?), &mut module_rs, "\t")?; - add_manual(&mut module_rs, module)?; - writeln!(module_rs, "}}")?; - let _ = fs::remove_file(module_src_file); - - // merge multiple *-.type.rs files into a single types.rs - let mut header_written = false; - let mut type_files = files_with_extension(&OUT_DIR, "rs")? - .filter(|f| is_type_file(f, module)) - .collect::>(); - type_files.sort_unstable(); - for entry in type_files { - if entry.metadata().map(|meta| meta.len()).unwrap_or(0) > 0 { - if !header_written { - write_has_module(&mut types_rs, module)?; - writeln!(types_rs, "mod {module}_types {{")?; - writeln!(types_rs, "\tuse crate::{{mod_prelude::*, core, types, sys}};")?; - writeln!(types_rs)?; - header_written = true; +impl Jobserver { + pub fn build() -> Result { + unsafe { jobserver::Client::from_env() } + .and_then(|client| { + let own_token_released = client.release_raw().is_ok(); + let available_jobs = client.available().unwrap_or(0); + if available_jobs > 0 { + eprintln!("=== Using environment job server with the the amount of available jobs: {available_jobs}"); + Some(Jobserver { + client, + reacquire_token_on_drop: own_token_released, + }) + } else { + if own_token_released { + client.acquire_raw().expect("Can't reacquire build script thread token"); + } + eprintln!( + "=== Available jobs from the environment created jobserver is: {available_jobs} or there is an error reading that value" + ); + None } - copy_indent(BufReader::new(File::open(&entry)?), &mut types_rs, "\t")?; - } - let _ = fs::remove_file(entry); - } - if header_written { - writeln!(types_rs, "}}")?; - write_has_module(&mut types_rs, module)?; - writeln!(types_rs, "pub use {module}_types::*;")?; - writeln!(types_rs)?; - } - - // merge module-specific *.externs.rs and generated type-specific *.type.externs.rs into a single sys.rs - let externs_rs = OUT_DIR.join(format!("{module}.externs.rs")); - write_has_module(&mut sys_rs, module)?; - writeln!(sys_rs, "mod {module}_sys {{")?; - writeln!(sys_rs, "\tuse super::*;")?; - writeln!(sys_rs)?; - writeln!(sys_rs, "\textern \"C\" {{")?; - copy_indent(BufReader::new(File::open(&externs_rs)?), &mut sys_rs, "\t\t")?; - let _ = fs::remove_file(externs_rs); - let mut type_extern_files = files_with_extension(&OUT_DIR, "rs")? - .filter(|f| is_type_externs_file(f, module)) - .collect::>(); - type_extern_files.sort_unstable(); - for entry in type_extern_files { - if entry.metadata().map(|meta| meta.len()).unwrap_or(0) > 0 { - copy_indent(BufReader::new(File::open(&entry)?), &mut sys_rs, "\t\t")?; - } - let _ = fs::remove_file(entry); - } - writeln!(sys_rs, "\t}}")?; - writeln!(sys_rs, "}}")?; - write_has_module(&mut sys_rs, module)?; - writeln!(sys_rs, "pub use {module}_sys::*;")?; - writeln!(sys_rs)?; - } - writeln!(hub_rs, "pub mod types {{")?; - write!(hub_rs, "\t")?; - write_module_include(&mut hub_rs, "types")?; - writeln!(hub_rs, "}}")?; - writeln!(hub_rs, "#[doc(hidden)]")?; - writeln!(hub_rs, "pub mod sys {{")?; - write!(hub_rs, "\t")?; - write_module_include(&mut hub_rs, "sys")?; - writeln!(hub_rs, "}}")?; - - add_manual(&mut types_rs, "types")?; - - add_manual(&mut sys_rs, "sys")?; - - // write hub_prelude that imports all module-specific preludes - writeln!(hub_rs, "pub mod hub_prelude {{")?; - for module in modules { - write!(hub_rs, "\t")?; - write_has_module(&mut hub_rs, module)?; - writeln!(hub_rs, "\tpub use super::{module}::prelude::*;")?; - } - writeln!(hub_rs, "}}")?; - eprintln!("=== Total binding collection time: {:?}", start.elapsed()); - Ok(()) -} - -fn build_job_server() -> Result { - unsafe { jobserver::Client::from_env() } - .and_then(|client| { - let own_token_released = client.release_raw().is_ok(); - let available_jobs = client.available().unwrap_or(0); - if available_jobs > 0 { - eprintln!("=== Using environment job server with the the amount of available jobs: {available_jobs}"); - Some(Jobserver { + }) + .or_else(|| { + let num_jobs = env::var("NUM_JOBS") + .ok() + .and_then(|jobs| jobs.parse().ok()) + .or_else(|| thread::available_parallelism().map(|p| p.get()).ok()) + .unwrap_or(2) + .max(1); + eprintln!("=== Creating a new job server with num_jobs: {num_jobs}"); + jobserver::Client::new(num_jobs).ok().map(|client| Jobserver { client, - reacquire_token_on_drop: own_token_released, + reacquire_token_on_drop: false, }) - } else { - if own_token_released { - client.acquire_raw().expect("Can't reacquire build script thread token"); - } - eprintln!( - "=== Available jobs from the environment created jobserver is: {available_jobs} or there is an error reading that value" - ); - None - } - }) - .or_else(|| { - let num_jobs = env::var("NUM_JOBS") - .ok() - .and_then(|jobs| jobs.parse().ok()) - .or_else(|| thread::available_parallelism().map(|p| p.get()).ok()) - .unwrap_or(2) - .max(1); - eprintln!("=== Creating a new job server with num_jobs: {num_jobs}"); - jobserver::Client::new(num_jobs).ok().map(|client| Jobserver { - client, - reacquire_token_on_drop: false, }) - }) - .ok_or_else(|| "Can't create job server".into()) -} - -pub struct Jobserver { - client: jobserver::Client, - reacquire_token_on_drop: bool, + .ok_or_else(|| "Can't create job server".into()) + } } impl Drop for Jobserver { diff --git a/build/generator/collector.rs b/build/generator/collector.rs new file mode 100644 index 00000000..5c048e29 --- /dev/null +++ b/build/generator/collector.rs @@ -0,0 +1,219 @@ +use std::ffi::OsStr; +use std::fs::{File, OpenOptions}; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::path::Path; +use std::time::Instant; +use std::{fs, io}; + +use super::super::{files_with_extension, Result}; + +pub struct Collector<'r> { + modules: &'r [String], + target_module_dir: &'r Path, + manual_dir: &'r Path, + out_dir: &'r Path, +} + +impl<'r> Collector<'r> { + pub fn new(modules: &'r [String], target_module_dir: &'r Path, manual_dir: &'r Path, out_dir: &'r Path) -> Self { + Self { + modules, + target_module_dir, + manual_dir, + out_dir, + } + } + + pub fn collect_bindings(&self) -> Result<()> { + if !self.target_module_dir.exists() { + fs::create_dir(self.target_module_dir)?; + } + for path in files_with_extension(self.target_module_dir, "rs")? { + let _ = fs::remove_file(path); + } + + fn write_module_include(write: &mut BufWriter, module: &str) -> Result<()> { + // Use include instead of #[path] attribute because rust-analyzer doesn't handle #[path] inside other include! too well: + // https://github.com/twistedfall/opencv-rust/issues/418 + // https://github.com/rust-lang/rust-analyzer/issues/11682 + Ok(writeln!( + write, + r#"include!(concat!(env!("OUT_DIR"), "/opencv/{module}.rs"));"# + )?) + } + + let start = Instant::now(); + let mut hub_rs = BufWriter::new(File::create(self.target_module_dir.join("hub.rs"))?); + + let mut types_rs = BufWriter::new(File::create(self.target_module_dir.join("types.rs"))?); + writeln!(types_rs)?; + + let mut sys_rs = BufWriter::new(File::create(self.target_module_dir.join("sys.rs"))?); + writeln!(sys_rs, "use crate::{{mod_prelude_sys::*, core}};")?; + writeln!(sys_rs)?; + + for module in self.modules { + // add module entry to hub.rs + write_has_module(&mut hub_rs, module)?; + write_module_include(&mut hub_rs, module)?; + self.collect_module(module, &mut sys_rs, &mut types_rs)? + } + writeln!(hub_rs, "pub mod types {{")?; + write!(hub_rs, "\t")?; + write_module_include(&mut hub_rs, "types")?; + writeln!(hub_rs, "}}")?; + writeln!(hub_rs, "#[doc(hidden)]")?; + writeln!(hub_rs, "pub mod sys {{")?; + write!(hub_rs, "\t")?; + write_module_include(&mut hub_rs, "sys")?; + writeln!(hub_rs, "}}")?; + + self.write_use_manual(&mut types_rs, "types")?; + + self.write_use_manual(&mut sys_rs, "sys")?; + + // write hub_prelude that imports all module-specific preludes + writeln!(hub_rs, "pub mod hub_prelude {{")?; + for module in self.modules { + write!(hub_rs, "\t")?; + write_has_module(&mut hub_rs, module)?; + writeln!(hub_rs, "\tpub use super::{module}::prelude::*;")?; + } + writeln!(hub_rs, "}}")?; + eprintln!("=== Total binding collection time: {:?}", start.elapsed()); + Ok(()) + } + + fn collect_module(&self, module: &str, sys_rs: &mut impl Write, types_rs: &mut impl Write) -> Result<()> { + // merge multiple *-type.cpp files into a single module_types.hpp + let module_cpp = self.out_dir.join(format!("{module}.cpp")); + if module_cpp.is_file() { + let module_types_cpp = self.out_dir.join(format!("{module}_types.hpp")); + let mut module_types_file = BufWriter::new( + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(module_types_cpp)?, + ); + let mut type_files = files_with_extension(self.out_dir, "cpp")? + .filter(|f| is_type_file(f, module)) + .collect::>(); + type_files.sort_unstable(); + for entry in type_files { + io::copy(&mut BufReader::new(File::open(&entry)?), &mut module_types_file)?; + let _ = fs::remove_file(entry); + } + } + + // move the module file into opencv/ + let module_filename = format!("{module}.rs"); + let module_src_file = self.out_dir.join(&module_filename); + let mut module_rs = BufWriter::new(File::create(self.target_module_dir.join(&module_filename))?); + // Need to wrap modules inside `mod { }` because they have top-level comments (//!) and those don't play well when + // module file is include!d (as opposed to connecting the module with `mod` from the parent module). + // The same doesn't apply to `sys` and `types` below because they don't contain top-level comments. + writeln!(module_rs, "pub mod {module} {{")?; + copy_indent(BufReader::new(File::open(&module_src_file)?), &mut module_rs, "\t")?; + self.write_use_manual(&mut module_rs, module)?; + writeln!(module_rs, "}}")?; + let _ = fs::remove_file(module_src_file); + + // merge multiple *-.type.rs files into a single types.rs + let mut header_written = false; + let mut type_files = files_with_extension(self.out_dir, "rs")? + .filter(|f| is_type_file(f, module)) + .collect::>(); + type_files.sort_unstable(); + for entry in type_files { + if entry.metadata().map(|meta| meta.len()).unwrap_or(0) > 0 { + if !header_written { + write_has_module(types_rs, module)?; + writeln!(types_rs, "mod {module}_types {{")?; + writeln!(types_rs, "\tuse crate::{{mod_prelude::*, core, types, sys}};")?; + writeln!(types_rs)?; + header_written = true; + } + copy_indent(BufReader::new(File::open(&entry)?), types_rs, "\t")?; + } + let _ = fs::remove_file(entry); + } + if header_written { + writeln!(types_rs, "}}")?; + write_has_module(types_rs, module)?; + writeln!(types_rs, "pub use {module}_types::*;")?; + writeln!(types_rs)?; + } + + // merge module-specific *.externs.rs and generated type-specific *.type.externs.rs into a single sys.rs + let externs_rs = self.out_dir.join(format!("{module}.externs.rs")); + write_has_module(sys_rs, module)?; + writeln!(sys_rs, "mod {module}_sys {{")?; + writeln!(sys_rs, "\tuse super::*;")?; + writeln!(sys_rs)?; + writeln!(sys_rs, "\textern \"C\" {{")?; + copy_indent(BufReader::new(File::open(&externs_rs)?), sys_rs, "\t\t")?; + let _ = fs::remove_file(externs_rs); + let mut type_extern_files = files_with_extension(self.out_dir, "rs")? + .filter(|f| is_type_externs_file(f, module)) + .collect::>(); + type_extern_files.sort_unstable(); + for entry in type_extern_files { + if entry.metadata().map(|meta| meta.len()).unwrap_or(0) > 0 { + copy_indent(BufReader::new(File::open(&entry)?), sys_rs, "\t\t")?; + } + let _ = fs::remove_file(entry); + } + writeln!(sys_rs, "\t}}")?; + writeln!(sys_rs, "}}")?; + write_has_module(sys_rs, module)?; + writeln!(sys_rs, "pub use {module}_sys::*;")?; + writeln!(sys_rs)?; + Ok(()) + } + + fn write_use_manual(&self, file: &mut BufWriter, module: &str) -> Result { + if self.manual_dir.join(format!("{module}.rs")).exists() { + writeln!(file, "pub use crate::manual::{module}::*;")?; + Ok(true) + } else { + Ok(false) + } + } +} + +fn is_type_file(path: &Path, module: &str) -> bool { + path.file_stem().and_then(OsStr::to_str).map_or(false, |stem| { + let mut stem_chars = stem.chars(); + (&mut stem_chars).take(3).all(|c| c.is_ascii_digit()) && // first 3 chars are digits + matches!(stem_chars.next(), Some('-')) && // dash + module.chars().zip(&mut stem_chars).all(|(m, s)| m == s) && // module name + matches!(stem_chars.next(), Some('-')) && // dash + stem.ends_with(".type") // ends with ".type" + }) +} + +fn is_type_externs_file(path: &Path, module: &str) -> bool { + path.file_stem().and_then(OsStr::to_str).map_or(false, |stem| { + let mut stem_chars = stem.chars(); + (&mut stem_chars).take(3).all(|c| c.is_ascii_digit()) && // first 3 chars are digits + matches!(stem_chars.next(), Some('-')) && // dash + module.chars().zip(&mut stem_chars).all(|(m, s)| m == s) && // module name + matches!(stem_chars.next(), Some('-')) && // dash + stem.ends_with(".type.externs") // ends with ".type" + }) +} + +fn copy_indent(mut read: impl BufRead, write: &mut impl Write, indent: &str) -> Result<()> { + let mut line = Vec::with_capacity(100); + while read.read_until(b'\n', &mut line)? != 0 { + write.write_all(indent.as_bytes())?; + write.write_all(&line)?; + line.clear(); + } + Ok(()) +} + +fn write_has_module(write: &mut impl Write, module: &str) -> Result<()> { + Ok(writeln!(write, "#[cfg(ocvrs_has_module_{module})]")?) +}