Skip to content

Commit

Permalink
Cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@twistedfall
twistedfall committed Sep 18, 2024
1 parent 19e96ef commit c2feda9
Show file tree
Hide file tree
Showing 2 changed files with 264 additions and 228 deletions.
273 changes: 45 additions & 228 deletions build/generator.rs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
use std::ffi::OsStr;
use std::fs::{File, OpenOptions};
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use std::{env, fs, io, thread};
use std::{env, fs, thread};

use collector::Collector;
use opencv_binding_generator::{Generator, IteratorExt};

use super::docs::transfer_bindings_to_docs;
use super::{files_with_extension, files_with_predicate, Library, Result, MODULES, OUT_DIR, SRC_CPP_DIR, SRC_DIR};
use super::{files_with_predicate, Library, Result, MODULES, OUT_DIR, SRC_CPP_DIR, SRC_DIR};

#[path = "generator/collector.rs"]
mod collector;

pub struct BindingGenerator {
build_script_path: PathBuf,
Expand Down Expand Up @@ -47,7 +48,8 @@ impl BindingGenerator {

self.run(modules, opencv_header_dir, opencv)?;

collect_generated_bindings(modules, &target_module_dir, &manual_dir)?;
let collector = Collector::new(modules, &target_module_dir, &manual_dir, &OUT_DIR);
collector.collect_bindings()?;

if let Some(target_docs_dir) = target_docs_dir {
if !target_docs_dir.exists() {
Expand Down Expand Up @@ -82,7 +84,7 @@ impl BindingGenerator {
.into_iter()
.map(|p| p.to_str().expect("Can't convert additional include dir to UTF-8 string"))
.join(",");
let job_server = build_job_server()?;
let job_server = Jobserver::build()?;
let start = Instant::now();
eprintln!("=== Generating {} modules", modules.len());
thread::scope(|scope| {
Expand Down Expand Up @@ -123,233 +125,48 @@ impl BindingGenerator {
}
}

fn is_type_file(path: &Path, module: &str) -> bool {
path.file_stem().and_then(OsStr::to_str).map_or(false, |stem| {
let mut stem_chars = stem.chars();
(&mut stem_chars).take(3).all(|c| c.is_ascii_digit()) && // first 3 chars are digits
matches!(stem_chars.next(), Some('-')) && // dash
module.chars().zip(&mut stem_chars).all(|(m, s)| m == s) && // module name
matches!(stem_chars.next(), Some('-')) && // dash
stem.ends_with(".type") // ends with ".type"
})
}

fn is_type_externs_file(path: &Path, module: &str) -> bool {
path.file_stem().and_then(OsStr::to_str).map_or(false, |stem| {
let mut stem_chars = stem.chars();
(&mut stem_chars).take(3).all(|c| c.is_ascii_digit()) && // first 3 chars are digits
matches!(stem_chars.next(), Some('-')) && // dash
module.chars().zip(&mut stem_chars).all(|(m, s)| m == s) && // module name
matches!(stem_chars.next(), Some('-')) && // dash
stem.ends_with(".type.externs") // ends with ".type"
})
}

fn copy_indent(mut read: impl BufRead, mut write: impl Write, indent: &str) -> Result<()> {
let mut line = Vec::with_capacity(100);
while read.read_until(b'\n', &mut line)? != 0 {
write.write_all(indent.as_bytes())?;
write.write_all(&line)?;
line.clear();
}
Ok(())
pub struct Jobserver {
client: jobserver::Client,
reacquire_token_on_drop: bool,
}

fn collect_generated_bindings(modules: &[String], target_module_dir: &Path, manual_dir: &Path) -> Result<()> {
if !target_module_dir.exists() {
fs::create_dir(target_module_dir)?;
}
for path in files_with_extension(target_module_dir, "rs")? {
let _ = fs::remove_file(path);
}

fn write_has_module(mut write: impl Write, module: &str) -> Result<()> {
Ok(writeln!(write, "#[cfg(ocvrs_has_module_{module})]")?)
}

fn write_module_include(write: &mut BufWriter<File>, module: &str) -> Result<()> {
// Use include instead of #[path] attribute because rust-analyzer doesn't handle #[path] inside other include! too well:
// https://github.com/twistedfall/opencv-rust/issues/418
// https://github.com/rust-lang/rust-analyzer/issues/11682
Ok(writeln!(
write,
r#"include!(concat!(env!("OUT_DIR"), "/opencv/{module}.rs"));"#
)?)
}

let add_manual = |file: &mut BufWriter<File>, module: &str| -> Result<bool> {
if manual_dir.join(format!("{module}.rs")).exists() {
writeln!(file, "pub use crate::manual::{module}::*;")?;
Ok(true)
} else {
Ok(false)
}
};

let start = Instant::now();
let mut hub_rs = BufWriter::new(File::create(target_module_dir.join("hub.rs"))?);

let mut types_rs = BufWriter::new(File::create(target_module_dir.join("types.rs"))?);
writeln!(types_rs)?;

let mut sys_rs = BufWriter::new(File::create(target_module_dir.join("sys.rs"))?);
writeln!(sys_rs, "use crate::{{mod_prelude_sys::*, core}};")?;
writeln!(sys_rs)?;

for module in modules {
// merge multiple *-type.cpp files into a single module_types.hpp
let module_cpp = OUT_DIR.join(format!("{module}.cpp"));
if module_cpp.is_file() {
let module_types_cpp = OUT_DIR.join(format!("{module}_types.hpp"));
let mut module_types_file = BufWriter::new(
OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(module_types_cpp)?,
);
let mut type_files = files_with_extension(&OUT_DIR, "cpp")?
.filter(|f| is_type_file(f, module))
.collect::<Vec<_>>();
type_files.sort_unstable();
for entry in type_files {
io::copy(&mut BufReader::new(File::open(&entry)?), &mut module_types_file)?;
let _ = fs::remove_file(entry);
}
}

// add module entry to hub.rs and move the module file into opencv/
write_has_module(&mut hub_rs, module)?;
write_module_include(&mut hub_rs, module)?;
let module_filename = format!("{module}.rs");
let module_src_file = OUT_DIR.join(&module_filename);
let mut module_rs = BufWriter::new(File::create(target_module_dir.join(&module_filename))?);
// Need to wrap modules inside `mod { }` because they have top-level comments (//!) and those don't play well when
// module file is include!d (as opposed to connecting the module with `mod` from the parent module).
// The same doesn't apply to `sys` and `types` below because they don't contain top-level comments.
writeln!(module_rs, "pub mod {module} {{")?;
copy_indent(BufReader::new(File::open(&module_src_file)?), &mut module_rs, "\t")?;
add_manual(&mut module_rs, module)?;
writeln!(module_rs, "}}")?;
let _ = fs::remove_file(module_src_file);

// merge multiple *-.type.rs files into a single types.rs
let mut header_written = false;
let mut type_files = files_with_extension(&OUT_DIR, "rs")?
.filter(|f| is_type_file(f, module))
.collect::<Vec<_>>();
type_files.sort_unstable();
for entry in type_files {
if entry.metadata().map(|meta| meta.len()).unwrap_or(0) > 0 {
if !header_written {
write_has_module(&mut types_rs, module)?;
writeln!(types_rs, "mod {module}_types {{")?;
writeln!(types_rs, "\tuse crate::{{mod_prelude::*, core, types, sys}};")?;
writeln!(types_rs)?;
header_written = true;
impl Jobserver {
pub fn build() -> Result<Self> {
unsafe { jobserver::Client::from_env() }
.and_then(|client| {
let own_token_released = client.release_raw().is_ok();
let available_jobs = client.available().unwrap_or(0);
if available_jobs > 0 {
eprintln!("=== Using environment job server with the the amount of available jobs: {available_jobs}");
Some(Jobserver {
client,
reacquire_token_on_drop: own_token_released,
})
} else {
if own_token_released {
client.acquire_raw().expect("Can't reacquire build script thread token");
}
eprintln!(
"=== Available jobs from the environment created jobserver is: {available_jobs} or there is an error reading that value"
);
None
}
copy_indent(BufReader::new(File::open(&entry)?), &mut types_rs, "\t")?;
}
let _ = fs::remove_file(entry);
}
if header_written {
writeln!(types_rs, "}}")?;
write_has_module(&mut types_rs, module)?;
writeln!(types_rs, "pub use {module}_types::*;")?;
writeln!(types_rs)?;
}

// merge module-specific *.externs.rs and generated type-specific *.type.externs.rs into a single sys.rs
let externs_rs = OUT_DIR.join(format!("{module}.externs.rs"));
write_has_module(&mut sys_rs, module)?;
writeln!(sys_rs, "mod {module}_sys {{")?;
writeln!(sys_rs, "\tuse super::*;")?;
writeln!(sys_rs)?;
writeln!(sys_rs, "\textern \"C\" {{")?;
copy_indent(BufReader::new(File::open(&externs_rs)?), &mut sys_rs, "\t\t")?;
let _ = fs::remove_file(externs_rs);
let mut type_extern_files = files_with_extension(&OUT_DIR, "rs")?
.filter(|f| is_type_externs_file(f, module))
.collect::<Vec<_>>();
type_extern_files.sort_unstable();
for entry in type_extern_files {
if entry.metadata().map(|meta| meta.len()).unwrap_or(0) > 0 {
copy_indent(BufReader::new(File::open(&entry)?), &mut sys_rs, "\t\t")?;
}
let _ = fs::remove_file(entry);
}
writeln!(sys_rs, "\t}}")?;
writeln!(sys_rs, "}}")?;
write_has_module(&mut sys_rs, module)?;
writeln!(sys_rs, "pub use {module}_sys::*;")?;
writeln!(sys_rs)?;
}
writeln!(hub_rs, "pub mod types {{")?;
write!(hub_rs, "\t")?;
write_module_include(&mut hub_rs, "types")?;
writeln!(hub_rs, "}}")?;
writeln!(hub_rs, "#[doc(hidden)]")?;
writeln!(hub_rs, "pub mod sys {{")?;
write!(hub_rs, "\t")?;
write_module_include(&mut hub_rs, "sys")?;
writeln!(hub_rs, "}}")?;

add_manual(&mut types_rs, "types")?;

add_manual(&mut sys_rs, "sys")?;

// write hub_prelude that imports all module-specific preludes
writeln!(hub_rs, "pub mod hub_prelude {{")?;
for module in modules {
write!(hub_rs, "\t")?;
write_has_module(&mut hub_rs, module)?;
writeln!(hub_rs, "\tpub use super::{module}::prelude::*;")?;
}
writeln!(hub_rs, "}}")?;
eprintln!("=== Total binding collection time: {:?}", start.elapsed());
Ok(())
}

fn build_job_server() -> Result<Jobserver> {
unsafe { jobserver::Client::from_env() }
.and_then(|client| {
let own_token_released = client.release_raw().is_ok();
let available_jobs = client.available().unwrap_or(0);
if available_jobs > 0 {
eprintln!("=== Using environment job server with the the amount of available jobs: {available_jobs}");
Some(Jobserver {
})
.or_else(|| {
let num_jobs = env::var("NUM_JOBS")
.ok()
.and_then(|jobs| jobs.parse().ok())
.or_else(|| thread::available_parallelism().map(|p| p.get()).ok())
.unwrap_or(2)
.max(1);
eprintln!("=== Creating a new job server with num_jobs: {num_jobs}");
jobserver::Client::new(num_jobs).ok().map(|client| Jobserver {
client,
reacquire_token_on_drop: own_token_released,
reacquire_token_on_drop: false,
})
} else {
if own_token_released {
client.acquire_raw().expect("Can't reacquire build script thread token");
}
eprintln!(
"=== Available jobs from the environment created jobserver is: {available_jobs} or there is an error reading that value"
);
None
}
})
.or_else(|| {
let num_jobs = env::var("NUM_JOBS")
.ok()
.and_then(|jobs| jobs.parse().ok())
.or_else(|| thread::available_parallelism().map(|p| p.get()).ok())
.unwrap_or(2)
.max(1);
eprintln!("=== Creating a new job server with num_jobs: {num_jobs}");
jobserver::Client::new(num_jobs).ok().map(|client| Jobserver {
client,
reacquire_token_on_drop: false,
})
})
.ok_or_else(|| "Can't create job server".into())
}

pub struct Jobserver {
client: jobserver::Client,
reacquire_token_on_drop: bool,
.ok_or_else(|| "Can't create job server".into())
}
}

impl Drop for Jobserver {
Expand Down
Loading

0 comments on commit c2feda9

Please sign in to comment.