Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Print some information from try builds to GitHub summary #114623

Merged
merged 1 commit into from
Oct 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2634,6 +2634,7 @@ dependencies = [
"serde",
"serde_json",
"sysinfo",
"tabled",
"tar",
"tempfile",
"xz",
Expand Down Expand Up @@ -2694,6 +2695,17 @@ dependencies = [
"unwind",
]

[[package]]
name = "papergrid"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2ccbe15f2b6db62f9a9871642746427e297b0ceb85f9a7f1ee5ff47d184d0c8"
dependencies = [
"bytecount",
"fnv",
"unicode-width",
]

[[package]]
name = "parking_lot"
version = "0.11.2"
Expand Down Expand Up @@ -2902,6 +2914,30 @@ dependencies = [
"pad",
]

[[package]]
name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn 1.0.109",
"version_check",
]

[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
]

[[package]]
name = "proc-macro-hack"
version = "0.5.20+deprecated"
Expand Down Expand Up @@ -5153,6 +5189,30 @@ dependencies = [
"test",
]

[[package]]
name = "tabled"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d38d39c754ae037a9bc3ca1580a985db7371cd14f1229172d1db9093feb6739"
dependencies = [
"papergrid",
"tabled_derive",
"unicode-width",
]

[[package]]
name = "tabled_derive"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99f688a08b54f4f02f0a3c382aefdb7884d3d69609f785bd253dc033243e3fe4"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.109",
]

[[package]]
name = "tar"
version = "0.4.38"
Expand Down
6 changes: 6 additions & 0 deletions src/ci/docker/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,9 @@ else
BASE_COMMIT=""
fi

SUMMARY_FILE=github-summary.md
touch $objdir/${SUMMARY_FILE}

docker \
run \
--workdir /checkout/obj \
Expand All @@ -275,6 +278,7 @@ docker \
--env CI \
--env GITHUB_ACTIONS \
--env GITHUB_REF \
--env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \
--env TOOLSTATE_REPO_ACCESS_TOKEN \
--env TOOLSTATE_REPO \
--env TOOLSTATE_PUBLISH \
Expand All @@ -289,6 +293,8 @@ docker \
rust-ci \
$command

cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}"

if [ -f /.dockerenv ]; then
rm -rf $objdir
docker cp checkout:/checkout/obj $objdir
Expand Down
1 change: 1 addition & 0 deletions src/tools/opt-dist/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ glob = "0.3"
tempfile = "3.5"
derive_builder = "0.12"
clap = { version = "4", features = ["derive"] }
tabled = "0.13"
9 changes: 7 additions & 2 deletions src/tools/opt-dist/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@ use crate::exec::{cmd, Bootstrap};
use crate::tests::run_tests;
use crate::timer::Timer;
use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
use crate::utils::artifact_size::print_binary_sizes;
use crate::utils::io::{copy_directory, move_directory, reset_directory};
use crate::utils::{
clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
retry_action, with_log_group,
clear_llvm_files, format_env_variables, print_free_disk_space, retry_action, with_log_group,
write_timer_to_summary,
};

mod bolt;
Expand Down Expand Up @@ -359,6 +360,10 @@ fn main() -> anyhow::Result<()> {
let result = execute_pipeline(&env, &mut timer, build_args);
log::info!("Timer results\n{}", timer.format_stats());

if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") {
write_timer_to_summary(&summary_path, &timer)?;
}

print_free_disk_space()?;
result.context("Optimized build pipeline has failed")?;
print_binary_sizes(&env)?;
Expand Down
61 changes: 61 additions & 0 deletions src/tools/opt-dist/src/utils/artifact_size.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
use std::io::Write;

use tabled::builder::Builder;
use tabled::settings::object::Columns;
use tabled::settings::style::{BorderChar, Offset};
use tabled::settings::{Modify, Style};

use crate::environment::Environment;
use crate::utils::io::get_files_from_dir;

pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
use humansize::format_size;
use humansize::BINARY;
use std::fmt::Write;

let root = env.build_artifacts().join("stage2");

let mut files = get_files_from_dir(&root.join("bin"), None)?;
files.extend(get_files_from_dir(&root.join("lib"), Some(".so"))?);
files.sort_unstable();

let items: Vec<_> = files
.into_iter()
.map(|file| {
let size = std::fs::metadata(file.as_std_path()).map(|m| m.len()).unwrap_or(0);
let size_formatted = format_size(size, BINARY);
let name = file.file_name().unwrap().to_string();
(name, size_formatted)
})
.collect();

// Write to log
let mut output = String::new();
for (name, size_formatted) in items.iter() {
let name = format!("{}:", name);
writeln!(output, "{name:<50}{size_formatted:>10}")?;
}
log::info!("Rustc artifact size\n{output}");

// Write to GitHub summary
if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") {
let mut builder = Builder::default();
for (name, size_formatted) in items {
builder.push_record(vec![name, size_formatted]);
}

builder.set_header(vec!["Artifact", "Size"]);
let mut table = builder.build();

let mut file = std::fs::File::options().append(true).create(true).open(summary_path)?;
writeln!(
file,
"# Artifact size\n{}\n",
table.with(Style::markdown()).with(
Modify::new(Columns::single(1)).with(BorderChar::horizontal(':', Offset::End(0))),
)
)?;
}

Ok(())
}
51 changes: 25 additions & 26 deletions src/tools/opt-dist/src/utils/mod.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
pub mod io;
use sysinfo::{DiskExt, RefreshKind, System, SystemExt};

use crate::environment::Environment;
use crate::utils::io::{delete_directory, get_files_from_dir};
use humansize::{format_size, BINARY};
use crate::timer::Timer;
use crate::utils::io::delete_directory;
use humansize::BINARY;
use std::time::Duration;
use sysinfo::{DiskExt, RefreshKind, System, SystemExt};

pub mod artifact_size;
pub mod io;

pub fn format_env_variables() -> String {
let vars = std::env::vars().map(|(key, value)| format!("{key}={value}")).collect::<Vec<_>>();
Expand All @@ -26,28 +29,6 @@ pub fn print_free_disk_space() -> anyhow::Result<()> {
Ok(())
}

pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
use std::fmt::Write;

let root = env.build_artifacts().join("stage2");

let mut files = get_files_from_dir(&root.join("bin"), None)?;
files.extend(get_files_from_dir(&root.join("lib"), Some(".so"))?);
files.sort_unstable();

let mut output = String::new();
for file in files {
let size = std::fs::metadata(file.as_std_path())?.len();
let size_formatted = format_size(size, BINARY);
let name = format!("{}:", file.file_name().unwrap());
writeln!(output, "{name:<50}{size_formatted:>10}")?;
}

log::info!("Rustc artifact size\n{output}");

Ok(())
}

pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
// Bootstrap currently doesn't support rebuilding LLVM when PGO options
// change (or any other llvm-related options); so just clear out the relevant
Expand All @@ -58,6 +39,24 @@ pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
Ok(())
}

/// Write the formatted statistics of the timer to a Github Actions summary.
pub fn write_timer_to_summary(path: &str, timer: &Timer) -> anyhow::Result<()> {
use std::io::Write;

let mut file = std::fs::File::options().append(true).create(true).open(path)?;
writeln!(
file,
r#"# Step durations

```
{}
```
"#,
timer.format_stats()
)?;
Ok(())
}

/// Wraps all output produced within the `func` closure in a CI output group, if we're running in
/// CI.
pub fn with_log_group<F: FnOnce() -> R, R>(group: &str, func: F) -> R {
Expand Down
Loading