From 16a78526cedfdb9af5d6befce3c77436aabc9a2f Mon Sep 17 00:00:00 2001 From: Jonathan Pallant Date: Mon, 25 Nov 2024 11:26:47 +0000 Subject: [PATCH 1/4] generate-copyright: Ensure output has UNIX line-endings for consistency. --- src/tools/generate-copyright/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index f9d96b594626d..f83d16d0cabf3 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -57,6 +57,10 @@ fn main() -> Result<(), Error> { dependencies: collected_cargo_metadata, }; let output = template.render()?; + // Git stores text files with \n, but this file may contain \r\n in files + // copied from dependencies. Normalise them before we write them out, for + // consistency. + let output = output.replace("\r\n", "\n"); std::fs::write(&dest_file, output)?; // Output libstd subset file @@ -65,6 +69,10 @@ fn main() -> Result<(), Error> { dependencies: library_collected_cargo_metadata, }; let output = template.render()?; + // Git stores text files with \n, but this file may contain \r\n in files + // copied from dependencies. Normalise them before we write them out, for + // consistency. + let output = output.replace("\r\n", "\n"); std::fs::write(&libstd_dest_file, output)?; Ok(()) From 03cdaeed972c730912713ed3151f353367a87561 Mon Sep 17 00:00:00 2001 From: Jonathan Pallant Date: Mon, 25 Nov 2024 14:14:57 +0000 Subject: [PATCH 2/4] collect-license-metadata: move JSON to root, and add a 'check' mode --- REUSE.toml | 1 + license-metadata.json | 269 ++++++++++++++++++ src/bootstrap/src/core/build_steps/run.rs | 3 +- src/bootstrap/src/core/build_steps/test.rs | 32 +++ src/bootstrap/src/core/builder/mod.rs | 1 + .../collect-license-metadata/src/main.rs | 39 ++- .../collect-license-metadata/src/reuse.rs | 4 +- 7 files changed, 335 insertions(+), 14 deletions(-) create mode 100644 license-metadata.json diff --git a/REUSE.toml b/REUSE.toml index 3a11c37854e64..6b16d97ed8068 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -28,6 +28,7 @@ path = [ "COPYRIGHT", "INSTALL.md", "LICENSE-APACHE", + "license-metadata.json", "LICENSE-MIT", "README.md", "RELEASES.md", diff --git a/license-metadata.json b/license-metadata.json new file mode 100644 index 0000000000000..09cc369356584 --- /dev/null +++ b/license-metadata.json @@ -0,0 +1,269 @@ +{ + "files": { + "children": [ + { + "children": [ + { + "children": [ + { + "children": [ + { + "children": [ + { + "directories": [], + "files": [ + "analyzer-decls.h", + "malloc-macro.h" + ], + "license": { + "copyright": [ + "2000-2024 Free Software Foundation, Inc" + ], + "spdx": "GPL-2.0-only" + }, + "type": "group" + } + ], + "license": { + "copyright": [ + "2007-2011 Atheros Communications Inc", + "2011-2012,2017 Qualcomm Atheros, Inc", + "2016-2017 Erik Stromdahl " + ], + "spdx": "ISC" + }, + "name": "c-c++-common/analyzer", + "type": "directory" + } + ], + "license": { + "copyright": [ + "2000-2024 Free Software Foundation, Inc" + ], + "spdx": "GPL-2.0-only" + }, + "name": "gcc/testsuite", + "type": "directory" + }, + { + "license": { + "copyright": [ + "2000-2024 Free Software Foundation, Inc" + ], + "spdx": "GCC-exception-3.1" + }, + "name": "libstdc++-v3/config/os/aix/os_defines.h", + "type": "file" + } + ], + "license": { + "copyright": [ + "1997-2024 Free Software Foundation, Inc" + ], + "spdx": "GPL-3.0-or-later" + }, + "name": "src/gcc", + "type": "directory" + }, + { + "children": [ + { + "license": { + "copyright": [ + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": "noscript.css", + "type": "file" + }, + { + "license": { + "copyright": [ + "Nicolas Gallagher and Jonathan Neal" + ], + "spdx": "MIT" + }, + "name": "normalize.css", + "type": "file" + } + ], + "license": { + "copyright": [ + "2016 Ike Ku, Jessica Stokes and Leon Guan", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": "src/librustdoc/html/static/css", + "type": "directory" + }, + { + "children": [ + { + "license": { + "copyright": [ + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": "README.txt", + "type": "file" + }, + { + "directories": [], + "files": [ + "FiraSans-LICENSE.txt", + "FiraSans-Medium.woff2", + "FiraSans-Regular.woff2" + ], + "license": { + "copyright": [ + "2014, Mozilla Foundation", + "2014, Telefonica S.A" + ], + "spdx": "OFL-1.1" + }, + "type": "group" + }, + { + "directories": [], + "files": [ + "NanumBarunGothic-LICENSE.txt", + "NanumBarunGothic.ttf.woff2" + ], + "license": { + "copyright": [ + "2010 NAVER Corporation" + ], + "spdx": "OFL-1.1" + }, + "type": "group" + } + ], + "license": { + "copyright": [ + "2010, 2012, 2014-2023, Adobe Systems Incorporated" + ], + "spdx": "OFL-1.1" + }, + "name": "src/librustdoc/html/static/fonts", + "type": "directory" + }, + { + "license": { + "copyright": [ + "2003-2019 University of Illinois at Urbana-Champaign", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 WITH LLVM-exception AND (Apache-2.0 OR MIT)" + }, + "name": "compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp", + "type": "file" + }, + { + "children": [], + "license": { + "copyright": [ + "2014 Alex Crichton", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": "library/backtrace", + "type": "directory" + }, + { + "license": { + "copyright": [ + "1991-2024 Unicode, Inc" + ], + "spdx": "Unicode-3.0" + }, + "name": "library/core/src/unicode/unicode_data.rs", + "type": "file" + }, + { + "children": [], + "license": { + "copyright": [ + "2019 The Crossbeam Project Developers", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": "library/std/src/sync/mpmc", + "type": "directory" + }, + { + "license": { + "copyright": [ + "2016 The Fuchsia Authors", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "BSD-2-Clause AND (Apache-2.0 OR MIT)" + }, + "name": "library/std/src/sys/sync/mutex/fuchsia.rs", + "type": "file" + }, + { + "children": [], + "license": { + "copyright": [ + "Rust on Embedded Devices Working Group", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR CC-BY-SA-4.0 OR MIT" + }, + "name": "src/doc/embedded-book", + "type": "directory" + }, + { + "children": [], + "license": { + "copyright": [ + "2014 Jorge Aparicio", + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": "src/doc/rust-by-example", + "type": "directory" + }, + { + "license": { + "copyright": [ + "2014-2021 Knut Sveidqvist" + ], + "spdx": "MIT" + }, + "name": "src/doc/rustc-dev-guide/mermaid.min.js", + "type": "file" + }, + { + "children": [], + "license": { + "copyright": [ + "2003-2019 University of Illinois at Urbana-Champaign", + "2003-2019 by the contributors listed in CREDITS.TXT (https://github.com/rust-lang/llvm-project/blob/7738295178045041669876bf32b0543ec8319a5c/llvm/CREDITS.TXT)", + "2010 Apple Inc" + ], + "spdx": "Apache-2.0 WITH LLVM-exception AND NCSA" + }, + "name": "src/llvm-project", + "type": "directory" + } + ], + "license": { + "copyright": [ + "The Rust Project Developers (see https://thanks.rust-lang.org)" + ], + "spdx": "Apache-2.0 OR MIT" + }, + "name": ".", + "type": "directory" + } + ], + "type": "root" + } +} \ No newline at end of file diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 1a0a90564e6ad..4052de9a240ca 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -181,8 +181,7 @@ impl Step for CollectLicenseMetadata { panic!("REUSE is required to collect the license metadata"); }; - // Temporary location, it will be moved to src/etc once it's accurate. - let dest = builder.out.join("license-metadata.json"); + let dest = builder.src.join("license-metadata.json"); let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); cmd.env("REUSE_EXE", reuse); diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 1cabd1c39f198..58d55d65aea26 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -3675,3 +3675,35 @@ impl Step for TestFloatParse { cargo_run.into_cmd().run(builder); } } + +#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +pub struct CollectLicenseMetadata; + +impl Step for CollectLicenseMetadata { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/collect-license-metadata") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CollectLicenseMetadata); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let Some(reuse) = &builder.config.reuse else { + panic!("REUSE is required to collect the license metadata"); + }; + + let dest = builder.src.join("license-metadata.json"); + + let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); + cmd.env("REUSE_EXE", reuse); + cmd.env("DEST", &dest); + cmd.env("ONLY_CHECK", "1"); + cmd.run(builder); + + dest + } +} diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index d59e0fa728807..e6902bb8cee5f 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -915,6 +915,7 @@ impl<'a> Builder<'a> { test::HtmlCheck, test::RustInstaller, test::TestFloatParse, + test::CollectLicenseMetadata, // Run bootstrap close to the end as it's unlikely to fail test::Bootstrap, // Run run-make last, since these won't pass without make on Windows diff --git a/src/tools/collect-license-metadata/src/main.rs b/src/tools/collect-license-metadata/src/main.rs index dce36bb17b600..08a30d0b8994f 100644 --- a/src/tools/collect-license-metadata/src/main.rs +++ b/src/tools/collect-license-metadata/src/main.rs @@ -4,7 +4,7 @@ mod reuse; use std::path::PathBuf; -use anyhow::Error; +use anyhow::{Context, Error}; use crate::licenses::LicensesInterner; @@ -12,10 +12,12 @@ use crate::licenses::LicensesInterner; /// /// You should probably let `bootstrap` execute this program instead of running it directly. /// -/// Run `x.py run collect-license-metadata` +/// * Run `x.py run collect-license-metadata` to re-regenerate the file. +/// * Run `x.py test collect-license-metadata` to check if the file you have is correct. fn main() -> Result<(), Error> { let reuse_exe: PathBuf = std::env::var_os("REUSE_EXE").expect("Missing REUSE_EXE").into(); let dest: PathBuf = std::env::var_os("DEST").expect("Missing DEST").into(); + let only_check = std::env::var_os("ONLY_CHECK").is_some(); let mut interner = LicensesInterner::new(); let paths = crate::reuse::collect(&reuse_exe, &mut interner)?; @@ -23,15 +25,32 @@ fn main() -> Result<(), Error> { let mut tree = crate::path_tree::build(paths); tree.simplify(); - if let Some(parent) = dest.parent() { - std::fs::create_dir_all(parent)?; + let output = serde_json::json!({ + "files": crate::path_tree::expand_interned_licenses(tree, &interner) + }); + + if only_check { + println!("loading existing license information"); + let existing = std::fs::read_to_string(&dest).with_context(|| { + format!("Failed to read existing license JSON at {}", dest.display()) + })?; + let existing_json: serde_json::Value = + serde_json::from_str(&existing).with_context(|| { + format!("Failed to read existing license JSON at {}", dest.display()) + })?; + if existing_json != output { + eprintln!("The existing {} file is out of date.", dest.display()); + eprintln!("Run ./x run collect-license-metadata to update it."); + anyhow::bail!("The existing {} file doesn't match what REUSE reports.", dest.display()); + } + println!("license information matches"); + } else { + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&dest, &serde_json::to_vec_pretty(&output)?)?; + println!("license information written to {}", dest.display()); } - std::fs::write( - &dest, - &serde_json::to_vec_pretty(&serde_json::json!({ - "files": crate::path_tree::expand_interned_licenses(tree, &interner), - }))?, - )?; Ok(()) } diff --git a/src/tools/collect-license-metadata/src/reuse.rs b/src/tools/collect-license-metadata/src/reuse.rs index e5ee8f0da5ef9..dbe46781b7c5b 100644 --- a/src/tools/collect-license-metadata/src/reuse.rs +++ b/src/tools/collect-license-metadata/src/reuse.rs @@ -10,10 +10,10 @@ pub(crate) fn collect( reuse_exe: &Path, interner: &mut LicensesInterner, ) -> Result, Error> { - eprintln!("gathering license information from REUSE"); + println!("gathering license information from REUSE (this might take a minute...)"); let start = Instant::now(); let raw = &obtain_spdx_document(reuse_exe)?; - eprintln!("finished gathering the license information from REUSE in {:.2?}", start.elapsed()); + println!("finished gathering the license information from REUSE in {:.2?}", start.elapsed()); let document = spdx_rs::parsers::spdx_from_tag_value(&raw)?; From 587369b95e2e5d99b5d808180b90542a6b38096a Mon Sep 17 00:00:00 2001 From: Jonathan Pallant Date: Mon, 25 Nov 2024 14:15:08 +0000 Subject: [PATCH 3/4] Run the license-metadata check in CI. This will tell you if license-metadata.json is out of date. --- src/ci/docker/host-x86_64/mingw-check/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index f0afb570cc4a0..d408cd518a00b 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -63,6 +63,7 @@ ENV SCRIPT \ /scripts/validate-toolstate.sh && \ /scripts/validate-error-codes.sh && \ reuse --include-submodules lint && \ + python3 ../x.py test collect-license-metadata && \ # Runs checks to ensure that there are no issues in our JS code. es-check es2019 ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \ From db71194416700d2c1365ae1ee7f882aa3a055a67 Mon Sep 17 00:00:00 2001 From: Jonathan Pallant Date: Mon, 25 Nov 2024 14:18:08 +0000 Subject: [PATCH 4/4] generate-copyright: Use license-metadata.json from git. --- src/bootstrap/src/core/build_steps/run.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 4052de9a240ca..c76504761beb2 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -208,8 +208,7 @@ impl Step for GenerateCopyright { } fn run(self, builder: &Builder<'_>) -> Self::Output { - let license_metadata = builder.ensure(CollectLicenseMetadata); - + let license_metadata = builder.src.join("license-metadata.json"); let dest = builder.out.join("COPYRIGHT.html"); let dest_libstd = builder.out.join("COPYRIGHT-library.html");