diff --git a/Makefile.toml b/Makefile.toml index 1e9783a837a..d756cd20a9e 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -20,7 +20,7 @@ DOCKER_BUILDKIT = "1" [env.development] # Defined here to allow us to override ${BUILDSYS_ARCH} on the command line. -BUILDSYS_SDK_IMAGE = "thar/sdk-${BUILDSYS_ARCH}:v0.7" +BUILDSYS_SDK_IMAGE = "thar/sdk-${BUILDSYS_ARCH}:v0.8" # Permit pulling directly Upstream URLs when lookaside cache results in MISSes. BUILDSYS_ALLOW_UPSTREAM_SOURCE_URL = "true" # Extra flags used when spawning containers. diff --git a/extras/sdk-container/Dockerfile b/extras/sdk-container/Dockerfile index 3de064d888c..8ecacba15c2 100644 --- a/extras/sdk-container/Dockerfile +++ b/extras/sdk-container/Dockerfile @@ -336,8 +336,8 @@ FROM sdk-rust as sdk-license-scan USER root RUN \ - mkdir -p /usr/libexec/tools /home/builder/license-scan && \ - chown -R builder:builder /usr/libexec/tools /home/builder/license-scan + mkdir -p /usr/libexec/tools /home/builder/license-scan /usr/share/licenses/thar-license-scan && \ + chown -R builder:builder /usr/libexec/tools /home/builder/license-scan /usr/share/licenses/thar-license-scan ARG SPDXVER="3.7" @@ -354,6 +354,13 @@ COPY license-scan /home/builder/license-scan RUN cargo build --release --locked RUN install -p -m 0755 target/release/thar-license-scan /usr/libexec/tools/ RUN cp -r license-list-data/json/details /usr/libexec/tools/spdx-data +# quine - scan the license tool itself for licenses +RUN \ + /usr/libexec/tools/thar-license-scan \ + --clarify clarify.toml \ + --spdx-data /usr/libexec/tools/spdx-data \ + --out-dir /usr/share/licenses/thar-license-scan/vendor \ + cargo --locked Cargo.toml # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= @@ -391,6 +398,8 @@ COPY --chown=0:0 --from=sdk-go /home/builder/go/src /usr/libexec/go/src/ # "sdk-license-scan" has our attribution generation tool. COPY --chown=0:0 --from=sdk-license-scan /usr/libexec/tools/ /usr/libexec/tools/ +# quine - include the licenses for the license scan tool itself +COPY --chown=0:0 --from=sdk-license-scan /usr/share/licenses/thar-license-scan/ /usr/share/licenses/thar-license-scan/ # Add Rust programs and libraries to the path. RUN \ diff --git a/extras/sdk-container/Makefile b/extras/sdk-container/Makefile index 773224b0ee0..2456e004ba3 100644 --- a/extras/sdk-container/Makefile +++ b/extras/sdk-container/Makefile @@ -1,6 +1,6 @@ ARCH ?= $(shell uname -m) -VERSION := v0.7 +VERSION := v0.8 TAG := thar/sdk-$(ARCH):$(VERSION) ARCHIVE := thar-sdk-$(ARCH)-$(VERSION).tar.gz diff --git a/extras/sdk-container/license-scan/Cargo.lock b/extras/sdk-container/license-scan/Cargo.lock index b1e29f1169c..0a4a4b8bc2b 100644 --- a/extras/sdk-container/license-scan/Cargo.lock +++ b/extras/sdk-container/license-scan/Cargo.lock @@ -86,6 +86,17 @@ dependencies = [ "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cargo_metadata" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cc" version = "1.0.50" @@ -495,6 +506,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -611,6 +623,7 @@ version = "0.1.0" dependencies = [ "anyhow 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", "askalono 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cargo_metadata 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "ignore 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -749,6 +762,7 @@ dependencies = [ "checksum bstr 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3ede750122d9d1f87919570cb2cccee38c84fbc8c5599b25c289af40625b7030" "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" +"checksum cargo_metadata 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "46e3374c604fb39d1a2f35ed5e4a4e30e60d01fab49446e08f1b3e9a90aef202" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" diff --git a/extras/sdk-container/license-scan/Cargo.toml b/extras/sdk-container/license-scan/Cargo.toml index 6434fcf65ac..986c7ce6021 100644 --- a/extras/sdk-container/license-scan/Cargo.toml +++ b/extras/sdk-container/license-scan/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] anyhow = "1" askalono = "0.4" +cargo_metadata = "0.9" ignore = "0.4" lazy_static = "1" serde = { version = "1", features = ["derive"] } diff --git a/extras/sdk-container/license-scan/clarify.toml b/extras/sdk-container/license-scan/clarify.toml new file mode 100644 index 00000000000..d4ebfcca0b3 --- /dev/null +++ b/extras/sdk-container/license-scan/clarify.toml @@ -0,0 +1,62 @@ +[clarify.askalono] +expression = "Apache-2.0" +license-files = [ + { path = "LICENSE", hash = 0x18785531 }, + { path = "NOTICE", hash = 0x96b3ea7d }, +] +skip-files = [ + "src/license.rs" # source code named "license"... +] + +[clarify.backtrace-sys] +# backtrace-sys is MIT/Apache-2.0, libbacktrace is BSD-3-Clause +expression = "(MIT OR Apache-2.0) AND BSD-3-Clause" +license-files = [ + { path = "src/libbacktrace/LICENSE", hash = 0x0ce09262 }, +] + +[clarify.crossbeam-channel] +expression = "(MIT OR Apache-2.0) AND BSD-2-Clause AND CC-BY-3.0" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0xbc436f08 }, + { path = "LICENSE-THIRD-PARTY", hash = 0xc6242648 }, +] + +[clarify.crossbeam-queue] +expression = "(MIT OR Apache-2.0) AND BSD-2-Clause-FreeBSD" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0xbc436f08 }, + { path = "LICENSE-THIRD-PARTY", hash = 0x7e40bc60 }, +] + +[clarify.regex] +expression = "MIT OR Apache-2.0" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0xb755395b }, +] +skip-files = [ + "src/testdata/LICENSE", # we aren't using the test data +] + +[clarify.regex-syntax] +expression = "(MIT OR Apache-2.0) AND Unicode-DFS-2016" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0xb755395b }, + { path = "src/unicode_tables/LICENSE-UNICODE", hash = 0xa7f28b93 }, +] + +[clarify.zstd-sys] +# zstd-sys is MIT OR Apache-2.0 +# libzstd is GPL-2.0-only OR BSD-3-Clause (selecting BSD-3-Clause) +expression = "(MIT OR Apache-2.0) AND BSD-3-Clause" +license-files = [ + { path = "zstd/LICENSE", hash = 0x79cda15 }, +] +skip-files = [ + "zstd/COPYING", # copy of the GPLv2 we are not choosing from libzstd's dual license + "zstd/contrib/linux-kernel/COPYING", # kernel source and patches for adding zstd (?!), not used +] diff --git a/extras/sdk-container/license-scan/src/main.rs b/extras/sdk-container/license-scan/src/main.rs index e41a777d47c..f158d7a8b83 100644 --- a/extras/sdk-container/license-scan/src/main.rs +++ b/extras/sdk-container/license-scan/src/main.rs @@ -2,7 +2,7 @@ #![warn(clippy::pedantic)] #![allow(clippy::redundant_closure_for_method_calls)] -use anyhow::{bail, ensure, Context, Result}; +use anyhow::{anyhow, bail, ensure, Context, Result}; use askalono::{ScanStrategy, Store, TextData}; use ignore::types::{Types, TypesBuilder}; use ignore::WalkBuilder; @@ -42,6 +42,18 @@ enum Cmd { /// Path to the vendor directory of a project. vendor_dir: PathBuf, }, + Cargo { + /// Path to Cargo.toml for a project. + manifest_path: PathBuf, + + /// Equivalent to `cargo --locked` + #[structopt(long)] + locked: bool, + + /// Equivalent to `cargo --offline` + #[structopt(long)] + offline: bool, + }, } fn main() -> Result<()> { @@ -75,6 +87,53 @@ fn main() -> Result<()> { &opt.out_dir.join(&repo), &scanner, &clarify, + None, + )?; + } + Ok(()) + } + Cmd::Cargo { + manifest_path, + locked, + offline, + } => { + let mut builder = cargo_metadata::MetadataCommand::new(); + builder.manifest_path(manifest_path); + if locked { + builder.other_options(&["--locked".to_owned()]); + } + if offline { + builder.other_options(&["--offline".to_owned()]); + } + let metadata = builder.exec()?; + for package in metadata.packages { + if package.source.is_none() { + if let Some(publish) = package.publish { + if publish.is_empty() { + // `package.source` is None if the project is a local project; + // `package.publish` is an empty Vec if `publish = false` is set + continue; + } + } + } + write_attribution( + &package.name, + package + .manifest_path + .parent() + .expect("expected a path to Cargo.toml to have a parent"), + &opt.out_dir + .join(format!("{}-{}", package.name, package.version)), + &scanner, + &clarify, + if let Some(license) = package.license { + Some(Expression::parse(&unslash(&license)).map_err(|err| { + // spdx errors use the lifetime of the string + anyhow!(err.to_string()) + })?) + } else { + None + }, )?; } Ok(()) @@ -203,6 +262,12 @@ lazy_static::lazy_static! { }; } +/// Replace '/' characters in a license string with 'OR'. (crates.io allows '/' instead of 'OR' for +/// compatibility.) +fn unslash(s: &str) -> String { + s.split('/').map(str::trim).collect::>().join(" OR ") +} + /// Returns true if the file is expected to not be a license text (such as the Apache-2.0 NOTICE /// file). fn non_license(path: &Path) -> bool { @@ -222,14 +287,19 @@ fn hash(data: &[u8]) -> u32 { .expect("XxHash32 returned hash larger than 32 bits") } +#[allow(clippy::too_many_lines)] // maybe someday... fn write_attribution( name: &str, scan_dir: &Path, out_dir: &Path, scanner: &ScanStrategy<'_>, clarifications: &Clarifications, + stated_license: Option, ) -> Result<()> { eprintln!("{}:", name); + if let Some(stated_license) = stated_license.as_ref() { + eprintln!(" + {} (stated in metadata)", stated_license); + } let mut files = HashMap::new(); for entry in WalkBuilder::new(scan_dir).types(TYPES.clone()).build() { let entry = entry?; @@ -266,6 +336,21 @@ fn write_attribution( file_hash ); } else { + if stated_license.is_some() { + // if the package states a license and we heuristically detect that this is + // a top-level "either license, at your option" file, ignore it + let trainwreck = data.split_whitespace().collect::>().join(" "); + if trainwreck.contains("under the terms of either license") + || trainwreck.contains("at your option") + { + eprintln!( + " + {} (hash = 0x{:x}) detected as non-license file", + file.display(), + file_hash + ); + continue; + } + } bail!( "failed to detect any license from {} (hash = 0x{:x}), \ please add a clarification", @@ -282,15 +367,41 @@ fn write_attribution( result.license.name, result.score, ); - licenses.push(result.license.name); + if let Some(stated_license) = stated_license.as_ref() { + // The license we detected should be included in the stated license string, + // otherwise we know the stated license is incomplete, in which case we should + // have had a clarification. + ensure!( + stated_license.requirements().any(|er| { + // `er` is an `ExpressionReq`; `er.req` is a `LicenseReq`. + // `er.req.license.id()` returns `Option`. + er.req.license.id().is_some() + && er.req.license.id() == spdx::license_id(result.license.name) + }), + "detected license \"{}\" from {} is not present in the license \ + field \"{}\" for {}", + result.license.name, + file.display(), + stated_license, + name + ); + } else { + licenses.push(result.license.name); + } } } - licenses.sort(); - licenses.dedup(); - let expression = licenses.join(" AND "); - eprintln!(" = {}", expression); + copy_files(out_dir, &files, &[])?; - expression + + if let Some(stated_license) = stated_license { + stated_license.to_string() + } else { + licenses.sort(); + licenses.dedup(); + let expression = licenses.join(" AND "); + eprintln!(" = {}", expression); + expression + } }; fs::create_dir_all(out_dir)?; diff --git a/packages/release/release.spec b/packages/release/release.spec index 8533d10024e..1f6bacab8a1 100644 --- a/packages/release/release.spec +++ b/packages/release/release.spec @@ -57,6 +57,7 @@ Requires: %{_cross_os}updog Requires: %{_cross_os}util-linux Requires: %{_cross_os}preinit Requires: %{_cross_os}wicked +Requires: %{_cross_os}workspaces %description %{summary}. diff --git a/packages/workspaces/workspaces.spec b/packages/workspaces/workspaces.spec index 659184ec425..42432e9f63f 100644 --- a/packages/workspaces/workspaces.spec +++ b/packages/workspaces/workspaces.spec @@ -218,6 +218,12 @@ install -d %{buildroot}%{_cross_tmpfilesdir} install -p -m 0644 %{S:200} %{buildroot}%{_cross_tmpfilesdir}/migration.conf install -p -m 0644 %{S:201} %{buildroot}%{_cross_tmpfilesdir}/host-containers.conf +%cross_scan_attribution --clarify %{_builddir}/workspaces/clarify.toml \ + cargo --offline --locked %{_builddir}/workspaces/Cargo.toml + +%files +%{_cross_attribution_vendor_dir} + %files -n %{_cross_os}apiserver %{_cross_bindir}/apiserver %{_cross_unitdir}/apiserver.service diff --git a/workspaces/clarify.toml b/workspaces/clarify.toml new file mode 100644 index 00000000000..45fb00fc5d9 --- /dev/null +++ b/workspaces/clarify.toml @@ -0,0 +1,73 @@ +[clarify.backtrace-sys] +# backtrace-sys is MIT/Apache-2.0, libbacktrace is BSD-3-Clause +expression = "(MIT OR Apache-2.0) AND BSD-3-Clause" +license-files = [ + { path = "src/libbacktrace/LICENSE", hash = 0x0ce09262 }, +] + +[clarify.crossbeam-queue] +expression = "(MIT OR Apache-2.0) AND BSD-2-Clause-FreeBSD" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0x386ca1bc }, + { path = "LICENSE-THIRD-PARTY", hash = 0x7e40bc60 }, +] + +[clarify.lz4-sys] +# The lz4-sys crate's license is listed as MIT. +# +# lz4-sys compiles four files from liblz4 as a static library: +# - lib/lz4.c +# - lib/lz4frame.c +# - lib/lz4hc.c +# - lib/xxhash.c +# +# liblz4's LICENSE file states: +# > This repository uses 2 different licenses : +# > - all files in the `lib` directory use a BSD 2-Clause license +# > - all other files use a GPLv2 license, unless explicitly stated otherwise +expression = "MIT AND BSD-2-Clause" +license-files = [ + { path = "liblz4/lib/LICENSE", hash = 0x5c0c490b }, +] +skip-files = [ + "liblz4/LICENSE", # top-level explainer file + "liblz4/contrib/djgpp/LICENSE", + "liblz4/examples/COPYING", + "liblz4/programs/COPYING", + "liblz4/tests/COPYING", +] + +[clarify.pulldown-cmark] +expression = "MIT" +license-files = [ + { path = "LICENSE", hash = 0x4cb272b3 }, +] +skip-files = [ + "third_party/CommonMark/LICENSE", # only contains the commonmark specification +] + +[clarify.regex] +expression = "MIT OR Apache-2.0" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0xb755395b }, +] +skip-files = [ + "src/testdata/LICENSE", # we aren't using the test data +] + +[clarify.regex-syntax] +expression = "(MIT OR Apache-2.0) AND Unicode-DFS-2016" +license-files = [ + { path = "LICENSE-APACHE", hash = 0x24b54f4b }, + { path = "LICENSE-MIT", hash = 0xb755395b }, + { path = "src/unicode_tables/LICENSE-UNICODE", hash = 0xa7f28b93 }, +] + +[clarify.ring] +expression = "MIT AND ISC AND OpenSSL" +license-files = [ + { path = "LICENSE", hash = 0xbd0eed23 }, + { path = "third_party/fiat/LICENSE", hash = 0xdad0527d }, +]