From 525811dc6a3c53b9c0c0d7b435260c1dfd2e1b8a Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Thu, 29 Oct 2020 10:21:26 +0000 Subject: [PATCH 01/22] content hash for src and bin files --- .../rustc_codegen_llvm/src/back/archive.rs | 6 +- compiler/rustc_codegen_llvm/src/base.rs | 51 +++++++++ compiler/rustc_codegen_llvm/src/metadata.rs | 20 +++- compiler/rustc_codegen_ssa/src/back/link.rs | 31 +++++- .../src/back/symbol_export.rs | 7 +- compiler/rustc_codegen_ssa/src/lib.rs | 4 +- compiler/rustc_interface/Cargo.toml | 1 + compiler/rustc_interface/src/passes.rs | 103 +++++++++++++++--- compiler/rustc_metadata/src/rmeta/encoder.rs | 7 +- .../src/middle/exported_symbols.rs | 5 + compiler/rustc_span/src/lib.rs | 9 ++ 11 files changed, 211 insertions(+), 33 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 595655b2ca26f..b42c3d1f4a497 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -10,7 +10,9 @@ use std::str; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind}; use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder}; -use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME}; +use rustc_codegen_ssa::{ + looks_like_rust_object_file, METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX, +}; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -130,7 +132,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { self.add_archive(rlib, move |fname: &str| { // Ignore metadata files, no matter the name. - if fname == METADATA_FILENAME { + if fname.starts_with(METADATA_FILE_PREFIX) && fname.ends_with(METADATA_FILE_EXTENSION) { return true; } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 1090d4a25c7cf..08580685720e3 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -28,6 +28,8 @@ use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_data_structures::svh::Svh; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::middle::cstore::EncodedMetadata; @@ -41,6 +43,7 @@ use std::ffi::CString; use std::time::Instant; pub fn write_compressed_metadata<'tcx>( + // backend: &T, tcx: TyCtxt<'tcx>, metadata: &EncodedMetadata, llvm_module: &mut ModuleLlvm, @@ -70,6 +73,9 @@ pub fn write_compressed_metadata<'tcx>( let directive = format!(".section {}", section_name); llvm::LLVMSetModuleInlineAsm2(metadata_llmod, directive.as_ptr().cast(), directive.len()) } + + let svh = tcx.crate_hash(LOCAL_CRATE); + let _llglobal = add_svh(tcx, &llvm_module, &svh); } pub struct ValueIter<'ll> { @@ -93,6 +99,45 @@ pub fn iter_globals(llmod: &'ll llvm::Module) -> ValueIter<'ll> { unsafe { ValueIter { cur: llvm::LLVMGetFirstGlobal(llmod), step: llvm::LLVMGetNextGlobal } } } +pub fn svh_symbol_name( + tcx: TyCtxt<'_>, + svh: &Svh, //, cgu_name: Symbol +) -> String { + format!("rust_svh_{}_{}", tcx.original_crate_name(LOCAL_CRATE), svh) +} + +fn add_svh(tcx: TyCtxt<'tcx>, llvm_module: &'ll ModuleLlvm, svh: &Svh) -> &'ll Value { + // Add SVH @todo insert symbol here + let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod()); + + let svh_bytes: Vec = format!("{}", svh).as_bytes().to_vec(); + let llconst = common::bytes_in_context(metadata_llcx, &svh_bytes); + let name = svh_symbol_name(tcx, &svh); //cgu_name + let buf = CString::new(name).unwrap(); + let llglobal = + unsafe { llvm::LLVMAddGlobal(metadata_llmod, common::val_ty(llconst), buf.as_ptr()) }; + unsafe { + llvm::LLVMSetInitializer(llglobal, llconst); + + let section_name = format!( + "__DATA,.rust_svh_hash", //TO DO make work for non-OSX not ,no_dead_strip + ); + let name = SmallCStr::new(§ion_name); + llvm::LLVMSetSection(llglobal, name.as_ptr()); + + // Also generate a .section directive to force no + // flags, at least for ELF outputs, so that the + // metadata doesn't get loaded into memory. + let directive = format!(".section {}", section_name); + llvm::LLVMRustAppendModuleInlineAsm( + metadata_llmod, + directive.as_ptr().cast(), + directive.len(), + ) + } + llglobal +} + pub fn compile_codegen_unit( tcx: TyCtxt<'tcx>, cgu_name: Symbol, @@ -116,6 +161,7 @@ pub fn compile_codegen_unit( let llvm_module = ModuleLlvm::new(tcx, &cgu_name.as_str()); { let cx = CodegenCx::new(tcx, cgu, &llvm_module); + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); for &(mono_item, (linkage, visibility)) in &mono_items { mono_item.predefine::>(&cx, linkage, visibility); @@ -129,6 +175,11 @@ pub fn compile_codegen_unit( // If this codegen unit contains the main function, also create the // wrapper here if let Some(entry) = maybe_create_entry_wrapper::>(&cx) { + let svh = tcx.crate_hash(LOCAL_CRATE); + + let llglobal = add_svh(tcx, &llvm_module, &svh); //&mut cx, + cx.add_used_global(llglobal); + attributes::sanitize(&cx, SanitizerSet::empty(), entry); } diff --git a/compiler/rustc_codegen_llvm/src/metadata.rs b/compiler/rustc_codegen_llvm/src/metadata.rs index 9036428c04b8a..34aedd2361753 100644 --- a/compiler/rustc_codegen_llvm/src/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/metadata.rs @@ -1,12 +1,11 @@ use crate::llvm; use crate::llvm::archive_ro::ArchiveRO; use crate::llvm::{mk_section_iter, False, ObjectFile}; -use rustc_middle::middle::cstore::MetadataLoader; -use rustc_target::spec::Target; - -use rustc_codegen_ssa::METADATA_FILENAME; +use rustc_codegen_ssa::{METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX}; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; +use rustc_middle::middle::cstore::MetadataLoader; +use rustc_target::spec::Target; use tracing::debug; use rustc_fs_util::path_to_c_string; @@ -30,10 +29,19 @@ impl MetadataLoader for LlvmMetadataLoader { let buf: OwningRef<_, [u8]> = archive.try_map(|ar| { ar.iter() .filter_map(|s| s.ok()) - .find(|sect| sect.name() == Some(METADATA_FILENAME)) + .find(|sect| { + if let Some(n) = sect.name() { + n.starts_with(METADATA_FILE_PREFIX) && n.ends_with(METADATA_FILE_EXTENSION) + } else { + false + } + }) .map(|s| s.data()) .ok_or_else(|| { - debug!("didn't find '{}' in the archive", METADATA_FILENAME); + debug!( + "didn't find '{}*{}' in the archive", + METADATA_FILE_PREFIX, METADATA_FILE_EXTENSION + ); format!("failed to read rlib metadata: '{}'", filename.display()) }) })?; diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 63d0a88858ec1..de6493777ce94 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -1,4 +1,5 @@ use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::svh::Svh; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_fs_util::fix_windows_verbatim_for_gcc; use rustc_hir::def_id::CrateNum; @@ -21,7 +22,10 @@ use super::archive::ArchiveBuilder; use super::command::Command; use super::linker::{self, Linker}; use super::rpath::{self, RPathConfig}; -use crate::{looks_like_rust_object_file, CodegenResults, CrateInfo, METADATA_FILENAME}; +use crate::{ + looks_like_rust_object_file, CodegenResults, CrateInfo, METADATA_FILE_EXTENSION, + METADATA_FILE_PREFIX, +}; use cc::windows_registry; use tempfile::Builder as TempFileBuilder; @@ -270,8 +274,16 @@ pub fn each_linked_rlib( /// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a /// directory being searched for `extern crate` (observing an incomplete file). /// The returned path is the temporary file containing the complete metadata. -pub fn emit_metadata(sess: &Session, metadata: &EncodedMetadata, tmpdir: &MaybeTempDir) -> PathBuf { - let out_filename = tmpdir.as_ref().join(METADATA_FILENAME); +pub fn emit_metadata( + sess: &Session, + metadata: &EncodedMetadata, + tmpdir: &MaybeTempDir, + crate_name: &str, + svh: &Svh, +) -> PathBuf { + let out_filename = tmpdir + .as_ref() + .join(format!("{}{}-{}{}", METADATA_FILE_PREFIX, crate_name, svh, METADATA_FILE_EXTENSION)); let result = fs::write(&out_filename, &metadata.raw_data); if let Err(e) = result { @@ -355,12 +367,19 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( // not be correctly inferred once 'foo.o' is removed. // // Basically, all this means is that this code should not move above the - // code above. + // code above. @audit rlib add here + match flavor { RlibFlavor::Normal => { // Instead of putting the metadata in an object file section, rlibs // contain the metadata in a separate file. - ab.add_file(&emit_metadata(sess, &codegen_results.metadata, tmpdir)); + ab.add_file(&emit_metadata( + sess, + &codegen_results.metadata, + tmpdir, + &codegen_results.crate_name.as_str(), + &codegen_results.crate_hash, + )); // After adding all files to the archive, we need to update the // symbol table of the archive. This currently dies on macOS (see @@ -1916,7 +1935,7 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let mut any_objects = false; for f in archive.src_files() { - if f == METADATA_FILENAME { + if f.starts_with(METADATA_FILE_PREFIX) && f.ends_with(METADATA_FILE_EXTENSION) { archive.remove_file(&f); continue; } diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index dd8d751d04591..1861dda81cd28 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -9,7 +9,7 @@ use rustc_hir::Node; use rustc_index::vec::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::middle::exported_symbols::{ - metadata_symbol_name, ExportedSymbol, SymbolExportLevel, + metadata_symbol_name, svh_symbol_name, ExportedSymbol, SymbolExportLevel, }; use rustc_middle::ty::query::Providers; use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; @@ -218,6 +218,11 @@ fn exported_symbols_provider_local( let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); symbols.push((exported_symbol, SymbolExportLevel::Rust)); + + let symbol_name = svh_symbol_name(tcx); + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); + + symbols.push((exported_symbol, SymbolExportLevel::C)); } if tcx.sess.opts.share_generics() && tcx.local_crate_exports_generics() { diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 70b92b234e94c..46bf49aa8bb2c 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -56,8 +56,8 @@ pub struct ModuleCodegen { pub kind: ModuleKind, } -// FIXME(eddyb) maybe include the crate name in this? -pub const METADATA_FILENAME: &str = "lib.rmeta"; +pub const METADATA_FILE_PREFIX: &str = "lib"; +pub const METADATA_FILE_EXTENSION: &str = ".rmeta"; impl ModuleCodegen { pub fn into_compiled_module( diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index e214493a567b8..42dc84f7fa73e 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -43,6 +43,7 @@ rustc_resolve = { path = "../rustc_resolve" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_ty = { path = "../rustc_ty" } tempfile = "3.0.5" +md-5 = "0.8" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["libloaderapi"] } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 9dbd59506b188..a82a1be6a9b18 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -34,19 +34,21 @@ use rustc_session::output::{filename_for_input, filename_for_metadata}; use rustc_session::search_paths::PathKind; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_span::{FileName, RealFileName}; +use rustc_span::{FileName, RealFileName, SourceFile}; use rustc_trait_selection::traits; use rustc_typeck as typeck; use smallvec::SmallVec; use tracing::{info, warn}; +use md5::{Digest, Md5}; + use rustc_serialize::json; use tempfile::Builder as TempFileBuilder; use std::any::Any; use std::cell::RefCell; use std::ffi::OsString; -use std::io::{self, BufWriter, Write}; +use std::io::{self, BufWriter, Read, Write}; use std::lazy::SyncLazy; use std::path::PathBuf; use std::rc::Rc; @@ -602,6 +604,12 @@ fn escape_dep_env(symbol: Symbol) -> String { escaped } +enum FileHash { + SourceFile(Lrc), + BinaryHash(u64, Vec), + HashInFilename(u64), +} + fn write_out_deps( sess: &Session, boxed_resolver: &Steal>>, @@ -617,13 +625,17 @@ fn write_out_deps( let result = (|| -> io::Result<()> { // Build a list of files used to compile the output and // write Makefile-compatible dependency rules - let mut files: Vec = sess - .source_map() - .files() + let source_files = sess.source_map().files(); + let mut files: Vec<(String, Option)> = source_files .iter() .filter(|fmap| fmap.is_real_file()) .filter(|fmap| !fmap.is_imported()) - .map(|fmap| escape_dep_filename(&fmap.unmapped_path.as_ref().unwrap_or(&fmap.name))) + .map(|fmap| { + ( + escape_dep_filename(&fmap.unmapped_path.as_ref().unwrap_or(&fmap.name)), + Some(FileHash::SourceFile(fmap.clone())), + ) + }) .collect(); if let Some(ref backend) = sess.opts.debugging_opts.codegen_backend { @@ -635,16 +647,19 @@ fn write_out_deps( for cnum in resolver.cstore().crates_untracked() { let source = resolver.cstore().crate_source_untracked(cnum); if let Some((path, _)) = source.dylib { + let hash = hash_bin(&path).ok(); let file_name = FileName::Real(RealFileName::Named(path)); - files.push(escape_dep_filename(&file_name)); + files.push((escape_dep_filename(&file_name), hash)); } if let Some((path, _)) = source.rlib { + let hash = hash_bin_with_svh(&path).ok(); let file_name = FileName::Real(RealFileName::Named(path)); - files.push(escape_dep_filename(&file_name)); + files.push((escape_dep_filename(&file_name), hash)); } if let Some((path, _)) = source.rmeta { + let hash = hash_bin_with_svh(&path).ok(); let file_name = FileName::Real(RealFileName::Named(path)); - files.push(escape_dep_filename(&file_name)); + files.push((escape_dep_filename(&file_name), hash)); } } }); @@ -652,14 +667,34 @@ fn write_out_deps( let mut file = BufWriter::new(fs::File::create(&deps_filename)?); for path in out_filenames { - writeln!(file, "{}: {}\n", path.display(), files.join(" "))?; + let line: Vec<&str> = files.iter().map(|(f, _)| f.as_ref()).collect(); + writeln!(file, "{}: {}\n", path.display(), line.join(" "))?; } // Emit a fake target for each input file to the compilation. This // prevents `make` from spitting out an error if a file is later // deleted. For more info see #28735 - for path in files { + for (path, file_hash) in files { writeln!(file, "{}:", path)?; + + match file_hash { + Some(FileHash::SourceFile(src_file)) => { + writeln!( + file, + "# size:{} {}:{:?}", + src_file.byte_length(), + src_file.src_hash.kind.to_string(), + src_file.src_hash.hash_bytes() + )?; + } + Some(FileHash::BinaryHash(size, md5_hash)) => { + writeln!(file, "# size:{} {}:{:?}", size, "md5", md5_hash)?; + } + Some(FileHash::HashInFilename(size)) => { + writeln!(file, "# size:{} {}:0", size, "hash_in_filename",)?; + } + None => {} + } } // Emit special comments with information about accessed environment variables. @@ -699,6 +734,37 @@ fn write_out_deps( } } +fn hash_bin(path: &PathBuf) -> io::Result { + let hash = md5_hash_file(path)?; + let mut fname = path.to_str().unwrap().to_owned(); + fname.push_str(".md5"); + let size = fs::metadata(path).unwrap().len(); //FIXME calc while hashing + fs::write(fname, &hash)?; + Ok(FileHash::BinaryHash(size, hash)) +} + +/// For files with svh in their filename we indicate that we should +/// trust the contents are correct without further checks. +fn hash_bin_with_svh(path: &PathBuf) -> io::Result { + let size = fs::metadata(path)?.len(); + Ok(FileHash::HashInFilename(size)) +} + +fn md5_hash_file(path: &PathBuf) -> io::Result> { + let mut reader = io::BufReader::new(fs::File::open(&path)?); + + let mut hasher = Md5::new(); + let mut buffer = [0; 1024]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.input(&buffer[..count]); + } + Ok(hasher.result().as_slice().to_vec()) +} + pub fn prepare_outputs( sess: &Session, compiler: &Compiler, @@ -1009,11 +1075,11 @@ fn encode_and_write_metadata( .crate_types() .iter() .map(|ty| match *ty { - CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None, + CrateType::Executable | CrateType::Staticlib => MetadataKind::None, CrateType::Rlib => MetadataKind::Uncompressed, - CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, + CrateType::Cdylib | CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, }) .max() .unwrap_or(MetadataKind::None); @@ -1039,7 +1105,16 @@ fn encode_and_write_metadata( .tempdir_in(out_filename.parent().unwrap()) .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); - let metadata_filename = emit_metadata(tcx.sess, &metadata, &metadata_tmpdir); + + let crate_name = &tcx.crate_name(LOCAL_CRATE).as_str(); + + let metadata_filename = emit_metadata( + tcx.sess, + &metadata, + &metadata_tmpdir, + &crate_name, + &tcx.crate_hash(LOCAL_CRATE), + ); if let Err(e) = fs::rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 7e33a479228ba..674b2cb36484c 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -19,7 +19,7 @@ use rustc_middle::hir::map::Map; use rustc_middle::middle::cstore::{EncodedMetadata, ForeignModule, LinkagePreference, NativeLib}; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::{ - metadata_symbol_name, ExportedSymbol, SymbolExportLevel, + metadata_symbol_name, svh_symbol_name, ExportedSymbol, SymbolExportLevel, }; use rustc_middle::mir::interpret; use rustc_middle::traits::specialization_graph; @@ -1669,12 +1669,15 @@ impl EncodeContext<'a, 'tcx> { // The metadata symbol name is special. It should not show up in // downstream crates. let metadata_symbol_name = SymbolName::new(self.tcx, &metadata_symbol_name(self.tcx)); + let svh_symbol_name = SymbolName::new(self.tcx, &svh_symbol_name(self.tcx)); self.lazy( exported_symbols .iter() .filter(|&&(ref exported_symbol, _)| match *exported_symbol { - ExportedSymbol::NoDefId(symbol_name) => symbol_name != metadata_symbol_name, + ExportedSymbol::NoDefId(symbol_name) => { + symbol_name != metadata_symbol_name && symbol_name != svh_symbol_name + } _ => true, }) .cloned(), diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 276e45ce99b29..ed8e4faac821d 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -53,3 +53,8 @@ pub fn metadata_symbol_name(tcx: TyCtxt<'_>) -> String { tcx.crate_disambiguator(LOCAL_CRATE).to_fingerprint().to_hex() ) } + +/// Embed the hash that represents the crate for use as a fingerprint. +pub fn svh_symbol_name(tcx: TyCtxt<'_>) -> String { + format!("rust_svh_{}_{}", tcx.original_crate_name(LOCAL_CRATE), tcx.crate_hash(LOCAL_CRATE)) +} diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 79363c3a5ca5c..5d28601850ce4 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -1036,6 +1036,15 @@ pub enum SourceFileHashAlgorithm { Sha1, } +impl ToString for SourceFileHashAlgorithm { + fn to_string(&self) -> String { + match self { + SourceFileHashAlgorithm::Md5 => "md5".to_string(), + SourceFileHashAlgorithm::Sha1 => "sha1".to_string(), + } + } +} + impl FromStr for SourceFileHashAlgorithm { type Err = (); From 71982896d347ac5c31ca1b120b827b490be96acd Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Thu, 29 Oct 2020 10:22:28 +0000 Subject: [PATCH 02/22] Speculative: adding metadata to cdylib --- compiler/rustc_codegen_ssa/src/back/link.rs | 7 ++++++- compiler/rustc_codegen_ssa/src/back/symbol_export.rs | 6 ++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index de6493777ce94..4bd06bd6399d9 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -1403,6 +1403,11 @@ fn add_local_crate_metadata_objects( { cmd.add_object(obj); } + } else if crate_type == CrateType::Cdylib { + if let Some(obj) = codegen_results.metadata_module.as_ref().and_then(|m| m.object.as_ref()) + { + cmd.add_object(obj); + } } } @@ -1432,7 +1437,7 @@ fn link_local_crate_native_libs_and_dependent_crate_libs<'a, B: ArchiveBuilder<' // // The rationale behind this ordering is that those items lower down in the // list can't depend on items higher up in the list. For example nothing can - // depend on what we just generated (e.g., that'd be a circular dependency). + // depend on what we just generated (that'd be a circular dependency). // Upstream rust libraries are not allowed to depend on our local native // libraries as that would violate the structure of the DAG, in that // scenario they are required to link to them as well in a shared fashion. diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 1861dda81cd28..bf5038fd2bbfe 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -23,10 +23,8 @@ pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { fn crate_export_threshold(crate_type: CrateType) -> SymbolExportLevel { match crate_type { - CrateType::Executable | CrateType::Staticlib | CrateType::ProcMacro | CrateType::Cdylib => { - SymbolExportLevel::C - } - CrateType::Rlib | CrateType::Dylib => SymbolExportLevel::Rust, + CrateType::Executable | CrateType::Staticlib | CrateType::ProcMacro => SymbolExportLevel::C, + CrateType::Rlib | CrateType::Dylib | CrateType::Cdylib => SymbolExportLevel::Rust, } } From 0bc53ea2d901ec156c485656660a7374a0182e9b Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Thu, 29 Oct 2020 16:39:20 +0000 Subject: [PATCH 03/22] update cargo lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 65d20190c0db5..d7d669903b74c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3763,6 +3763,7 @@ name = "rustc_interface" version = "0.0.0" dependencies = [ "libc", + "md-5", "rustc-rayon", "rustc_ast", "rustc_ast_lowering", From f5322e91fb72cdbf8862b204b7464ded270795c6 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Thu, 29 Oct 2020 17:35:30 +0000 Subject: [PATCH 04/22] Popped crate_hash back for now, to get back to a working state. --- compiler/rustc_codegen_ssa/src/back/write.rs | 7 +++++-- compiler/rustc_codegen_ssa/src/lib.rs | 2 ++ compiler/rustc_interface/src/passes.rs | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 4d2cea18dcc68..5aa193aa35f75 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -13,6 +13,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::profiling::TimingGuard; use rustc_data_structures::profiling::VerboseTimingGuard; +use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::Lrc; use rustc_errors::emitter::Emitter; use rustc_errors::{DiagnosticId, FatalError, Handler, Level}; @@ -413,6 +414,7 @@ pub fn start_async_codegen( let sess = tcx.sess; let crate_name = tcx.crate_name(LOCAL_CRATE); + let crate_hash = tcx.crate_hash(LOCAL_CRATE); let no_builtins = tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::no_builtins); let is_compiler_builtins = tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::compiler_builtins); @@ -465,7 +467,7 @@ pub fn start_async_codegen( windows_subsystem, linker_info, crate_info, - + crate_hash, coordinator_send, codegen_worker_receive, shared_emitter_main, @@ -1710,6 +1712,7 @@ pub struct OngoingCodegen { pub windows_subsystem: Option, pub linker_info: LinkerInfo, pub crate_info: CrateInfo, + pub crate_hash: Svh, pub coordinator_send: Sender>, pub codegen_worker_receive: Receiver>, pub shared_emitter_main: SharedEmitterMain, @@ -1755,7 +1758,7 @@ impl OngoingCodegen { windows_subsystem: self.windows_subsystem, linker_info: self.linker_info, crate_info: self.crate_info, - + crate_hash: self.crate_hash, modules: compiled_modules.modules, allocator_module: compiled_modules.allocator_module, metadata_module: compiled_modules.metadata_module, diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 46bf49aa8bb2c..05aad6f205252 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -21,6 +21,7 @@ extern crate tracing; extern crate rustc_middle; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::CrateNum; use rustc_hir::LangItem; @@ -137,6 +138,7 @@ pub struct CodegenResults { pub windows_subsystem: Option, pub linker_info: back::linker::LinkerInfo, pub crate_info: CrateInfo, + pub crate_hash: Svh, } pub fn provide(providers: &mut Providers) { diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index a82a1be6a9b18..e5216ef6eed4e 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -639,7 +639,7 @@ fn write_out_deps( .collect(); if let Some(ref backend) = sess.opts.debugging_opts.codegen_backend { - files.push(backend.to_string()); + files.push((backend.to_string(), None)); // TO DO: is this something we need to hash? } if sess.binary_dep_depinfo() { From 243928ba2bb599bd0d0d01fe624484b7cfd603c2 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Fri, 30 Oct 2020 15:40:34 +0000 Subject: [PATCH 05/22] Revert "Speculative: adding metadata to cdylib" This reverts commit 71982896d347ac5c31ca1b120b827b490be96acd. --- compiler/rustc_codegen_ssa/src/back/link.rs | 7 +------ compiler/rustc_codegen_ssa/src/back/symbol_export.rs | 6 ++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 4bd06bd6399d9..de6493777ce94 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -1403,11 +1403,6 @@ fn add_local_crate_metadata_objects( { cmd.add_object(obj); } - } else if crate_type == CrateType::Cdylib { - if let Some(obj) = codegen_results.metadata_module.as_ref().and_then(|m| m.object.as_ref()) - { - cmd.add_object(obj); - } } } @@ -1437,7 +1432,7 @@ fn link_local_crate_native_libs_and_dependent_crate_libs<'a, B: ArchiveBuilder<' // // The rationale behind this ordering is that those items lower down in the // list can't depend on items higher up in the list. For example nothing can - // depend on what we just generated (that'd be a circular dependency). + // depend on what we just generated (e.g., that'd be a circular dependency). // Upstream rust libraries are not allowed to depend on our local native // libraries as that would violate the structure of the DAG, in that // scenario they are required to link to them as well in a shared fashion. diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index bf5038fd2bbfe..1861dda81cd28 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -23,8 +23,10 @@ pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { fn crate_export_threshold(crate_type: CrateType) -> SymbolExportLevel { match crate_type { - CrateType::Executable | CrateType::Staticlib | CrateType::ProcMacro => SymbolExportLevel::C, - CrateType::Rlib | CrateType::Dylib | CrateType::Cdylib => SymbolExportLevel::Rust, + CrateType::Executable | CrateType::Staticlib | CrateType::ProcMacro | CrateType::Cdylib => { + SymbolExportLevel::C + } + CrateType::Rlib | CrateType::Dylib => SymbolExportLevel::Rust, } } From 3a323306c8c1c9d3b2c3a7cbca67d3b8f863d1aa Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Fri, 30 Oct 2020 17:04:04 +0000 Subject: [PATCH 06/22] Put crate SVHes in .d dependency file --- Cargo.lock | 1 - compiler/rustc_interface/Cargo.toml | 1 - compiler/rustc_interface/src/passes.rs | 77 +++++++++++++------------- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7d669903b74c..65d20190c0db5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3763,7 +3763,6 @@ name = "rustc_interface" version = "0.0.0" dependencies = [ "libc", - "md-5", "rustc-rayon", "rustc_ast", "rustc_ast_lowering", diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index 42dc84f7fa73e..e214493a567b8 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -43,7 +43,6 @@ rustc_resolve = { path = "../rustc_resolve" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_ty = { path = "../rustc_ty" } tempfile = "3.0.5" -md-5 = "0.8" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["libloaderapi"] } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index e5216ef6eed4e..c629c9e47d920 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -7,6 +7,7 @@ use rustc_ast::ptr::P; use rustc_ast::{self as ast, token, visit}; use rustc_codegen_ssa::back::link::emit_metadata; use rustc_codegen_ssa::traits::CodegenBackend; +use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal}; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_data_structures::{box_region_allow_access, declare_box_region_type, parallel}; @@ -40,7 +41,7 @@ use rustc_typeck as typeck; use smallvec::SmallVec; use tracing::{info, warn}; -use md5::{Digest, Md5}; +//use md5::{Digest, Md5}; use rustc_serialize::json; use tempfile::Builder as TempFileBuilder; @@ -48,7 +49,7 @@ use tempfile::Builder as TempFileBuilder; use std::any::Any; use std::cell::RefCell; use std::ffi::OsString; -use std::io::{self, BufWriter, Read, Write}; +use std::io::{self, BufWriter, Write}; use std::lazy::SyncLazy; use std::path::PathBuf; use std::rc::Rc; @@ -606,8 +607,8 @@ fn escape_dep_env(symbol: Symbol) -> String { enum FileHash { SourceFile(Lrc), - BinaryHash(u64, Vec), - HashInFilename(u64), + BinaryHash(u64, String), + //HashInFilename(u64), } fn write_out_deps( @@ -646,18 +647,19 @@ fn write_out_deps( boxed_resolver.borrow().borrow_mut().access(|resolver| { for cnum in resolver.cstore().crates_untracked() { let source = resolver.cstore().crate_source_untracked(cnum); + let svh = resolver.cstore().crate_hash_untracked(cnum); if let Some((path, _)) = source.dylib { - let hash = hash_bin(&path).ok(); + let hash = hash_bin(&path, &svh).ok(); let file_name = FileName::Real(RealFileName::Named(path)); files.push((escape_dep_filename(&file_name), hash)); } if let Some((path, _)) = source.rlib { - let hash = hash_bin_with_svh(&path).ok(); + let hash = hash_bin(&path, &svh).ok(); let file_name = FileName::Real(RealFileName::Named(path)); files.push((escape_dep_filename(&file_name), hash)); } if let Some((path, _)) = source.rmeta { - let hash = hash_bin_with_svh(&path).ok(); + let hash = hash_bin(&path, &svh).ok(); let file_name = FileName::Real(RealFileName::Named(path)); files.push((escape_dep_filename(&file_name), hash)); } @@ -687,12 +689,12 @@ fn write_out_deps( src_file.src_hash.hash_bytes() )?; } - Some(FileHash::BinaryHash(size, md5_hash)) => { - writeln!(file, "# size:{} {}:{:?}", size, "md5", md5_hash)?; - } - Some(FileHash::HashInFilename(size)) => { - writeln!(file, "# size:{} {}:0", size, "hash_in_filename",)?; + Some(FileHash::BinaryHash(size, svh)) => { + writeln!(file, "# size:{} {}:{}", size, "svh", svh)?; } + // Some(FileHash::HashInFilename(size)) => { + // writeln!(file, "# size:{} {}:0", size, "hash_in_filename",)?; + // } None => {} } } @@ -734,36 +736,35 @@ fn write_out_deps( } } -fn hash_bin(path: &PathBuf) -> io::Result { - let hash = md5_hash_file(path)?; - let mut fname = path.to_str().unwrap().to_owned(); - fname.push_str(".md5"); - let size = fs::metadata(path).unwrap().len(); //FIXME calc while hashing - fs::write(fname, &hash)?; - Ok(FileHash::BinaryHash(size, hash)) +fn hash_bin(path: &PathBuf, svh: &Svh) -> io::Result { + let size = fs::metadata(path)?.len(); + let svh_str = format!("{}", svh); + Ok(FileHash::BinaryHash(size, svh_str)) } /// For files with svh in their filename we indicate that we should /// trust the contents are correct without further checks. -fn hash_bin_with_svh(path: &PathBuf) -> io::Result { - let size = fs::metadata(path)?.len(); - Ok(FileHash::HashInFilename(size)) -} - -fn md5_hash_file(path: &PathBuf) -> io::Result> { - let mut reader = io::BufReader::new(fs::File::open(&path)?); - - let mut hasher = Md5::new(); - let mut buffer = [0; 1024]; - loop { - let count = reader.read(&mut buffer)?; - if count == 0 { - break; - } - hasher.input(&buffer[..count]); - } - Ok(hasher.result().as_slice().to_vec()) -} +/// FIXME: is that really an svh in the filename - that's potentially overridden by an arg. +/// not sure we can trust it. +// fn hash_bin_with_svh(path: &PathBuf) -> io::Result { +// let size = fs::metadata(path)?.len(); +// Ok(FileHash::HashInFilename(size)) +// } + +// fn md5_hash_file(path: &PathBuf) -> io::Result> { +// let mut reader = io::BufReader::new(fs::File::open(&path)?); + +// let mut hasher = Md5::new(); +// let mut buffer = [0; 1024]; +// loop { +// let count = reader.read(&mut buffer)?; +// if count == 0 { +// break; +// } +// hasher.input(&buffer[..count]); +// } +// Ok(hasher.result().as_slice().to_vec()) +// } pub fn prepare_outputs( sess: &Session, From 4268215ba91df3fbda7a275817c8f79f0b0d3243 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sat, 31 Oct 2020 11:50:12 +0000 Subject: [PATCH 07/22] Use hex for src hashes --- compiler/rustc_interface/src/passes.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index c629c9e47d920..bb9c9056b8cc6 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -681,20 +681,22 @@ fn write_out_deps( match file_hash { Some(FileHash::SourceFile(src_file)) => { + let bytes = src_file.src_hash.hash_bytes(); + let mut hash_as_hex = String::with_capacity(bytes.len() * 2); + for byte in bytes { + hash_as_hex.push_str(&format!("{:x}", byte)); + } writeln!( file, - "# size:{} {}:{:?}", + "# size:{} {}:{}", src_file.byte_length(), src_file.src_hash.kind.to_string(), - src_file.src_hash.hash_bytes() + hash_as_hex )?; } Some(FileHash::BinaryHash(size, svh)) => { writeln!(file, "# size:{} {}:{}", size, "svh", svh)?; } - // Some(FileHash::HashInFilename(size)) => { - // writeln!(file, "# size:{} {}:0", size, "hash_in_filename",)?; - // } None => {} } } From 5dac651cae584c0735a3d075c6948aa27460e7c7 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sat, 31 Oct 2020 17:32:34 +0000 Subject: [PATCH 08/22] initial support for cranelift --- .../rustc_codegen_cranelift/src/archive.rs | 47 ++++----------- .../rustc_codegen_cranelift/src/driver/aot.rs | 60 +++++-------------- 2 files changed, 26 insertions(+), 81 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs index 6382f8df3446b..c726d5c88b29d 100644 --- a/compiler/rustc_codegen_cranelift/src/archive.rs +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -5,17 +5,14 @@ use std::fs::File; use std::path::{Path, PathBuf}; use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder}; -use rustc_codegen_ssa::METADATA_FILENAME; +use rustc_codegen_ssa::{METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX}; use rustc_session::Session; use object::{Object, SymbolKind}; #[derive(Debug)] enum ArchiveEntry { - FromArchive { - archive_index: usize, - entry_index: usize, - }, + FromArchive { archive_index: usize, entry_index: usize }, File(PathBuf), } @@ -46,10 +43,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { let entry = entry.unwrap(); entries.push(( String::from_utf8(entry.header().identifier().to_vec()).unwrap(), - ArchiveEntry::FromArchive { - archive_index: 0, - entry_index: i, - }, + ArchiveEntry::FromArchive { archive_index: 0, entry_index: i }, )); i += 1; } @@ -95,14 +89,9 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) { let location = find_library(name, &self.lib_search_paths, self.sess); - self.add_archive(location.clone(), |_| false) - .unwrap_or_else(|e| { - panic!( - "failed to add native library {}: {}", - location.to_string_lossy(), - e - ); - }); + self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| { + panic!("failed to add native library {}: {}", location.to_string_lossy(), e); + }); } fn add_rlib( @@ -116,7 +105,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { self.add_archive(rlib.to_owned(), move |fname: &str| { // Ignore metadata files, no matter the name. - if fname == METADATA_FILENAME { + if fname.starts_with(METADATA_FILE_PREFIX) && fname.ends_with(METADATA_FILE_EXTENSION) { return true; } @@ -156,10 +145,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { // FIXME only read the symbol table of the object files to avoid having to keep all // object files in memory at once, or read them twice. let data = match entry { - ArchiveEntry::FromArchive { - archive_index, - entry_index, - } => { + ArchiveEntry::FromArchive { archive_index, entry_index } => { // FIXME read symbols from symtab use std::io::Read; let (ref _src_archive_path, ref mut src_archive) = @@ -225,10 +211,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { err )); }), - entries - .iter() - .map(|(name, _)| name.as_bytes().to_vec()) - .collect(), + entries.iter().map(|(name, _)| name.as_bytes().to_vec()).collect(), ar::GnuSymbolTableFormat::Size32, symbol_table, ) @@ -271,8 +254,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { .expect("Couldn't run ranlib"); if !status.success() { - self.sess - .fatal(&format!("Ranlib exited with code {:?}", status.code())); + self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code())); } } } @@ -292,13 +274,8 @@ impl<'a> ArArchiveBuilder<'a> { let file_name = String::from_utf8(entry.header().identifier().to_vec()) .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; if !skip(&file_name) { - self.entries.push(( - file_name, - ArchiveEntry::FromArchive { - archive_index, - entry_index: i, - }, - )); + self.entries + .push((file_name, ArchiveEntry::FromArchive { archive_index, entry_index: i })); } i += 1; } diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index ff0b994c9a9f4..61c91406bf5bb 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -51,13 +51,10 @@ fn emit_module( let product = map_product(product); - let tmp_file = tcx - .output_filenames(LOCAL_CRATE) - .temp_path(OutputType::Object, Some(&name)); + let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name)); let obj = product.object.write().unwrap(); if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess - .fatal(&format!("error writing object file: {}", err)); + tcx.sess.fatal(&format!("error writing object file: {}", err)); } let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { @@ -71,12 +68,7 @@ fn emit_module( }; ModuleCodegenResult( - CompiledModule { - name, - kind, - object: Some(tmp_file), - bytecode: None, - }, + CompiledModule { name, kind, object: Some(tmp_file), bytecode: None }, work_product, ) } @@ -124,11 +116,7 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege // Initialize the global atomic mutex using a constructor for proc-macros. // FIXME implement atomic instructions in Cranelift. let mut init_atomics_mutex_from_constructor = None; - if tcx - .sess - .crate_types() - .contains(&rustc_session::config::CrateType::ProcMacro) - { + if tcx.sess.crate_types().contains(&rustc_session::config::CrateType::ProcMacro) { if mono_items.iter().any(|(mono_item, _)| match mono_item { rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx .symbol_name(Instance::mono(tcx, *def_id)) @@ -196,9 +184,7 @@ pub(super) fn run_aot( cgus.iter() .map(|cgu| { let cgu_reuse = determine_cgu_reuse(tcx, cgu); - tcx.sess - .cgu_reuse_tracker - .set_actual_reuse(&cgu.name().as_str(), cgu_reuse); + tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); match cgu_reuse { _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} @@ -275,8 +261,7 @@ pub(super) fn run_aot( }); if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess - .fatal(&format!("error writing metadata object file: {}", err)); + tcx.sess.fatal(&format!("error writing metadata object file: {}", err)); } (metadata_cgu_name, tmp_file) @@ -306,6 +291,7 @@ pub(super) fn run_aot( windows_subsystem: None, // Windows is not yet supported linker_info: LinkerInfo::new(tcx), crate_info: CrateInfo::new(tcx), + crate_hash: tcx.crate_hash(LOCAL_CRATE), }, work_products, )) @@ -333,8 +319,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", ); } else { - tcx.sess - .fatal("asm! and global_asm! are not yet supported on macOS and Windows"); + tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); } } @@ -344,19 +329,12 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { // Remove all LLVM style comments let global_asm = global_asm .lines() - .map(|line| { - if let Some(index) = line.find("//") { - &line[0..index] - } else { - line - } - }) + .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) .collect::>() .join("\n"); - let output_object_file = tcx - .output_filenames(LOCAL_CRATE) - .temp_path(OutputType::Object, Some(cgu_name)); + let output_object_file = + tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name)); // Assemble `global_asm` let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); @@ -366,16 +344,10 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { .stdin(Stdio::piped()) .spawn() .expect("Failed to spawn `as`."); - child - .stdin - .take() - .unwrap() - .write_all(global_asm.as_bytes()) - .unwrap(); + child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); let status = child.wait().expect("Failed to wait for `as`."); if !status.success() { - tcx.sess - .fatal(&format!("Failed to assemble `{}`", global_asm)); + tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); } // Link the global asm and main object file together @@ -419,11 +391,7 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR } let work_product_id = &cgu.work_product_id(); - if tcx - .dep_graph - .previous_work_product(work_product_id) - .is_none() - { + if tcx.dep_graph.previous_work_product(work_product_id).is_none() { // We don't have anything cached for this CGU. This can happen // if the CGU did not exist in the previous session. return CguReuse::No; From b23f5310bf382b3ef2a8fa523540456d4ea3d9cd Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 1 Nov 2020 11:08:09 +0000 Subject: [PATCH 09/22] support cranelift --- compiler/rustc_codegen_cranelift/src/metadata.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs index 04369bf89fd2d..38095c29f77e2 100644 --- a/compiler/rustc_codegen_cranelift/src/metadata.rs +++ b/compiler/rustc_codegen_cranelift/src/metadata.rs @@ -4,7 +4,7 @@ use std::convert::TryFrom; use std::fs::File; use std::path::Path; -use rustc_codegen_ssa::METADATA_FILENAME; +use rustc_codegen_ssa::{METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX}; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; @@ -23,7 +23,10 @@ impl MetadataLoader for CraneliftMetadataLoader { // Iterate over all entries in the archive: while let Some(entry_result) = archive.next_entry() { let mut entry = entry_result.map_err(|e| format!("{:?}", e))?; - if entry.header().identifier() == METADATA_FILENAME.as_bytes() { + let filename = String::from_utf8_lossy(entry.header().identifier()); + if filename.starts_with(METADATA_FILE_PREFIX) + && filename.ends_with(METADATA_FILE_EXTENSION) + { let mut buf = Vec::with_capacity( usize::try_from(entry.header().size()) .expect("Rlib metadata file too big to load into memory."), @@ -94,9 +97,7 @@ pub(crate) fn write_metadata( assert!(kind == MetadataKind::Compressed); let mut compressed = tcx.metadata_encoding_version(); - FrameEncoder::new(&mut compressed) - .write_all(&metadata.raw_data) - .unwrap(); + FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); product.add_rustc_section( rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), From 18fd2df0b1fa93fd5f54d17a0988137d1cab2349 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 1 Nov 2020 14:03:27 +0000 Subject: [PATCH 10/22] Add in svh to stable location in metadata. --- compiler/rustc_metadata/src/rmeta/encoder.rs | 3 +++ compiler/rustc_metadata/src/rmeta/mod.rs | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 674b2cb36484c..45b7aff773255 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2066,6 +2066,9 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { // Encode the rustc version string in a predictable location. rustc_version().encode(&mut ecx).unwrap(); + // Encode the crate's svh hash in a predictalbe location for cargo. + crate_svh(tcx).encode(&mut ecx).unwrap(); + // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 2bd2019d3cdb5..71235403ac88a 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -7,7 +7,7 @@ use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::MetadataRef; use rustc_hir as hir; use rustc_hir::def::CtorKind; -use rustc_hir::def_id::{DefId, DefIndex, DefPathHash}; +use rustc_hir::def_id::{DefId, DefIndex, DefPathHash, LOCAL_CRATE}; use rustc_hir::definitions::DefKey; use rustc_hir::lang_items; use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec}; @@ -15,7 +15,7 @@ use rustc_middle::hir::exports::Export; use rustc_middle::middle::cstore::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib}; use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel}; use rustc_middle::mir; -use rustc_middle::ty::{self, ReprOptions, Ty}; +use rustc_middle::ty::{self, ReprOptions, Ty, TyCtxt}; use rustc_serialize::opaque::Encoder; use rustc_session::config::SymbolManglingVersion; use rustc_session::CrateDisambiguator; @@ -41,6 +41,10 @@ crate fn rustc_version() -> String { format!("rustc {}", option_env!("CFG_VERSION").unwrap_or("unknown version")) } +crate fn crate_svh(tcx: TyCtxt<'_>) -> String { + tcx.crate_hash(LOCAL_CRATE).to_string() +} + /// Metadata encoding version. /// N.B., increment this if you change the format of metadata such that /// the rustc version can't be found to compare with `rustc_version()`. @@ -50,7 +54,7 @@ const METADATA_VERSION: u8 = 5; /// /// This header is followed by the position of the `CrateRoot`, /// which is encoded as a 32-bit big-endian unsigned integer, -/// and further followed by the rustc version string. +/// and further followed by the rustc version string and the svh. crate const METADATA_HEADER: &[u8; 8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION]; /// Additional metadata for a `Lazy` where `T` may not be `Sized`, From efb32e5b1a4c2553d0e15ebaec672a824c2200d5 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 1 Nov 2020 14:10:51 +0000 Subject: [PATCH 11/22] undoing cargo fmt --- .../rustc_codegen_cranelift/src/archive.rs | 43 +++++++++---- .../rustc_codegen_cranelift/src/driver/aot.rs | 60 ++++++++++++++----- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs index c726d5c88b29d..7add166188006 100644 --- a/compiler/rustc_codegen_cranelift/src/archive.rs +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -12,7 +12,10 @@ use object::{Object, SymbolKind}; #[derive(Debug)] enum ArchiveEntry { - FromArchive { archive_index: usize, entry_index: usize }, + FromArchive { + archive_index: usize, + entry_index: usize, + }, File(PathBuf), } @@ -43,7 +46,10 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { let entry = entry.unwrap(); entries.push(( String::from_utf8(entry.header().identifier().to_vec()).unwrap(), - ArchiveEntry::FromArchive { archive_index: 0, entry_index: i }, + ArchiveEntry::FromArchive { + archive_index: 0, + entry_index: i, + }, )); i += 1; } @@ -89,9 +95,14 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) { let location = find_library(name, &self.lib_search_paths, self.sess); - self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| { - panic!("failed to add native library {}: {}", location.to_string_lossy(), e); - }); + self.add_archive(location.clone(), |_| false) + .unwrap_or_else(|e| { + panic!( + "failed to add native library {}: {}", + location.to_string_lossy(), + e + ); + }); } fn add_rlib( @@ -145,7 +156,10 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { // FIXME only read the symbol table of the object files to avoid having to keep all // object files in memory at once, or read them twice. let data = match entry { - ArchiveEntry::FromArchive { archive_index, entry_index } => { + ArchiveEntry::FromArchive { + archive_index, + entry_index, + } => { // FIXME read symbols from symtab use std::io::Read; let (ref _src_archive_path, ref mut src_archive) = @@ -211,7 +225,10 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { err )); }), - entries.iter().map(|(name, _)| name.as_bytes().to_vec()).collect(), + entries + .iter() + .map(|(name, _)| name.as_bytes().to_vec()) + .collect(), ar::GnuSymbolTableFormat::Size32, symbol_table, ) @@ -254,7 +271,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { .expect("Couldn't run ranlib"); if !status.success() { - self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code())); + self.sess + .fatal(&format!("Ranlib exited with code {:?}", status.code())); } } } @@ -274,8 +292,13 @@ impl<'a> ArArchiveBuilder<'a> { let file_name = String::from_utf8(entry.header().identifier().to_vec()) .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; if !skip(&file_name) { - self.entries - .push((file_name, ArchiveEntry::FromArchive { archive_index, entry_index: i })); + self.entries.push(( + file_name, + ArchiveEntry::FromArchive { + archive_index, + entry_index: i, + }, + )); } i += 1; } diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 61c91406bf5bb..ff0b994c9a9f4 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -51,10 +51,13 @@ fn emit_module( let product = map_product(product); - let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name)); + let tmp_file = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Object, Some(&name)); let obj = product.object.write().unwrap(); if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing object file: {}", err)); + tcx.sess + .fatal(&format!("error writing object file: {}", err)); } let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { @@ -68,7 +71,12 @@ fn emit_module( }; ModuleCodegenResult( - CompiledModule { name, kind, object: Some(tmp_file), bytecode: None }, + CompiledModule { + name, + kind, + object: Some(tmp_file), + bytecode: None, + }, work_product, ) } @@ -116,7 +124,11 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege // Initialize the global atomic mutex using a constructor for proc-macros. // FIXME implement atomic instructions in Cranelift. let mut init_atomics_mutex_from_constructor = None; - if tcx.sess.crate_types().contains(&rustc_session::config::CrateType::ProcMacro) { + if tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::ProcMacro) + { if mono_items.iter().any(|(mono_item, _)| match mono_item { rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx .symbol_name(Instance::mono(tcx, *def_id)) @@ -184,7 +196,9 @@ pub(super) fn run_aot( cgus.iter() .map(|cgu| { let cgu_reuse = determine_cgu_reuse(tcx, cgu); - tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); + tcx.sess + .cgu_reuse_tracker + .set_actual_reuse(&cgu.name().as_str(), cgu_reuse); match cgu_reuse { _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} @@ -261,7 +275,8 @@ pub(super) fn run_aot( }); if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing metadata object file: {}", err)); + tcx.sess + .fatal(&format!("error writing metadata object file: {}", err)); } (metadata_cgu_name, tmp_file) @@ -291,7 +306,6 @@ pub(super) fn run_aot( windows_subsystem: None, // Windows is not yet supported linker_info: LinkerInfo::new(tcx), crate_info: CrateInfo::new(tcx), - crate_hash: tcx.crate_hash(LOCAL_CRATE), }, work_products, )) @@ -319,7 +333,8 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", ); } else { - tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); + tcx.sess + .fatal("asm! and global_asm! are not yet supported on macOS and Windows"); } } @@ -329,12 +344,19 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { // Remove all LLVM style comments let global_asm = global_asm .lines() - .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) + .map(|line| { + if let Some(index) = line.find("//") { + &line[0..index] + } else { + line + } + }) .collect::>() .join("\n"); - let output_object_file = - tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name)); + let output_object_file = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Object, Some(cgu_name)); // Assemble `global_asm` let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); @@ -344,10 +366,16 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { .stdin(Stdio::piped()) .spawn() .expect("Failed to spawn `as`."); - child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(global_asm.as_bytes()) + .unwrap(); let status = child.wait().expect("Failed to wait for `as`."); if !status.success() { - tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); + tcx.sess + .fatal(&format!("Failed to assemble `{}`", global_asm)); } // Link the global asm and main object file together @@ -391,7 +419,11 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR } let work_product_id = &cgu.work_product_id(); - if tcx.dep_graph.previous_work_product(work_product_id).is_none() { + if tcx + .dep_graph + .previous_work_product(work_product_id) + .is_none() + { // We don't have anything cached for this CGU. This can happen // if the CGU did not exist in the previous session. return CguReuse::No; From 4a1b9e040a7adb08d8a8eb0847957d16a59f3b4a Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 1 Nov 2020 14:22:34 +0000 Subject: [PATCH 12/22] Reducing diff --- compiler/rustc_codegen_llvm/src/base.rs | 1 - compiler/rustc_interface/src/passes.rs | 29 ++----------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 08580685720e3..5ffd187b5fbe6 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -43,7 +43,6 @@ use std::ffi::CString; use std::time::Instant; pub fn write_compressed_metadata<'tcx>( - // backend: &T, tcx: TyCtxt<'tcx>, metadata: &EncodedMetadata, llvm_module: &mut ModuleLlvm, diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index bb9c9056b8cc6..979f48d9a5533 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -608,7 +608,6 @@ fn escape_dep_env(symbol: Symbol) -> String { enum FileHash { SourceFile(Lrc), BinaryHash(u64, String), - //HashInFilename(u64), } fn write_out_deps( @@ -744,30 +743,6 @@ fn hash_bin(path: &PathBuf, svh: &Svh) -> io::Result { Ok(FileHash::BinaryHash(size, svh_str)) } -/// For files with svh in their filename we indicate that we should -/// trust the contents are correct without further checks. -/// FIXME: is that really an svh in the filename - that's potentially overridden by an arg. -/// not sure we can trust it. -// fn hash_bin_with_svh(path: &PathBuf) -> io::Result { -// let size = fs::metadata(path)?.len(); -// Ok(FileHash::HashInFilename(size)) -// } - -// fn md5_hash_file(path: &PathBuf) -> io::Result> { -// let mut reader = io::BufReader::new(fs::File::open(&path)?); - -// let mut hasher = Md5::new(); -// let mut buffer = [0; 1024]; -// loop { -// let count = reader.read(&mut buffer)?; -// if count == 0 { -// break; -// } -// hasher.input(&buffer[..count]); -// } -// Ok(hasher.result().as_slice().to_vec()) -// } - pub fn prepare_outputs( sess: &Session, compiler: &Compiler, @@ -1078,11 +1053,11 @@ fn encode_and_write_metadata( .crate_types() .iter() .map(|ty| match *ty { - CrateType::Executable | CrateType::Staticlib => MetadataKind::None, + CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None, CrateType::Rlib => MetadataKind::Uncompressed, - CrateType::Cdylib | CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, + CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, }) .max() .unwrap_or(MetadataKind::None); From e97953470d95b0302e5c99f9653c7f32e835ab21 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 1 Nov 2020 18:09:08 +0000 Subject: [PATCH 13/22] tidy --- compiler/rustc_codegen_ssa/src/back/link.rs | 3 +-- compiler/rustc_interface/src/passes.rs | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index de6493777ce94..aebdb33644384 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -367,8 +367,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( // not be correctly inferred once 'foo.o' is removed. // // Basically, all this means is that this code should not move above the - // code above. @audit rlib add here - + // code above. match flavor { RlibFlavor::Normal => { // Instead of putting the metadata in an object file section, rlibs diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 979f48d9a5533..29748e9eea7ba 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -41,8 +41,6 @@ use rustc_typeck as typeck; use smallvec::SmallVec; use tracing::{info, warn}; -//use md5::{Digest, Md5}; - use rustc_serialize::json; use tempfile::Builder as TempFileBuilder; From 21ad02652d69e97e9b1e1cae956a604f36df7d70 Mon Sep 17 00:00:00 2001 From: Squirrel Date: Fri, 13 Nov 2020 13:46:58 +0000 Subject: [PATCH 14/22] Apply suggestions from code review denser encoding format Co-authored-by: bjorn3 --- compiler/rustc_codegen_ssa/src/back/write.rs | 2 ++ compiler/rustc_interface/src/passes.rs | 2 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 2 +- compiler/rustc_metadata/src/rmeta/mod.rs | 4 +++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 5aa193aa35f75..43daabb55220d 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -468,6 +468,7 @@ pub fn start_async_codegen( linker_info, crate_info, crate_hash, + coordinator_send, codegen_worker_receive, shared_emitter_main, @@ -1759,6 +1760,7 @@ impl OngoingCodegen { linker_info: self.linker_info, crate_info: self.crate_info, crate_hash: self.crate_hash, + modules: compiled_modules.modules, allocator_module: compiled_modules.allocator_module, metadata_module: compiled_modules.metadata_module, diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 29748e9eea7ba..9d01dd86ed8b8 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -692,7 +692,7 @@ fn write_out_deps( )?; } Some(FileHash::BinaryHash(size, svh)) => { - writeln!(file, "# size:{} {}:{}", size, "svh", svh)?; + writeln!(file, "# size:{} svh:{}", size, svh)?; } None => {} } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 45b7aff773255..55377f1b03d45 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2066,7 +2066,7 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { // Encode the rustc version string in a predictable location. rustc_version().encode(&mut ecx).unwrap(); - // Encode the crate's svh hash in a predictalbe location for cargo. + // Encode the crate's svh hash in a predictable location for cargo. crate_svh(tcx).encode(&mut ecx).unwrap(); // Encode all the entries and extra information in the crate, diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 71235403ac88a..9ee849f094949 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -42,7 +42,9 @@ crate fn rustc_version() -> String { } crate fn crate_svh(tcx: TyCtxt<'_>) -> String { - tcx.crate_hash(LOCAL_CRATE).to_string() + let mut svh = [0; 64]; + svh.copy_from_slice(&tcx.crate_hash(LOCAL_CRATE).as_u64().to_le_bytes()); + svh } /// Metadata encoding version. From ced4bfb38c982b84f03c2a4f41a9a43f1dfd12bb Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sat, 14 Nov 2020 10:46:23 +0000 Subject: [PATCH 15/22] Work on non-osx platforms --- .../rustc_codegen_cranelift/src/driver/aot.rs | 1 + compiler/rustc_codegen_llvm/src/base.rs | 5 ++--- compiler/rustc_span/src/lib.rs | 17 ++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index ff0b994c9a9f4..efb9aeb6f6d76 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -299,6 +299,7 @@ pub(super) fn run_aot( Box::new(( CodegenResults { crate_name: tcx.crate_name(LOCAL_CRATE), + crate_hash: tcx.crate_hash(LOCAL_CRATE), modules, allocator_module, metadata_module, diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5ffd187b5fbe6..a4e387668cb76 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -117,10 +117,9 @@ fn add_svh(tcx: TyCtxt<'tcx>, llvm_module: &'ll ModuleLlvm, svh: &Svh) -> &'ll V unsafe { llvm::LLVMAddGlobal(metadata_llmod, common::val_ty(llconst), buf.as_ptr()) }; unsafe { llvm::LLVMSetInitializer(llglobal, llconst); + let section_name = + if tcx.sess.target.options.is_like_osx { "__DATA,.rust_svh" } else { ".rust_svh" }; - let section_name = format!( - "__DATA,.rust_svh_hash", //TO DO make work for non-OSX not ,no_dead_strip - ); let name = SmallCStr::new(§ion_name); llvm::LLVMSetSection(llglobal, name.as_ptr()); diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index 5d28601850ce4..ace24940cb144 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -47,19 +47,18 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{Lock, Lrc}; +use md5::Md5; +use sha1::Digest; +use sha1::Sha1; use std::borrow::Cow; use std::cell::RefCell; use std::cmp::{self, Ordering}; -use std::fmt; +use std::fmt::{self, Display}; use std::hash::Hash; use std::ops::{Add, Sub}; use std::path::{Path, PathBuf}; use std::str::FromStr; -use md5::Md5; -use sha1::Digest; -use sha1::Sha1; - use tracing::debug; #[cfg(test)] @@ -1036,11 +1035,11 @@ pub enum SourceFileHashAlgorithm { Sha1, } -impl ToString for SourceFileHashAlgorithm { - fn to_string(&self) -> String { +impl Display for SourceFileHashAlgorithm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - SourceFileHashAlgorithm::Md5 => "md5".to_string(), - SourceFileHashAlgorithm::Sha1 => "sha1".to_string(), + SourceFileHashAlgorithm::Md5 => write!(f, "md5"), + SourceFileHashAlgorithm::Sha1 => write!(f, "sha1"), } } } From b3452a4b271bfcf7df23a4db7ba7f535ef5f0cd8 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sat, 14 Nov 2020 17:55:22 +0000 Subject: [PATCH 16/22] svh moved before version and metadata version bumped --- compiler/rustc_metadata/src/locator.rs | 3 +++ compiler/rustc_metadata/src/rmeta/decoder.rs | 25 ++++++++++++++++++-- compiler/rustc_metadata/src/rmeta/encoder.rs | 5 ++-- compiler/rustc_metadata/src/rmeta/mod.rs | 9 +++---- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs index f225f8acc89ff..2feecb673a166 100644 --- a/compiler/rustc_metadata/src/locator.rs +++ b/compiler/rustc_metadata/src/locator.rs @@ -650,6 +650,9 @@ impl<'a> CrateLocator<'a> { } let hash = root.hash(); + assert_eq!(Some(hash), metadata.get_svh()); + // FIXME: we could move this earlier and check against the + // metadata hash as we could reject without decompressing the metadata blob. if let Some(expected_hash) = self.hash { if hash != expected_hash { info!("Rejecting via hash: expected {} got {}", expected_hash, hash); diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index b01a55b48da66..071c11c8fd584 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -594,15 +594,36 @@ impl MetadataBlob { } crate fn get_rustc_version(&self) -> String { - Lazy::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap()) + if self.is_compatible() { + Lazy::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4 + 1 + mem::size_of::<[u8;64]>() ).unwrap()) + .decode(self) + } else { + // Assume older... + Lazy::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap()) .decode(self) + } + } + + //used only by an assert? + crate fn get_svh(&self) -> Option { + if !self.is_compatible() { + return None; + } + let hash = + Lazy::<[u8; 64]>::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap()) + .decode(self); + + // Currently only the first 8 bytes are in use of the svh + let mut smol_hash = [0_u8; 8]; + smol_hash.copy_from_slice(&hash[..8]); + Some(Svh::new(u64::from_le_bytes(smol_hash))) } crate fn get_root(&self) -> CrateRoot<'tcx> { let slice = self.raw_bytes(); let offset = METADATA_HEADER.len(); let pos = (((slice[offset + 0] as u32) << 24) - | ((slice[offset + 1] as u32) << 16) + | ((slice[offset + 1] as u32) << 16)//TO DO replace with just one function call? | ((slice[offset + 2] as u32) << 8) | ((slice[offset + 3] as u32) << 0)) as usize; Lazy::>::from_position(NonZeroUsize::new(pos).unwrap()).decode(self) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 55377f1b03d45..3b036e4b3cf42 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2063,11 +2063,12 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { }; drop(source_map_files); + // Encode the crate's svh hash in a predictable location for cargo. + crate_svh(tcx).encode(&mut ecx).unwrap(); + // Encode the rustc version string in a predictable location. rustc_version().encode(&mut ecx).unwrap(); - // Encode the crate's svh hash in a predictable location for cargo. - crate_svh(tcx).encode(&mut ecx).unwrap(); // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 9ee849f094949..3d94fccede6f5 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -41,16 +41,17 @@ crate fn rustc_version() -> String { format!("rustc {}", option_env!("CFG_VERSION").unwrap_or("unknown version")) } -crate fn crate_svh(tcx: TyCtxt<'_>) -> String { - let mut svh = [0; 64]; - svh.copy_from_slice(&tcx.crate_hash(LOCAL_CRATE).as_u64().to_le_bytes()); +crate fn crate_svh(tcx: TyCtxt<'_>) -> [u8; 64] { + let mut svh = [0; 64]; // 512bits for future expansion of svh size. + let svh_front = &mut svh[0..8]; + svh_front.copy_from_slice(&tcx.crate_hash(LOCAL_CRATE).as_u64().to_le_bytes()); svh } /// Metadata encoding version. /// N.B., increment this if you change the format of metadata such that /// the rustc version can't be found to compare with `rustc_version()`. -const METADATA_VERSION: u8 = 5; +const METADATA_VERSION: u8 = 6; /// Metadata header which includes `METADATA_VERSION`. /// From aaccda244d0369cb5cdb7f02b8e45ffeca22e22a Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 15 Nov 2020 06:44:39 +0000 Subject: [PATCH 17/22] EXEs can't be dependend on so no need currently to add an svh in them. --- compiler/rustc_codegen_llvm/src/base.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index a4e387668cb76..ad46a11285cc2 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -173,11 +173,6 @@ pub fn compile_codegen_unit( // If this codegen unit contains the main function, also create the // wrapper here if let Some(entry) = maybe_create_entry_wrapper::>(&cx) { - let svh = tcx.crate_hash(LOCAL_CRATE); - - let llglobal = add_svh(tcx, &llvm_module, &svh); //&mut cx, - cx.add_used_global(llglobal); - attributes::sanitize(&cx, SanitizerSet::empty(), entry); } From e4c08f3ec4fa899ce3aff17d685a7a67f93ff3da Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 15 Nov 2020 12:58:38 +0000 Subject: [PATCH 18/22] Fixup following introduction of sha256 --- compiler/rustc_span/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index bcad0cd51faf6..3fb9a06286118 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -47,9 +47,6 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{Lock, Lrc}; -use md5::Md5; -use sha1::Digest; -use sha1::Sha1; use std::borrow::Cow; use std::cell::RefCell; use std::cmp::{self, Ordering}; @@ -1046,6 +1043,7 @@ impl Display for SourceFileHashAlgorithm { match self { SourceFileHashAlgorithm::Md5 => write!(f, "md5"), SourceFileHashAlgorithm::Sha1 => write!(f, "sha1"), + SourceFileHashAlgorithm::Sha256 => write!(f, "sha256"), } } } From a9c9e6b456fb416e566d312c8f9dfadfc63eea1e Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 15 Nov 2020 14:03:27 +0000 Subject: [PATCH 19/22] cargo fmt --- compiler/rustc_metadata/src/rmeta/decoder.rs | 9 ++++++--- compiler/rustc_metadata/src/rmeta/encoder.rs | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 1df30d981c553..993eee6c12978 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -595,12 +595,15 @@ impl MetadataBlob { crate fn get_rustc_version(&self) -> String { if self.is_compatible() { - Lazy::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4 + 1 + mem::size_of::<[u8;64]>() ).unwrap()) - .decode(self) + Lazy::::from_position( + NonZeroUsize::new(METADATA_HEADER.len() + 4 + 1 + mem::size_of::<[u8; 64]>()) + .unwrap(), + ) + .decode(self) } else { // Assume older... Lazy::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap()) - .decode(self) + .decode(self) } } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index f3e8d035139be..83f9f3f7d8332 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2072,7 +2072,6 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { // Encode the rustc version string in a predictable location. rustc_version().encode(&mut ecx).unwrap(); - // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); From 5021fd79b7f7ab2c9316fe9f298891e2637beb4a Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Sun, 15 Nov 2020 20:47:51 +0000 Subject: [PATCH 20/22] consistent ordering --- compiler/rustc_codegen_ssa/src/back/write.rs | 2 +- compiler/rustc_codegen_ssa/src/lib.rs | 2 +- .../run-make-fulldeps/hotplug_codegen_backend/the_backend.rs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 7126ec0cf4a37..fe0b2fea57e82 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -1757,11 +1757,11 @@ impl OngoingCodegen { ( CodegenResults { crate_name: self.crate_name, + crate_hash: self.crate_hash, metadata: self.metadata, windows_subsystem: self.windows_subsystem, linker_info: self.linker_info, crate_info: self.crate_info, - crate_hash: self.crate_hash, modules: compiled_modules.modules, allocator_module: compiled_modules.allocator_module, diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 05aad6f205252..d34487bc319ab 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -131,6 +131,7 @@ pub struct CrateInfo { #[derive(Encodable, Decodable)] pub struct CodegenResults { pub crate_name: Symbol, + pub crate_hash: Svh, pub modules: Vec, pub allocator_module: Option, pub metadata_module: Option, @@ -138,7 +139,6 @@ pub struct CodegenResults { pub windows_subsystem: Option, pub linker_info: back::linker::LinkerInfo, pub crate_info: CrateInfo, - pub crate_hash: Svh, } pub fn provide(providers: &mut Providers) { diff --git a/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs b/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs index 0e1bef6f68d53..860d2c9991b3d 100644 --- a/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs +++ b/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs @@ -60,6 +60,7 @@ impl CodegenBackend for TheBackend { Box::new(CodegenResults { crate_name: tcx.crate_name(LOCAL_CRATE), + crate_hash: tcx.crate_hash(LOCAL_CRATE), modules: vec![], allocator_module: None, metadata_module: None, From 1b3c8c0b08cf6081fcfa05fc14ebc79cd26cc325 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Wed, 18 Nov 2020 06:39:48 +0000 Subject: [PATCH 21/22] renamed fn --- compiler/rustc_codegen_llvm/src/base.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index ad46a11285cc2..62cadb94b1bd2 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -74,7 +74,7 @@ pub fn write_compressed_metadata<'tcx>( } let svh = tcx.crate_hash(LOCAL_CRATE); - let _llglobal = add_svh(tcx, &llvm_module, &svh); + add_svh_symbol(tcx, &llvm_module, &svh); } pub struct ValueIter<'ll> { @@ -105,8 +105,7 @@ pub fn svh_symbol_name( format!("rust_svh_{}_{}", tcx.original_crate_name(LOCAL_CRATE), svh) } -fn add_svh(tcx: TyCtxt<'tcx>, llvm_module: &'ll ModuleLlvm, svh: &Svh) -> &'ll Value { - // Add SVH @todo insert symbol here +fn add_svh_symbol(tcx: TyCtxt<'tcx>, llvm_module: &'ll ModuleLlvm, svh: &Svh) { let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod()); let svh_bytes: Vec = format!("{}", svh).as_bytes().to_vec(); @@ -133,7 +132,6 @@ fn add_svh(tcx: TyCtxt<'tcx>, llvm_module: &'ll ModuleLlvm, svh: &Svh) -> &'ll V directive.len(), ) } - llglobal } pub fn compile_codegen_unit( From 6848ea6c74b7ad4641b7323c7c8b7dea6ec1edbd Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Wed, 18 Nov 2020 06:41:20 +0000 Subject: [PATCH 22/22] ./x.py fmt --- compiler/rustc_interface/src/passes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 7fe0e49df3af9..f2045a6469d46 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -6,8 +6,8 @@ use rustc_ast::mut_visit::MutVisitor; use rustc_ast::{self as ast, visit}; use rustc_codegen_ssa::back::link::emit_metadata; use rustc_codegen_ssa::traits::CodegenBackend; -use rustc_data_structures::svh::Svh; use rustc_data_structures::steal::Steal; +use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal}; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_data_structures::{box_region_allow_access, declare_box_region_type, parallel};