[WIP] Content hash support. (See also cargo changes)

compiler/rustc_codegen_cranelift/src/archive.rs

@@ -5,7 +5,7 @@ use std::fs::File;
 use std::path::{Path, PathBuf};
 
 use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder};
-use rustc_codegen_ssa::METADATA_FILENAME;
+use rustc_codegen_ssa::{METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX};
 use rustc_session::Session;
 
 use object::{Object, SymbolKind};
@@ -116,7 +116,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
 
         self.add_archive(rlib.to_owned(), move |fname: &str| {
             // Ignore metadata files, no matter the name.
-            if fname == METADATA_FILENAME {
+            if fname.starts_with(METADATA_FILE_PREFIX) && fname.ends_with(METADATA_FILE_EXTENSION) {
                 return true;
             }
 

compiler/rustc_codegen_cranelift/src/metadata.rs

@@ -4,7 +4,7 @@ use std::convert::TryFrom;
 use std::fs::File;
 use std::path::Path;
 
-use rustc_codegen_ssa::METADATA_FILENAME;
+use rustc_codegen_ssa::{METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX};
 use rustc_data_structures::owning_ref::OwningRef;
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
@@ -23,7 +23,10 @@ impl MetadataLoader for CraneliftMetadataLoader {
         // Iterate over all entries in the archive:
         while let Some(entry_result) = archive.next_entry() {
             let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
+            let filename = String::from_utf8_lossy(entry.header().identifier());
+            if filename.starts_with(METADATA_FILE_PREFIX)
+                && filename.ends_with(METADATA_FILE_EXTENSION)
+            {
                 let mut buf = Vec::with_capacity(
                     usize::try_from(entry.header().size())
                         .expect("Rlib metadata file too big to load into memory."),
@@ -94,9 +97,7 @@ pub(crate) fn write_metadata<P: WriteMetadata>(
 
     assert!(kind == MetadataKind::Compressed);
     let mut compressed = tcx.metadata_encoding_version();
-    FrameEncoder::new(&mut compressed)
-        .write_all(&metadata.raw_data)
-        .unwrap();
+    FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
 
     product.add_rustc_section(
         rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx),

compiler/rustc_codegen_llvm/src/back/archive.rs

@@ -10,7 +10,9 @@ use std::str;
 use crate::llvm::archive_ro::{ArchiveRO, Child};
 use crate::llvm::{self, ArchiveKind};
 use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder};
-use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME};
+use rustc_codegen_ssa::{
+    looks_like_rust_object_file, METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX,
+};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 
@@ -130,7 +132,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
 
         self.add_archive(rlib, move |fname: &str| {
             // Ignore metadata files, no matter the name.
-            if fname == METADATA_FILENAME {
+            if fname.starts_with(METADATA_FILE_PREFIX) && fname.ends_with(METADATA_FILE_EXTENSION) {
                 return true;
             }
 

compiler/rustc_codegen_llvm/src/base.rs

@@ -28,6 +28,8 @@ use rustc_codegen_ssa::mono_item::MonoItemExt;
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind};
 use rustc_data_structures::small_c_str::SmallCStr;
+use rustc_data_structures::svh::Svh;
+use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::dep_graph;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::cstore::EncodedMetadata;
@@ -41,6 +43,7 @@ use std::ffi::CString;
 use std::time::Instant;
 
 pub fn write_compressed_metadata<'tcx>(
+    //    backend: &T,
     tcx: TyCtxt<'tcx>,
     metadata: &EncodedMetadata,
     llvm_module: &mut ModuleLlvm,
@@ -70,6 +73,9 @@ pub fn write_compressed_metadata<'tcx>(
         let directive = format!(".section {}", section_name);
         llvm::LLVMSetModuleInlineAsm2(metadata_llmod, directive.as_ptr().cast(), directive.len())
     }
+
+    let svh = tcx.crate_hash(LOCAL_CRATE);
+    let _llglobal = add_svh(tcx, &llvm_module, &svh);
 }
 
 pub struct ValueIter<'ll> {
@@ -93,6 +99,45 @@ pub fn iter_globals(llmod: &'ll llvm::Module) -> ValueIter<'ll> {
     unsafe { ValueIter { cur: llvm::LLVMGetFirstGlobal(llmod), step: llvm::LLVMGetNextGlobal } }
 }
 
+pub fn svh_symbol_name(
+    tcx: TyCtxt<'_>,
+    svh: &Svh, //, cgu_name: Symbol
+) -> String {
+    format!("rust_svh_{}_{}", tcx.original_crate_name(LOCAL_CRATE), svh)
+}
+
+fn add_svh(tcx: TyCtxt<'tcx>, llvm_module: &'ll ModuleLlvm, svh: &Svh) -> &'ll Value {
+    // Add SVH @todo insert symbol here
+    let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod());
+
+    let svh_bytes: Vec<u8> = format!("{}", svh).as_bytes().to_vec();
+    let llconst = common::bytes_in_context(metadata_llcx, &svh_bytes);
+    let name = svh_symbol_name(tcx, &svh); //cgu_name
+    let buf = CString::new(name).unwrap();
+    let llglobal =
+        unsafe { llvm::LLVMAddGlobal(metadata_llmod, common::val_ty(llconst), buf.as_ptr()) };
+    unsafe {
+        llvm::LLVMSetInitializer(llglobal, llconst);
+
+        let section_name = format!(
+            "__DATA,.rust_svh_hash", //TO DO make work for non-OSX not ,no_dead_strip
+        );
+        let name = SmallCStr::new(&section_name);
+        llvm::LLVMSetSection(llglobal, name.as_ptr());
+
+        // Also generate a .section directive to force no
+        // flags, at least for ELF outputs, so that the
+        // metadata doesn't get loaded into memory.
+        let directive = format!(".section {}", section_name);
+        llvm::LLVMRustAppendModuleInlineAsm(
+            metadata_llmod,
+            directive.as_ptr().cast(),
+            directive.len(),
+        )
+    }
+    llglobal
+}
+
 pub fn compile_codegen_unit(
     tcx: TyCtxt<'tcx>,
     cgu_name: Symbol,
@@ -116,6 +161,7 @@ pub fn compile_codegen_unit(
         let llvm_module = ModuleLlvm::new(tcx, &cgu_name.as_str());
         {
             let cx = CodegenCx::new(tcx, cgu, &llvm_module);
+
             let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
             for &(mono_item, (linkage, visibility)) in &mono_items {
                 mono_item.predefine::<Builder<'_, '_, '_>>(&cx, linkage, visibility);
@@ -129,6 +175,11 @@ pub fn compile_codegen_unit(
             // If this codegen unit contains the main function, also create the
             // wrapper here
             if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx) {
+                let svh = tcx.crate_hash(LOCAL_CRATE);
+
+                let llglobal = add_svh(tcx, &llvm_module, &svh); //&mut cx,
+                cx.add_used_global(llglobal);
+
                 attributes::sanitize(&cx, SanitizerSet::empty(), entry);
             }
 

compiler/rustc_codegen_llvm/src/metadata.rs

@@ -1,12 +1,11 @@
 use crate::llvm;
 use crate::llvm::archive_ro::ArchiveRO;
 use crate::llvm::{mk_section_iter, False, ObjectFile};
-use rustc_middle::middle::cstore::MetadataLoader;
-use rustc_target::spec::Target;
-
-use rustc_codegen_ssa::METADATA_FILENAME;
+use rustc_codegen_ssa::{METADATA_FILE_EXTENSION, METADATA_FILE_PREFIX};
 use rustc_data_structures::owning_ref::OwningRef;
 use rustc_data_structures::rustc_erase_owner;
+use rustc_middle::middle::cstore::MetadataLoader;
+use rustc_target::spec::Target;
 use tracing::debug;
 
 use rustc_fs_util::path_to_c_string;
@@ -30,10 +29,19 @@ impl MetadataLoader for LlvmMetadataLoader {
         let buf: OwningRef<_, [u8]> = archive.try_map(|ar| {
             ar.iter()
                 .filter_map(|s| s.ok())
-                .find(|sect| sect.name() == Some(METADATA_FILENAME))
+                .find(|sect| {
+                    if let Some(n) = sect.name() {
+                        n.starts_with(METADATA_FILE_PREFIX) && n.ends_with(METADATA_FILE_EXTENSION)
+                    } else {
+                        false
+                    }
+                })
                 .map(|s| s.data())
                 .ok_or_else(|| {
-                    debug!("didn't find '{}' in the archive", METADATA_FILENAME);
+                    debug!(
+                        "didn't find '{}*{}' in the archive",
+                        METADATA_FILE_PREFIX, METADATA_FILE_EXTENSION
+                    );
                     format!("failed to read rlib metadata: '{}'", filename.display())
                 })
         })?;

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -1,4 +1,5 @@
 use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::svh::Svh;
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
 use rustc_hir::def_id::CrateNum;
@@ -21,7 +22,10 @@ use super::archive::ArchiveBuilder;
 use super::command::Command;
 use super::linker::{self, Linker};
 use super::rpath::{self, RPathConfig};
-use crate::{looks_like_rust_object_file, CodegenResults, CrateInfo, METADATA_FILENAME};
+use crate::{
+    looks_like_rust_object_file, CodegenResults, CrateInfo, METADATA_FILE_EXTENSION,
+    METADATA_FILE_PREFIX,
+};
 
 use cc::windows_registry;
 use tempfile::Builder as TempFileBuilder;
@@ -270,8 +274,16 @@ pub fn each_linked_rlib(
 /// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a
 /// directory being searched for `extern crate` (observing an incomplete file).
 /// The returned path is the temporary file containing the complete metadata.
-pub fn emit_metadata(sess: &Session, metadata: &EncodedMetadata, tmpdir: &MaybeTempDir) -> PathBuf {
-    let out_filename = tmpdir.as_ref().join(METADATA_FILENAME);
+pub fn emit_metadata(
+    sess: &Session,
+    metadata: &EncodedMetadata,
+    tmpdir: &MaybeTempDir,
+    crate_name: &str,
+    svh: &Svh,
+) -> PathBuf {
+    let out_filename = tmpdir
+        .as_ref()
+        .join(format!("{}{}-{}{}", METADATA_FILE_PREFIX, crate_name, svh, METADATA_FILE_EXTENSION));
     let result = fs::write(&out_filename, &metadata.raw_data);
 
     if let Err(e) = result {
@@ -355,12 +367,19 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
     //   not be correctly inferred once 'foo.o' is removed.
     //
     // Basically, all this means is that this code should not move above the
-    // code above.
+    // code above. @audit rlib add here
+
     match flavor {
         RlibFlavor::Normal => {
             // Instead of putting the metadata in an object file section, rlibs
             // contain the metadata in a separate file.
-            ab.add_file(&emit_metadata(sess, &codegen_results.metadata, tmpdir));
+            ab.add_file(&emit_metadata(
+                sess,
+                &codegen_results.metadata,
+                tmpdir,
+                &codegen_results.crate_name.as_str(),
+                &codegen_results.crate_hash,
+            ));
 
             // After adding all files to the archive, we need to update the
             // symbol table of the archive. This currently dies on macOS (see
@@ -1916,7 +1935,7 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>(
 
             let mut any_objects = false;
             for f in archive.src_files() {
-                if f == METADATA_FILENAME {
+                if f.starts_with(METADATA_FILE_PREFIX) && f.ends_with(METADATA_FILE_EXTENSION) {
                     archive.remove_file(&f);
                     continue;
                 }

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -9,7 +9,7 @@ use rustc_hir::Node;
 use rustc_index::vec::IndexVec;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::exported_symbols::{
-    metadata_symbol_name, ExportedSymbol, SymbolExportLevel,
+    metadata_symbol_name, svh_symbol_name, ExportedSymbol, SymbolExportLevel,
 };
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
@@ -218,6 +218,11 @@ fn exported_symbols_provider_local(
         let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name));
 
         symbols.push((exported_symbol, SymbolExportLevel::Rust));
+
+        let symbol_name = svh_symbol_name(tcx);
+        let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name));
+
+        symbols.push((exported_symbol, SymbolExportLevel::C));
     }
 
     if tcx.sess.opts.share_generics() && tcx.local_crate_exports_generics() {

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -13,6 +13,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::profiling::TimingGuard;
 use rustc_data_structures::profiling::VerboseTimingGuard;
+use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::emitter::Emitter;
 use rustc_errors::{DiagnosticId, FatalError, Handler, Level};
@@ -413,6 +414,7 @@ pub fn start_async_codegen<B: ExtraBackendMethods>(
     let sess = tcx.sess;
 
     let crate_name = tcx.crate_name(LOCAL_CRATE);
+    let crate_hash = tcx.crate_hash(LOCAL_CRATE);
     let no_builtins = tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::no_builtins);
     let is_compiler_builtins =
         tcx.sess.contains_name(&tcx.hir().krate().item.attrs, sym::compiler_builtins);
@@ -465,7 +467,7 @@ pub fn start_async_codegen<B: ExtraBackendMethods>(
         windows_subsystem,
         linker_info,
         crate_info,
-
+        crate_hash,
         coordinator_send,
         codegen_worker_receive,
         shared_emitter_main,
@@ -1710,6 +1712,7 @@ pub struct OngoingCodegen<B: ExtraBackendMethods> {
     pub windows_subsystem: Option<String>,
     pub linker_info: LinkerInfo,
     pub crate_info: CrateInfo,
+    pub crate_hash: Svh,
     pub coordinator_send: Sender<Box<dyn Any + Send>>,
     pub codegen_worker_receive: Receiver<Message<B>>,
     pub shared_emitter_main: SharedEmitterMain,
@@ -1755,7 +1758,7 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
                 windows_subsystem: self.windows_subsystem,
                 linker_info: self.linker_info,
                 crate_info: self.crate_info,
-
+                crate_hash: self.crate_hash,
                 modules: compiled_modules.modules,
                 allocator_module: compiled_modules.allocator_module,
                 metadata_module: compiled_modules.metadata_module,

compiler/rustc_codegen_ssa/src/lib.rs

@@ -21,6 +21,7 @@ extern crate tracing;
 extern crate rustc_middle;
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
 use rustc_hir::def_id::CrateNum;
 use rustc_hir::LangItem;
@@ -56,8 +57,8 @@ pub struct ModuleCodegen<M> {
     pub kind: ModuleKind,
 }
 
-// FIXME(eddyb) maybe include the crate name in this?
-pub const METADATA_FILENAME: &str = "lib.rmeta";
+pub const METADATA_FILE_PREFIX: &str = "lib";
+pub const METADATA_FILE_EXTENSION: &str = ".rmeta";
 
 impl<M> ModuleCodegen<M> {
     pub fn into_compiled_module(
@@ -137,6 +138,7 @@ pub struct CodegenResults {
     pub windows_subsystem: Option<String>,
     pub linker_info: back::linker::LinkerInfo,
     pub crate_info: CrateInfo,
+    pub crate_hash: Svh,
 }
 
 pub fn provide(providers: &mut Providers) {