diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index fa49763e312f13..5f32d086904ea2 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1320,10 +1320,10 @@ static void runThinLTOBackend( Conf.CGFileType = getCodeGenFileType(Action); break; } - if (Error E = - thinBackend(Conf, -1, AddStream, *M, *CombinedIndex, ImportList, - ModuleToDefinedGVSummaries[M->getModuleIdentifier()], - /* ModuleMap */ nullptr, CGOpts.CmdArgs)) { + if (Error E = thinBackend( + Conf, -1, AddStream, *M, *CombinedIndex, ImportList, + ModuleToDefinedGVSummaries[M->getModuleIdentifier()], + /* ModuleMap */ nullptr, Conf.CodeGenOnly, CGOpts.CmdArgs)) { handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { errs() << "Error running ThinLTO backend: " << EIB.message() << '\n'; }); diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt index ecf6ce609e59f2..f028ff9e806e4a 100644 --- a/lld/MachO/CMakeLists.txt +++ b/lld/MachO/CMakeLists.txt @@ -44,6 +44,7 @@ add_lld_library(lldMachO Core DebugInfoDWARF Demangle + IPO LTO MC ObjCARCOpts diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index ab4abb1fa97efc..88117c4d41546b 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -1326,7 +1326,8 @@ static void codegenDataGenerate() { TimeTraceScope timeScope("Generating codegen data"); OutlinedHashTreeRecord globalOutlineRecord; - for (ConcatInputSection *isec : inputSections) + StableFunctionMapRecord globalMergeRecord; + for (ConcatInputSection *isec : inputSections) { if (isec->getSegName() == segment_names::data && isec->getName() == section_names::outlinedHashTree) { // Read outlined hash tree from each section. @@ -1337,11 +1338,25 @@ static void codegenDataGenerate() { // Merge it to the global hash tree. globalOutlineRecord.merge(localOutlineRecord); } + if (isec->getSegName() == segment_names::data && + isec->getName() == section_names::functionmap) { + // Read stable functions from each section. + StableFunctionMapRecord localMergeRecord; + auto *data = isec->data.data(); + localMergeRecord.deserialize(data); + + // Merge it to the global function map. + globalMergeRecord.merge(localMergeRecord); + } + } + + globalMergeRecord.finalize(); CodeGenDataWriter Writer; if (!globalOutlineRecord.empty()) Writer.addRecord(globalOutlineRecord); - + if (!globalMergeRecord.empty()) + Writer.addRecord(globalMergeRecord); std::error_code EC; auto fileName = config->codegenDataGeneratePath; assert(!fileName.empty()); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index 7ef0e31066f372..b86520d36cda5b 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -339,6 +339,7 @@ constexpr const char const_[] = "__const"; constexpr const char lazySymbolPtr[] = "__la_symbol_ptr"; constexpr const char lazyBinding[] = "__lazy_binding"; constexpr const char literals[] = "__literals"; +constexpr const char functionmap[] = "__llvm_merge"; constexpr const char moduleInitFunc[] = "__mod_init_func"; constexpr const char moduleTermFunc[] = "__mod_term_func"; constexpr const char nonLazySymbolPtr[] = "__nl_symbol_ptr"; diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp index 6527cbb68f2498..a3aa1587dee16d 100644 --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO.h" #include "llvm/Transforms/ObjCARC.h" using namespace lld; @@ -38,6 +39,8 @@ static std::string getThinLTOOutputFile(StringRef modulePath) { config->thinLTOPrefixReplaceNew); } +extern cl::opt EnableGlobalMergeFunc; + static lto::Config createConfig() { lto::Config c; c.Options = initTargetOptionsFromCodeGenFlags(); @@ -48,7 +51,10 @@ static lto::Config createConfig() { c.CPU = getCPUStr(); c.MAttrs = getMAttrs(); c.DiagHandler = diagnosticHandler; - + c.PreCodeGenPassesHook = [](legacy::PassManager &pm) { + if (EnableGlobalMergeFunc) + pm.add(createGlobalMergeFuncPass()); + }; c.AlwaysEmitRegularLTOObj = !config->ltoObjPath.empty(); c.TimeTraceEnabled = config->timeTraceEnabled; @@ -97,7 +103,6 @@ BitcodeCompiler::BitcodeCompiler() { onIndexWrite, config->thinLTOEmitIndexFiles, config->thinLTOEmitImportsFiles); } - ltoObj = std::make_unique(createConfig(), backend); } diff --git a/lld/test/MachO/cgdata-generate.s b/lld/test/MachO/cgdata-generate.s index 174df39d666c5d..f942ae07f64e0e 100644 --- a/lld/test/MachO/cgdata-generate.s +++ b/lld/test/MachO/cgdata-generate.s @@ -3,12 +3,12 @@ # RUN: rm -rf %t; split-file %s %t -# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. # RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata -# RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ][ ]*/ /g; s/^[ ]*//; s/[ ]*$//; s/[ ]/,0x/g; s/^/0x/' > %t/raw-1-bytes.txt +# RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ][ ]*/ /g; s/^[ ]*//; s/[ ]*$//; s/[ ]/,0x/g; s/^/0x/' > %t/raw-1-bytes.txt # RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-template.s > %t/merge-1.s # RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata -# RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ][ ]*/ /g; s/^[ ]*//; s/[ ]*$//; s/[ ]/,0x/g; s/^/0x/' > %t/raw-2-bytes.txt +# RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ][ ]*/ /g; s/^[ ]*//; s/[ ]*$//; s/[ ]/,0x/g; s/^/0x/' > %t/raw-2-bytes.txt # RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-template.s > %t/merge-2.s # RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/merge-1.s -o %t/merge-1.o diff --git a/llvm/include/llvm/CGData/CodeGenData.h b/llvm/include/llvm/CGData/CodeGenData.h index 84133a433170fe..fcd91ae02d6dc4 100644 --- a/llvm/include/llvm/CGData/CodeGenData.h +++ b/llvm/include/llvm/CGData/CodeGenData.h @@ -18,6 +18,7 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/CGData/OutlinedHashTree.h" #include "llvm/CGData/OutlinedHashTreeRecord.h" +#include "llvm/CGData/StableFunctionMapRecord.h" #include "llvm/IR/Module.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/ErrorHandling.h" @@ -39,7 +40,9 @@ enum class CGDataKind { Unknown = 0x0, // A function outlining info. FunctionOutlinedHashTree = 0x1, - LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FunctionOutlinedHashTree) + // A function merging info. + StableFunctionMergingMap = 0x2, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/StableFunctionMergingMap) }; const std::error_category &cgdata_category(); @@ -106,6 +109,8 @@ enum CGDataMode { class CodeGenData { /// Global outlined hash tree that has oulined hash sequences across modules. std::unique_ptr PublishedHashTree; + /// Global stable function map that has stable function info across modules. + std::unique_ptr PublishedStableFunctionMap; /// This flag is set when -fcodegen-data-generate is passed. /// Or, it can be mutated with -fcodegen-data-thinlto-two-rounds. @@ -129,12 +134,18 @@ class CodeGenData { bool hasOutlinedHashTree() { return PublishedHashTree && !PublishedHashTree->empty(); } + bool hasStableFunctionMap() { + return PublishedStableFunctionMap && !PublishedStableFunctionMap->empty(); + } /// Returns the outlined hash tree. This can be globally used in a read-only /// manner. const OutlinedHashTree *getOutlinedHashTree() { return PublishedHashTree.get(); } + const StableFunctionMap *getStableFunctionMap() { + return PublishedStableFunctionMap.get(); + } /// Returns true if we should write codegen data. bool emitCGData() { return EmitCGData; } @@ -145,6 +156,12 @@ class CodeGenData { // Ensure we disable emitCGData as we do not want to read and write both. EmitCGData = false; } + void + publishStableFunctionMap(std::unique_ptr FunctionMap) { + PublishedStableFunctionMap = std::move(FunctionMap); + // Ensure we disable emitCGData as we do not want to read and write both. + EmitCGData = false; + } }; namespace cgdata { @@ -153,10 +170,18 @@ inline bool hasOutlinedHashTree() { return CodeGenData::getInstance().hasOutlinedHashTree(); } +inline bool hasStableFunctionMap() { + return CodeGenData::getInstance().hasStableFunctionMap(); +} + inline const OutlinedHashTree *getOutlinedHashTree() { return CodeGenData::getInstance().getOutlinedHashTree(); } +inline const StableFunctionMap *getStableFunctionMap() { + return CodeGenData::getInstance().getStableFunctionMap(); +} + inline bool emitCGData() { return CodeGenData::getInstance().emitCGData(); } inline void @@ -164,6 +189,28 @@ publishOutlinedHashTree(std::unique_ptr HashTree) { CodeGenData::getInstance().publishOutlinedHashTree(std::move(HashTree)); } +inline void +publishStableFunctionMap(std::unique_ptr FunctionMap) { + CodeGenData::getInstance().publishStableFunctionMap(std::move(FunctionMap)); +} + +void initializeTwoCodegenRounds(); + +/// Save \p TheModule before the first codegen round. +/// \p Task represents the partition number in the parallel code generation +/// process. +void saveModuleForTwoRounds(const Module &TheModule, unsigned Task); + +/// Load the optimized module before the second codegen round. +std::unique_ptr loadModuleForTwoRounds(BitcodeModule &OrigModule, + unsigned Task, + LLVMContext &Context); + +/// Merge the codegen data from the input files in scratch vector in ThinLTO +/// two-codegen rounds. +Error mergeCodeGenData( + const std::unique_ptr>> InputFiles); + void warn(Error E, StringRef Whence = ""); void warn(Twine Message, std::string Whence = "", std::string Hint = ""); @@ -179,6 +226,8 @@ enum CGDataVersion { // Version 1 is the first version. This version supports the outlined // hash tree. Version1 = 1, + // Version 2 supports the stable function merging map. + Version2 = 2, CurrentVersion = CG_DATA_INDEX_VERSION }; const uint64_t Version = CGDataVersion::CurrentVersion; @@ -188,6 +237,7 @@ struct Header { uint32_t Version; uint32_t DataKind; uint64_t OutlinedHashTreeOffset; + uint64_t StableFunctionMapOffset; // New fields should only be added at the end to ensure that the size // computation is correct. The methods below need to be updated to ensure that diff --git a/llvm/include/llvm/CGData/CodeGenData.inc b/llvm/include/llvm/CGData/CodeGenData.inc index 08ec14ea051a0c..e0ae7a51024d87 100644 --- a/llvm/include/llvm/CGData/CodeGenData.inc +++ b/llvm/include/llvm/CGData/CodeGenData.inc @@ -20,6 +20,8 @@ #define CG_DATA_DEFINED CG_DATA_SECT_ENTRY(CG_outline, CG_DATA_QUOTE(CG_DATA_OUTLINE_COMMON), CG_DATA_OUTLINE_COFF, "__DATA,") +CG_DATA_SECT_ENTRY(CG_merge, CG_DATA_QUOTE(CG_DATA_MERGE_COMMON), + CG_DATA_MERGE_COFF, "__DATA,") #undef CG_DATA_SECT_ENTRY #endif @@ -27,20 +29,24 @@ CG_DATA_SECT_ENTRY(CG_outline, CG_DATA_QUOTE(CG_DATA_OUTLINE_COMMON), /* section name strings common to all targets other than WIN32 */ #define CG_DATA_OUTLINE_COMMON __llvm_outline +#define CG_DATA_MERGE_COMMON __llvm_merge /* Since cg data sections are not allocated, we don't need to * access them at runtime. */ #define CG_DATA_OUTLINE_COFF ".loutline" +#define CG_DATA_MERGE_COFF ".lmerge" #ifdef _WIN32 /* Runtime section names and name strings. */ -#define CG_DATA_SECT_NAME CG_DATA_OUTLINE_COFF +#define CG_DATA_OUTLINE_SECT_NAME CG_DATA_OUTLINE_COFF +#define CG_DATA_MERGE_SECT_NAME CG_DATA_MERGE_COFF #else /* Runtime section names and name strings. */ -#define CG_DATA_SECT_NAME CG_DATA_QUOTE(CG_DATA_OUTLINE_COMMON) +#define CG_DATA_OUTLINE_SECT_NAME CG_DATA_QUOTE(CG_DATA_OUTLINE_COMMON) +#define CG_DATA_MERGE_SECT_NAME CG_DATA_QUOTE(CG_DATA_MERGE_COMMON) #endif /* Indexed codegen data format version (start from 1). */ -#define CG_DATA_INDEX_VERSION 1 +#define CG_DATA_INDEX_VERSION 2 diff --git a/llvm/include/llvm/CGData/CodeGenDataReader.h b/llvm/include/llvm/CGData/CodeGenDataReader.h index 1ee4bfbe480233..2891e611430b02 100644 --- a/llvm/include/llvm/CGData/CodeGenDataReader.h +++ b/llvm/include/llvm/CGData/CodeGenDataReader.h @@ -15,6 +15,7 @@ #include "llvm/CGData/CodeGenData.h" #include "llvm/CGData/OutlinedHashTreeRecord.h" +#include "llvm/CGData/StableFunctionMapRecord.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/VirtualFileSystem.h" @@ -36,10 +37,15 @@ class CodeGenDataReader { virtual CGDataKind getDataKind() const = 0; /// Return true if the data has an outlined hash tree. virtual bool hasOutlinedHashTree() const = 0; + /// Return true if the data has a stable function map. + virtual bool hasStableFunctionMap() const = 0; /// Return the outlined hash tree that is released from the reader. std::unique_ptr releaseOutlinedHashTree() { return std::move(HashTreeRecord.HashTree); } + std::unique_ptr releaseStableFunctionMap() { + return std::move(FunctionMapRecord.FunctionMap); + } /// Factory method to create an appropriately typed reader for the given /// codegen data file path and file system. @@ -54,14 +60,20 @@ class CodeGenDataReader { /// Extract the cgdata embedded in sections from the given object file and /// merge them into the GlobalOutlineRecord. This is a static helper that /// is used by `llvm-cgdata --merge` or ThinLTO's two-codegen rounds. - static Error mergeFromObjectFile(const object::ObjectFile *Obj, - OutlinedHashTreeRecord &GlobalOutlineRecord); + static Error + mergeFromObjectFile(const object::ObjectFile *Obj, + OutlinedHashTreeRecord &GlobalOutlineRecord, + StableFunctionMapRecord &GlobalFunctionMapRecord); protected: /// The outlined hash tree that has been read. When it's released by /// releaseOutlinedHashTree(), it's no longer valid. OutlinedHashTreeRecord HashTreeRecord; + /// The stable function map that has been read. When it's released by + // releaseStableFunctionMap(), it's no longer valid. + StableFunctionMapRecord FunctionMapRecord; + /// Set the current error and return same. Error error(cgdata_error Err, const std::string &ErrMsg = "") { LastError = Err; @@ -112,6 +124,11 @@ class IndexedCodeGenDataReader : public CodeGenDataReader { return Header.DataKind & static_cast(CGDataKind::FunctionOutlinedHashTree); } + /// Return true if the header indicates the data has a stable function map. + bool hasStableFunctionMap() const override { + return Header.DataKind & + static_cast(CGDataKind::StableFunctionMergingMap); + } }; /// This format is a simple text format that's suitable for test data. @@ -147,6 +164,12 @@ class TextCodeGenDataReader : public CodeGenDataReader { return static_cast(DataKind) & static_cast(CGDataKind::FunctionOutlinedHashTree); } + /// Return true if the header indicates the data has a stable function map. + /// This does not mean that the data is still available. + bool hasStableFunctionMap() const override { + return static_cast(DataKind) & + static_cast(CGDataKind::StableFunctionMergingMap); + } }; } // end namespace llvm diff --git a/llvm/include/llvm/CGData/CodeGenDataWriter.h b/llvm/include/llvm/CGData/CodeGenDataWriter.h index 5cb8377b1d07e5..1c4247608999a7 100644 --- a/llvm/include/llvm/CGData/CodeGenDataWriter.h +++ b/llvm/include/llvm/CGData/CodeGenDataWriter.h @@ -15,6 +15,7 @@ #include "llvm/CGData/CodeGenData.h" #include "llvm/CGData/OutlinedHashTreeRecord.h" +#include "llvm/CGData/StableFunctionMapRecord.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" @@ -57,6 +58,9 @@ class CodeGenDataWriter { /// The outlined hash tree to be written. OutlinedHashTreeRecord HashTreeRecord; + /// The stable function map to be written. + StableFunctionMapRecord FunctionMapRecord; + /// A bit mask describing the kind of the codegen data. CGDataKind DataKind = CGDataKind::Unknown; @@ -64,9 +68,12 @@ class CodeGenDataWriter { CodeGenDataWriter() = default; ~CodeGenDataWriter() = default; - /// Add the outlined hash tree record. The input Record is released. + /// Add the outlined hash tree record. The input hash tree is released. void addRecord(OutlinedHashTreeRecord &Record); + /// Add the stable function map record. The input function map is released. + void addRecord(StableFunctionMapRecord &Record); + /// Write the codegen data to \c OS Error write(raw_fd_ostream &OS); @@ -81,11 +88,19 @@ class CodeGenDataWriter { return static_cast(DataKind) & static_cast(CGDataKind::FunctionOutlinedHashTree); } + /// Return true if the header indicates the data has a stable function map. + bool hasStableFunctionMap() const { + return static_cast(DataKind) & + static_cast(CGDataKind::StableFunctionMergingMap); + } private: /// The offset of the outlined hash tree in the file. uint64_t OutlinedHashTreeOffset; + /// The offset of the stable function map in the file. + uint64_t StableFunctionMapOffset; + /// Write the codegen data header to \c COS Error writeHeader(CGDataOStream &COS); diff --git a/llvm/include/llvm/CGData/StableFunctionMap.h b/llvm/include/llvm/CGData/StableFunctionMap.h new file mode 100644 index 00000000000000..5ff834205e6bc3 --- /dev/null +++ b/llvm/include/llvm/CGData/StableFunctionMap.h @@ -0,0 +1,140 @@ +//===- StableFunctionMap.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// TODO +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_CGDATA_STABLEFUNCTIONMAP_H +#define LLVM_CGDATA_STABLEFUNCTIONMAP_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StableHashing.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/IR/StructuralHash.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +namespace llvm { + +using IndexPairHash = std::pair; +using IndexOperandHashVecType = SmallVector; + +/// A stable function is a function with a stable hash while tracking the +/// locations of ignored operands and their hashes. +struct StableFunction { + /// The combined stable hash of the function. + stable_hash Hash; + /// The name of the function. + std::string FunctionName; + /// The name of the module the function is in. + std::string ModuleName; + /// The number of instructions. + unsigned InstCount; + /// A vector of pairs of IndexPair and operand hash which was skipped. + IndexOperandHashVecType IndexOperandHashes; + + StableFunction(stable_hash Hash, const std::string FunctionName, + const std::string ModuleName, unsigned InstCount, + IndexOperandHashVecType &&IndexOperandHashes) + : Hash(Hash), FunctionName(FunctionName), ModuleName(ModuleName), + InstCount(InstCount), + IndexOperandHashes(std::move(IndexOperandHashes)) {} + StableFunction() = default; +}; + +/// An efficient form of StableFunction for fast look-up +struct StableFunctionEntry { + /// The combined stable hash of the function. + stable_hash Hash; + /// Id of the function name. + unsigned FunctionNameId; + /// Id of the module name. + unsigned ModuleNameId; + /// The number of instructions. + unsigned InstCount; + /// A map from an IndexPair to a stable_hash which was skipped. + std::unique_ptr IndexOperandHashMap; + + StableFunctionEntry( + stable_hash Hash, unsigned FunctionNameId, unsigned ModuleNameId, + unsigned InstCount, + std::unique_ptr IndexOperandHashMap) + : Hash(Hash), FunctionNameId(FunctionNameId), ModuleNameId(ModuleNameId), + InstCount(InstCount), + IndexOperandHashMap(std::move(IndexOperandHashMap)) {} +}; + +using HashFuncsMapType = + DenseMap>>; + +class StableFunctionMap { + /// A map from a stable_hash to a vector of functions with that hash. + HashFuncsMapType HashToFuncs; + /// A vector of strings to hold names. + SmallVector IdToName; + /// A map from StringRef (name) to an ID. + StringMap NameToId; + /// True if the function map is finalized with minimal content. + bool Finalized = false; + +public: + /// Get the HashToFuncs map for serialization. + const HashFuncsMapType &getFunctionMap() const { return HashToFuncs; } + + /// Get the NameToId vector for serialization. + const SmallVector getNames() { return IdToName; } + + /// Get an existing ID associated with the given name or create a new ID if it + /// doesn't exist. + unsigned getIdOrCreateForName(StringRef Name); + + /// Get the name associated with a given ID + std::optional getNameForId(unsigned Id) const; + + /// Insert a `StableFunction` object into the function map. This method + /// handles the uniquing of string names and create a `StableFunctionEntry` + /// for insertion. + void insert(const StableFunction &Func); + + /// Insert a `StableFunctionEntry` into the function map directly. This + /// method assumes that string names have already been uniqued and the + /// `StableFunctionEntry` is ready for insertion. + void insert(std::unique_ptr FuncEntry) { + assert(!Finalized && "Cannot insert after finalization"); + HashToFuncs[FuncEntry->Hash].emplace_back(std::move(FuncEntry)); + } + + /// Merge a \p OtherMap into this function map. + void merge(const StableFunctionMap &OtherMap); + + /// \returns true if there is no stable function entry. + bool empty() { return size() == 0; } + + enum SizeType { + UniqueHashCount, // The number of unique hashes in HashToFuncs. + TotalFunctionCount, // The number of total functions in HashToFuncs. + MergeableFunctionCount, // The number of functions that can be merged based + // on their hash. + }; + + /// \returns the size of StableFunctionMap. + /// \p Type is the type of size to return. + size_t size(SizeType Type = UniqueHashCount) const; + + /// Finalize the stable function mape by trimming content. + void finalize(); +}; + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/CGData/StableFunctionMapRecord.h b/llvm/include/llvm/CGData/StableFunctionMapRecord.h new file mode 100644 index 00000000000000..94a6154cad336c --- /dev/null +++ b/llvm/include/llvm/CGData/StableFunctionMapRecord.h @@ -0,0 +1,64 @@ +//===- StableFunctionMapRecord.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// TODO +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_CGDATA_STABLEFUNCTIONMAPRECORD_H +#define LLVM_CGDATA_STABLEFUNCTIONMAPRECORD_H + +#include "llvm/CGData/StableFunctionMap.h" + +#include +#include + +namespace llvm { + +struct StableFunctionMapRecord { + std::unique_ptr FunctionMap; + + StableFunctionMapRecord() { + FunctionMap = std::make_unique(); + } + StableFunctionMapRecord(std::unique_ptr FunctionMap) + : FunctionMap(std::move(FunctionMap)) {} + + /// Serialize the stable function map to a raw_ostream. + void serialize(raw_ostream &OS) const; + + /// Deserialize the stable function map from a raw_ostream. + void deserialize(const unsigned char *&Ptr); + + /// Serialize the stable function map to a YAML stream. + void serializeYAML(yaml::Output &YOS) const; + + /// Deserialize the stable function map from a YAML stream. + void deserializeYAML(yaml::Input &YIS); + + /// Finalize the stable function map by trimming content. + void finalize() { FunctionMap->finalize(); } + + /// Merge the stable function map into this one. + void merge(const StableFunctionMapRecord &Other) { + FunctionMap->merge(*Other.FunctionMap); + } + + /// \returns true if the stable function map is empty. + bool empty() const { return FunctionMap->empty(); } + + /// Print the stable function map in a YAML format. + void print(raw_ostream &OS = llvm::errs()) const { + yaml::Output YOS(OS); + serializeYAML(YOS); + } +}; + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/IR/StructuralHash.h b/llvm/include/llvm/IR/StructuralHash.h index 57fb45db849110..95baabebb97ac2 100644 --- a/llvm/include/llvm/IR/StructuralHash.h +++ b/llvm/include/llvm/IR/StructuralHash.h @@ -14,6 +14,10 @@ #ifndef LLVM_IR_STRUCTURALHASH_H #define LLVM_IR_STRUCTURALHASH_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/StableHashing.h" +#include "llvm/IR/Instruction.h" #include namespace llvm { @@ -21,7 +25,8 @@ namespace llvm { class Function; class Module; -using IRHash = uint64_t; +using IRHash = stable_hash; +using OpndHash = stable_hash; /// Returns a hash of the function \p F. /// \param F The function to hash. @@ -36,6 +41,59 @@ IRHash StructuralHash(const Function &F, bool DetailedHash = false); /// composed the module hash. IRHash StructuralHash(const Module &M, bool DetailedHash = false); +/// The pair of an instruction index and a operand index. +using IndexPair = std::pair; + +/// A map from an instruction index to an instruction pointer. +using IndexInstrMap = MapVector; + +/// A map from an IndexPair to an OpndHash. +using IndexOperandHashMapType = DenseMap; + +/// A function that takes an instruction and an operand index and returns true +/// if the operand should be ignored in the function hash computation. +using IgnoreOperandFunc = std::function; + +struct FunctionHashInfo { + /// A hash value representing the structural content of the function + IRHash FunctionHash; + /// A mapping from instruction indices to instruction pointers + std::unique_ptr IndexInstruction; + /// A mapping from pairs of instruction indices and operand indices + /// to the hashes of the operands. This can be used to analyze or + /// reconstruct the differences in ignored operands + std::unique_ptr IndexOperandHashMap; + + FunctionHashInfo(IRHash FuntionHash, + std::unique_ptr IndexInstruction, + std::unique_ptr IndexOperandHashMap) + : FunctionHash(FuntionHash), + IndexInstruction(std::move(IndexInstruction)), + IndexOperandHashMap(std::move(IndexOperandHashMap)) {} +#if 0 // TODO: delete + // Disable copy operations + FunctionHashInfo(const FunctionHashInfo &) = delete; + FunctionHashInfo &operator=(const FunctionHashInfo &) = delete; + // Enable move operations + FunctionHashInfo(FunctionHashInfo &&) noexcept = default; + FunctionHashInfo &operator=(FunctionHashInfo &&) noexcept = default; +#endif +}; + +/// Computes a structural hash of a given function, considering the structure +/// and content of the function's instructions while allowing for selective +/// ignoring of certain operands based on custom criteria. This hash can be used +/// to identify functions that are structurally similar or identical, which is +/// useful in optimizations, deduplication, or analysis tasks. +/// \param F The function to hash. +/// \param IgnoreOp A callable that takes an instruction and one of its operands +/// and returns true if an operand should be ignored in the hash computation. +/// for the hash computation. +/// \return A FunctionHashInfo structure + +FunctionHashInfo StructuralHashWithDifferences(const Function &F, + IgnoreOperandFunc IgnoreOp); + } // end namespace llvm #endif diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 4352099d6dbb99..9aa36d5bb7f801 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -123,6 +123,7 @@ void initializeGCEmptyBasicBlocksPass(PassRegistry &); void initializeGCMachineCodeAnalysisPass(PassRegistry &); void initializeGCModuleInfoPass(PassRegistry &); void initializeGVNLegacyPassPass(PassRegistry &); +void initializeGlobalMergeFuncPass(PassRegistry &); void initializeGlobalMergePass(PassRegistry &); void initializeGlobalsAAWrapperPassPass(PassRegistry &); void initializeHardwareLoopsLegacyPass(PassRegistry &); diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h index de89f4bb10dff2..098c0491dfe70a 100644 --- a/llvm/include/llvm/LTO/LTOBackend.h +++ b/llvm/include/llvm/LTO/LTOBackend.h @@ -50,12 +50,14 @@ Error backend(const Config &C, AddStreamFn AddStream, /// already been mapped to memory and the corresponding BitcodeModule objects /// are saved in the ModuleMap. If \p ModuleMap is nullptr, module files will /// be mapped to memory on demand and at any given time during importing, only -/// one source module will be kept open at the most. +/// one source module will be kept open at the most. If \p CodeGenOnly is true, +/// the backend will skip optimization and only perform code generation. Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector *ModuleMap, + bool CodeGenOnly, const std::vector &CmdArgs = std::vector()); Error finalizeOptimizationRemarks( diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index 92b59a66567c95..ea3609a2b4bc71 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -79,6 +79,7 @@ struct ForcePassLinking { (void)llvm::createDomOnlyViewerWrapperPassPass(); (void)llvm::createDomViewerWrapperPassPass(); (void)llvm::createAlwaysInlinerLegacyPass(); + (void)llvm::createGlobalMergeFuncPass(); (void)llvm::createGlobalsAAWrapperPass(); (void)llvm::createInstSimplifyLegacyPass(); (void)llvm::createInstructionCombiningPass(); diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 13bc4700d87029..96b5b815132bc0 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -74,6 +74,7 @@ #include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/CFGuard.h" +#include "llvm/Transforms/IPO/GlobalMergeFunctions.h" #include "llvm/Transforms/Scalar/ConstantHoisting.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h index ee0e35aa618325..86a8654f56997c 100644 --- a/llvm/include/llvm/Transforms/IPO.h +++ b/llvm/include/llvm/Transforms/IPO.h @@ -55,6 +55,8 @@ enum class PassSummaryAction { Export, ///< Export information to summary. }; +Pass *createGlobalMergeFuncPass(); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/Transforms/IPO/GlobalMergeFunctions.h b/llvm/include/llvm/Transforms/IPO/GlobalMergeFunctions.h new file mode 100644 index 00000000000000..0de2d91a647579 --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/GlobalMergeFunctions.h @@ -0,0 +1,80 @@ +//===------ GlobalMergeFunctions.h - Global merge functions -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This file defines global merge functions pass and related data structure. +/// +//===----------------------------------------------------------------------===// + +#ifndef PIKA_TRANSFORMS_UTILS_GLOBALMERGEFUNCTIONS_H +#define PIKA_TRANSFORMS_UTILS_GLOBALMERGEFUNCTIONS_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StableHashing.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CGData/StableFunctionMap.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include +#include + +enum class HashFunctionMode { + None, + BuildingHashFuncion, + UsingHashFunction, +}; + +namespace llvm { + +// (inst, opnd) indices +using LocPair = std::pair; +// 64 bit constant hash +using ConstHash = uint64_t; +// A map of location pair to constant hash +using InstOpndIdConstHashMapTy = DenseMap; + +// A vector of locations (the pair of (instruction, operand) indices) reachable +// from a parameter. +using ParamLocs = SmallVector; +// A vector of parameters +using ParamLocsVecTy = SmallVector; +// A map of stable hash to a vector of stable functions + +/// GlobalMergeFunc finds functions which only differ by constants in +/// certain instructions, e.g. resulting from specialized functions of layout +/// compatible types. +/// Unlike PikaMergeFunc that directly compares IRs, this uses stable function +/// hash to find the merge candidate. Similar to the global outliner, we can run +/// codegen twice to collect function merge candidate in the first round, and +/// merge functions globally in the second round. +class GlobalMergeFunc : public ModulePass { + HashFunctionMode MergerMode = HashFunctionMode::None; + + std::unique_ptr LocalFunctionMap; + // StableFunctionMap LocalFunctionMap; +public: + static char ID; + + GlobalMergeFunc(); + + void initializeMergerMode(const Module &M); + + bool runOnModule(Module &M) override; + + /// Analyze module to create stable function into LocalFunctionMap. + void analyze(Module &M); + + /// Emit LocalFunctionMap into __llvm_merge section. + void emitFunctionMap(Module &M); + + /// Merge functions in the module using MFI. + bool merge(Module &M, const StableFunctionMap *FunctionMap); +}; + +} // end namespace llvm +#endif // PIKA_TRANSFORMS_UTILS_GLOBALMERGEFUNCTIONS_H diff --git a/llvm/lib/CGData/CMakeLists.txt b/llvm/lib/CGData/CMakeLists.txt index ff1aab920e7a8c..003173139f36c5 100644 --- a/llvm/lib/CGData/CMakeLists.txt +++ b/llvm/lib/CGData/CMakeLists.txt @@ -4,6 +4,8 @@ add_llvm_component_library(LLVMCGData CodeGenDataWriter.cpp OutlinedHashTree.cpp OutlinedHashTreeRecord.cpp + StableFunctionMap.cpp + StableFunctionMapRecord.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/CGData @@ -12,6 +14,8 @@ add_llvm_component_library(LLVMCGData intrinsics_gen LINK_COMPONENTS + BitReader + BitWriter Core Support Object diff --git a/llvm/lib/CGData/CodeGenData.cpp b/llvm/lib/CGData/CodeGenData.cpp index 55d2504231c744..60d7ae28f12f87 100644 --- a/llvm/lib/CGData/CodeGenData.cpp +++ b/llvm/lib/CGData/CodeGenData.cpp @@ -14,9 +14,11 @@ #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/CGData/CodeGenDataReader.h" #include "llvm/CGData/OutlinedHashTreeRecord.h" +#include "llvm/CGData/StableFunctionMapRecord.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" #include "llvm/Support/WithColor.h" #define DEBUG_TYPE "cg-data" @@ -30,6 +32,14 @@ cl::opt cl::opt CodeGenDataUsePath("codegen-data-use-path", cl::init(""), cl::Hidden, cl::desc("File path to where .cgdata file is read")); +cl::opt CodeGenDataThinLTOTwoRounds( + "codegen-data-thinlto-two-rounds", cl::init(false), cl::Hidden, + cl::desc("Enable two-round ThinLTO code generation. The first round " + "emits codegen data, while the second round uses the emitted " + "codegen data for further optimizations.")); + +// Path to where the optimized bitcodes are saved and restored for ThinLTO. +static SmallString<128> CodeGenDataThinLTOTwoRoundsPath; static std::string getCGDataErrString(cgdata_error Err, const std::string &ErrMsg = "") { @@ -139,7 +149,7 @@ CodeGenData &CodeGenData::getInstance() { std::call_once(CodeGenData::OnceFlag, []() { Instance = std::unique_ptr(new CodeGenData()); - if (CodeGenDataGenerate) + if (CodeGenDataGenerate || CodeGenDataThinLTOTwoRounds) Instance->EmitCGData = true; else if (!CodeGenDataUsePath.empty()) { // Initialize the global CGData if the input file name is given. @@ -156,6 +166,8 @@ CodeGenData &CodeGenData::getInstance() { auto Reader = ReaderOrErr->get(); if (Reader->hasOutlinedHashTree()) Instance->publishOutlinedHashTree(Reader->releaseOutlinedHashTree()); + if (Reader->hasStableFunctionMap()) + Instance->publishStableFunctionMap(Reader->releaseStableFunctionMap()); } }); return *(Instance.get()); @@ -178,18 +190,14 @@ Expected
Header::readFromBuffer(const unsigned char *Curr) { return make_error(cgdata_error::unsupported_version); H.DataKind = endian::readNext(Curr); - switch (H.Version) { - // When a new field is added to the header add a case statement here to - // compute the size as offset of the new field + size of the new field. This - // relies on the field being added to the end of the list. - static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version1, - "Please update the size computation below if a new field has " - "been added to the header, if not add a case statement to " - "fall through to the latest version."); - case 1ull: - H.OutlinedHashTreeOffset = + static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version2, + "Please update the offset computation below if a new field has " + "been added to the header."); + H.OutlinedHashTreeOffset = + endian::readNext(Curr); + if (H.Version >= 2) + H.StableFunctionMapOffset = endian::readNext(Curr); - } return H; } @@ -215,6 +223,84 @@ void warn(Error E, StringRef Whence) { } } +static std::string getPath(StringRef Dir, unsigned Task) { + llvm::SmallString<128> Path(Dir); + llvm::sys::path::append(Path, llvm::Twine(Task) + ".saved_copy.bc"); + return std::string(Path); +} + +void initializeTwoCodegenRounds() { + assert(CodeGenDataThinLTOTwoRounds); + if (auto EC = llvm::sys::fs::createUniqueDirectory( + "cgdata", CodeGenDataThinLTOTwoRoundsPath)) + report_fatal_error(Twine("Failed to create directory: ") + EC.message()); +} + +void saveModuleForTwoRounds(const Module &TheModule, unsigned Task) { + assert(sys::fs::is_directory(CodeGenDataThinLTOTwoRoundsPath)); + std::string Path = getPath(CodeGenDataThinLTOTwoRoundsPath, Task); + std::error_code EC; + raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::OF_None); + if (EC) + report_fatal_error(Twine("Failed to open ") + Path + + " to save optimized bitcode: " + EC.message()); + WriteBitcodeToFile(TheModule, OS, /*ShouldPreserveUseListOrder=*/true); +} + +std::unique_ptr loadModuleForTwoRounds(BitcodeModule &OrigModule, + unsigned Task, + LLVMContext &Context) { + assert(sys::fs::is_directory(CodeGenDataThinLTOTwoRoundsPath)); + std::string Path = getPath(CodeGenDataThinLTOTwoRoundsPath, Task); + auto FileOrError = MemoryBuffer::getFile(Path); + if (auto EC = FileOrError.getError()) + report_fatal_error(Twine("Failed to open ") + Path + + " to load optimized bitcode: " + EC.message()); + + std::unique_ptr FileBuffer = std::move(*FileOrError); + auto RestoredModule = parseBitcodeFile(*FileBuffer, Context); + if (!RestoredModule) + report_fatal_error(Twine("Failed to parse optimized bitcode loaded from ") + + Path + "\n"); + + // Restore the original module identifier. + (*RestoredModule)->setModuleIdentifier(OrigModule.getModuleIdentifier()); + return std::move(*RestoredModule); +} + +Error mergeCodeGenData( + const std::unique_ptr>> InputFiles) { + + OutlinedHashTreeRecord GlobalOutlineRecord; + StableFunctionMapRecord GlobalStableFunctionMapRecord; + for (auto &InputFile : *(InputFiles)) { + if (InputFile.empty()) + continue; + StringRef File = StringRef(InputFile.data(), InputFile.size()); + std::unique_ptr Buffer = MemoryBuffer::getMemBuffer( + File, "in-memory object file", /*RequiresNullTerminator=*/false); + Expected> BinOrErr = + object::ObjectFile::createObjectFile(Buffer->getMemBufferRef()); + if (!BinOrErr) + return BinOrErr.takeError(); + + std::unique_ptr &Obj = BinOrErr.get(); + if (auto E = CodeGenDataReader::mergeFromObjectFile( + Obj.get(), GlobalOutlineRecord, GlobalStableFunctionMapRecord)) + return E; + } + + GlobalStableFunctionMapRecord.finalize(); + + if (!GlobalOutlineRecord.empty()) + cgdata::publishOutlinedHashTree(std::move(GlobalOutlineRecord.HashTree)); + if (!GlobalStableFunctionMapRecord.empty()) + cgdata::publishStableFunctionMap( + std::move(GlobalStableFunctionMapRecord.FunctionMap)); + + return Error::success(); +} + } // end namespace cgdata } // end namespace llvm diff --git a/llvm/lib/CGData/CodeGenDataReader.cpp b/llvm/lib/CGData/CodeGenDataReader.cpp index f7f3a8f42af7e1..0a893f9a862077 100644 --- a/llvm/lib/CGData/CodeGenDataReader.cpp +++ b/llvm/lib/CGData/CodeGenDataReader.cpp @@ -31,12 +31,13 @@ setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) { } Error CodeGenDataReader::mergeFromObjectFile( - const object::ObjectFile *Obj, - OutlinedHashTreeRecord &GlobalOutlineRecord) { + const object::ObjectFile *Obj, OutlinedHashTreeRecord &GlobalOutlineRecord, + StableFunctionMapRecord &GlobalFunctionMapRecord) { Triple TT = Obj->makeTriple(); auto CGOutLineName = getCodeGenDataSectionName(CG_outline, TT.getObjectFormat(), false); - + auto CGMergeName = + getCodeGenDataSectionName(CG_merge, TT.getObjectFormat(), false); for (auto &Section : Obj->sections()) { Expected NameOrErr = Section.getName(); if (!NameOrErr) @@ -47,17 +48,23 @@ Error CodeGenDataReader::mergeFromObjectFile( auto *Data = reinterpret_cast(ContentsOrErr->data()); auto *EndData = Data + ContentsOrErr->size(); + // In case dealing with an executable that has concatenated cgdata, + // we want to merge them into a single cgdata. + // Although it's not a typical workflow, we support this scenario + // by looping over all data in the sections. if (*NameOrErr == CGOutLineName) { - // In case dealing with an executable that has concatenated cgdata, - // we want to merge them into a single cgdata. - // Although it's not a typical workflow, we support this scenario. while (Data != EndData) { OutlinedHashTreeRecord LocalOutlineRecord; LocalOutlineRecord.deserialize(Data); GlobalOutlineRecord.merge(LocalOutlineRecord); } + } else if (*NameOrErr == CGMergeName) { + while (Data != EndData) { + StableFunctionMapRecord LocalFunctionMapRecord; + LocalFunctionMapRecord.deserialize(Data); + GlobalFunctionMapRecord.merge(LocalFunctionMapRecord); + } } - // TODO: Add support for other cgdata sections. } return Error::success(); @@ -66,7 +73,8 @@ Error CodeGenDataReader::mergeFromObjectFile( Error IndexedCodeGenDataReader::read() { using namespace support; - // The smallest header with the version 1 is 24 bytes + // The smallest header with the version 1 is 24 bytes. + // Do not update this value even with the new version of the header. const unsigned MinHeaderSize = 24; if (DataBuffer->getBufferSize() < MinHeaderSize) return error(cgdata_error::bad_header); @@ -84,6 +92,12 @@ Error IndexedCodeGenDataReader::read() { return error(cgdata_error::eof); HashTreeRecord.deserialize(Ptr); } + if (hasStableFunctionMap()) { + const unsigned char *Ptr = Start + Header.StableFunctionMapOffset; + if (Ptr >= End) + return error(cgdata_error::eof); + FunctionMapRecord.deserialize(Ptr); + } return success(); } @@ -149,6 +163,8 @@ Error TextCodeGenDataReader::read() { StringRef Str = Line->drop_front().rtrim(); if (Str.equals_insensitive("outlined_hash_tree")) DataKind |= CGDataKind::FunctionOutlinedHashTree; + else if (Str.equals_insensitive("stable_function_map")) + DataKind |= CGDataKind::StableFunctionMergingMap; else return error(cgdata_error::bad_header); } @@ -167,8 +183,8 @@ Error TextCodeGenDataReader::read() { yaml::Input YOS(StringRef(Pos, Size)); if (hasOutlinedHashTree()) HashTreeRecord.deserializeYAML(YOS); - - // TODO: Add more yaml cgdata in order + if (hasStableFunctionMap()) + FunctionMapRecord.deserializeYAML(YOS); return Error::success(); } diff --git a/llvm/lib/CGData/CodeGenDataWriter.cpp b/llvm/lib/CGData/CodeGenDataWriter.cpp index 5f638be0fefe74..b97c9138593f7b 100644 --- a/llvm/lib/CGData/CodeGenDataWriter.cpp +++ b/llvm/lib/CGData/CodeGenDataWriter.cpp @@ -52,6 +52,13 @@ void CodeGenDataWriter::addRecord(OutlinedHashTreeRecord &Record) { DataKind |= CGDataKind::FunctionOutlinedHashTree; } +void CodeGenDataWriter::addRecord(StableFunctionMapRecord &Record) { + assert(Record.FunctionMap && "empty function map in the record"); + FunctionMapRecord.FunctionMap = std::move(Record.FunctionMap); + + DataKind |= CGDataKind::StableFunctionMergingMap; +} + Error CodeGenDataWriter::write(raw_fd_ostream &OS) { CGDataOStream COS(OS); return writeImpl(COS); @@ -68,8 +75,11 @@ Error CodeGenDataWriter::writeHeader(CGDataOStream &COS) { if (static_cast(DataKind & CGDataKind::FunctionOutlinedHashTree)) Header.DataKind |= static_cast(CGDataKind::FunctionOutlinedHashTree); - + if (static_cast(DataKind & CGDataKind::StableFunctionMergingMap)) + Header.DataKind |= + static_cast(CGDataKind::StableFunctionMergingMap); Header.OutlinedHashTreeOffset = 0; + Header.StableFunctionMapOffset = 0; // Only write up to the CGDataKind. We need to remember the offset of the // remaining fields to allow back-patching later. @@ -79,10 +89,15 @@ Error CodeGenDataWriter::writeHeader(CGDataOStream &COS) { // Save the location of Header.OutlinedHashTreeOffset field in \c COS. OutlinedHashTreeOffset = COS.tell(); - // Reserve the space for OutlinedHashTreeOffset field. COS.write(0); + // Save the location of Header.StableFunctionMapOffset field in \c COS. + StableFunctionMapOffset = COS.tell(); + + // Reserve the space for StableFunctionMapOffset field. + COS.write(0); + return Error::success(); } @@ -93,10 +108,14 @@ Error CodeGenDataWriter::writeImpl(CGDataOStream &COS) { uint64_t OutlinedHashTreeFieldStart = COS.tell(); if (hasOutlinedHashTree()) HashTreeRecord.serialize(COS.OS); + uint64_t StableFunctionMapFieldStart = COS.tell(); + if (hasStableFunctionMap()) + FunctionMapRecord.serialize(COS.OS); // Back patch the offsets. CGDataPatchItem PatchItems[] = { - {OutlinedHashTreeOffset, &OutlinedHashTreeFieldStart, 1}}; + {OutlinedHashTreeOffset, &OutlinedHashTreeFieldStart, 1}, + {StableFunctionMapOffset, &StableFunctionMapFieldStart, 1}}; COS.patch(PatchItems); return Error::success(); @@ -106,7 +125,8 @@ Error CodeGenDataWriter::writeHeaderText(raw_fd_ostream &OS) { if (hasOutlinedHashTree()) OS << "# Outlined stable hash tree\n:outlined_hash_tree\n"; - // TODO: Add more data types in this header + if (hasStableFunctionMap()) + OS << "# Stable function map\n:stable_function_map\n"; return Error::success(); } @@ -119,7 +139,8 @@ Error CodeGenDataWriter::writeText(raw_fd_ostream &OS) { if (hasOutlinedHashTree()) HashTreeRecord.serializeYAML(YOS); - // TODO: Write more yaml cgdata in order + if (hasStableFunctionMap()) + FunctionMapRecord.serializeYAML(YOS); return Error::success(); } diff --git a/llvm/lib/CGData/StableFunctionMap.cpp b/llvm/lib/CGData/StableFunctionMap.cpp new file mode 100644 index 00000000000000..8d9a6921594ee8 --- /dev/null +++ b/llvm/lib/CGData/StableFunctionMap.cpp @@ -0,0 +1,167 @@ +//===-- StableFunctionMap.cpp ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// + +#include "llvm/CGData/StableFunctionMap.h" + +#define DEBUG_TYPE "stable-function-map" + +using namespace llvm; + +unsigned StableFunctionMap::getIdOrCreateForName(StringRef Name) { + auto It = NameToId.find(Name); + if (It != NameToId.end()) { + return It->second; + } else { + unsigned Id = IdToName.size(); + assert(Id == NameToId.size() && "ID collision"); + IdToName.emplace_back(Name.str()); + NameToId[IdToName.back()] = Id; + return Id; + } +} + +std::optional StableFunctionMap::getNameForId(unsigned Id) const { + if (Id >= IdToName.size()) + return std::nullopt; + return IdToName[Id]; +} + +void StableFunctionMap::insert(const StableFunction &Func) { + assert(!Finalized && "Cannot insert after finalization"); + auto FuncNameId = getIdOrCreateForName(Func.FunctionName); + auto ModuleNameId = getIdOrCreateForName(Func.ModuleName); + auto IndexOperandHashMap = std::make_unique(); + for (auto &[Index, Hash] : Func.IndexOperandHashes) + (*IndexOperandHashMap)[Index] = Hash; + auto FuncEntry = std::make_unique( + Func.Hash, FuncNameId, ModuleNameId, Func.InstCount, + std::move(IndexOperandHashMap)); + insert(std::move(FuncEntry)); +} + +void StableFunctionMap::merge(const StableFunctionMap &OtherMap) { + assert(!Finalized && "Cannot merge after finalization"); + for (auto &[Hash, Funcs] : OtherMap.HashToFuncs) { + auto &ThisFuncs = HashToFuncs[Hash]; + for (auto &Func : Funcs) { + auto FuncNameId = + getIdOrCreateForName(*OtherMap.getNameForId(Func->FunctionNameId)); + auto ModuleNameId = + getIdOrCreateForName(*OtherMap.getNameForId(Func->ModuleNameId)); + auto ClonedIndexOperandHashMap = + std::make_unique(*Func->IndexOperandHashMap); + ThisFuncs.emplace_back(std::make_unique( + Func->Hash, FuncNameId, ModuleNameId, Func->InstCount, + std::move(ClonedIndexOperandHashMap))); + } + } +} + +size_t StableFunctionMap::size(SizeType Type) const { + switch (Type) { + case UniqueHashCount: + return HashToFuncs.size(); + case TotalFunctionCount: { + size_t Count = 0; + for (auto &Funcs : HashToFuncs) + Count += Funcs.second.size(); + return Count; + } + case MergeableFunctionCount: { + size_t Count = 0; + for (auto &[Hash, Funcs] : HashToFuncs) + if (Funcs.size() >= 2) + Count += Funcs.size(); + return Count; + } + } + return 0; +} + +using ParamLocs = SmallVector; +static void removeIdenticalIndexPair( + SmallVector> &SFS) { + auto &RSF = SFS[0]; + unsigned StableFunctionCount = SFS.size(); + + SmallVector ToDelete; + for (auto &[Pair, Hash] : *(RSF->IndexOperandHashMap)) { + bool Identical = true; + for (unsigned J = 1; J < StableFunctionCount; ++J) { + auto &SF = SFS[J]; + assert(SF->IndexOperandHashMap->count(Pair)); + auto SHash = (*SF->IndexOperandHashMap)[Pair]; + if (Hash != SHash) { + Identical = false; + break; + } + } + + // No need to parameterize them if the hashes are identical across stable + // functions. + if (Identical) + ToDelete.emplace_back(Pair); + } + + for (auto &Pair : ToDelete) + for (auto &SF : SFS) + SF->IndexOperandHashMap->erase(Pair); +} + +void StableFunctionMap::finalize() { + Finalized = true; + + for (auto It = HashToFuncs.begin(); It != HashToFuncs.end(); ++It) { + auto &[StableHash, SFS] = *It; + + // Group stable functions by ModuleIdentifier. + std::stable_sort(SFS.begin(), SFS.end(), + [&](const std::unique_ptr &L, + const std::unique_ptr &R) { + return *getNameForId(L->ModuleNameId) < + *getNameForId(R->ModuleNameId); + }); + + // Consider the first function as the root function. + auto &RSF = SFS[0]; + + bool IsValid = true; + unsigned StableFunctionCount = SFS.size(); + for (unsigned I = 1; I < StableFunctionCount; ++I) { + auto &SF = SFS[I]; + assert(RSF->Hash == SF->Hash); + if (RSF->InstCount != SF->InstCount) { + IsValid = false; + break; + } + if (RSF->IndexOperandHashMap->size() != SF->IndexOperandHashMap->size()) { + IsValid = false; + break; + } + for (auto &P : *RSF->IndexOperandHashMap) { + auto &InstOpndIndex = P.first; + if (!SF->IndexOperandHashMap->count(InstOpndIndex)) { + IsValid = false; + break; + } + } + } + if (!IsValid) { + HashToFuncs.erase(It); + continue; + } + + // Trim the index pair that has the same operand hash across + // stable functions. + removeIdenticalIndexPair(SFS); + } +} diff --git a/llvm/lib/CGData/StableFunctionMapRecord.cpp b/llvm/lib/CGData/StableFunctionMapRecord.cpp new file mode 100644 index 00000000000000..137a75cd0d73b6 --- /dev/null +++ b/llvm/lib/CGData/StableFunctionMapRecord.cpp @@ -0,0 +1,197 @@ +//===-- StableFunctionMapRecord.cpp ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TODO +// +//===----------------------------------------------------------------------===// + +#include "llvm/CGData/StableFunctionMapRecord.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/EndianStream.h" + +#define DEBUG_TYPE "stable-function-map-record" + +using namespace llvm; +using namespace llvm::support; + +LLVM_YAML_IS_SEQUENCE_VECTOR(IndexPairHash) +LLVM_YAML_IS_SEQUENCE_VECTOR(StableFunction) + +namespace llvm { +namespace yaml { + +template <> struct MappingTraits { + static void mapping(IO &IO, IndexPairHash &Key) { + IO.mapRequired("InstIndex", Key.first.first); + IO.mapRequired("OpndIndex", Key.first.second); + IO.mapRequired("OpndHash", Key.second); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, StableFunction &Func) { + IO.mapRequired("Hash", Func.Hash); + IO.mapRequired("FunctionName", Func.FunctionName); + IO.mapRequired("ModuleName", Func.ModuleName); + IO.mapRequired("InstCount", Func.InstCount); + IO.mapRequired("IndexOperandHashes", Func.IndexOperandHashes); + } +}; + +} // namespace yaml +} // namespace llvm + +// Get a sorted vector of StableFunctionEntry pointers. +static SmallVector +getStableFunctionEntries(StableFunctionMap &SFM) { + SmallVector FuncEntries; + for (const auto &P : SFM.getFunctionMap()) + for (auto &Func : P.second) + FuncEntries.emplace_back(Func.get()); + + std::stable_sort( + FuncEntries.begin(), FuncEntries.end(), [&](auto &A, auto &B) { + return std::tuple(A->Hash, SFM.getNameForId(A->ModuleNameId), + SFM.getNameForId(A->FunctionNameId)) < + std::tuple(B->Hash, SFM.getNameForId(B->ModuleNameId), + SFM.getNameForId(B->FunctionNameId)); + }); + return FuncEntries; +} + +// Get a sorted vector of IndexOperandHashes. +static IndexOperandHashVecType +getStableIndexOperandHashes(const StableFunctionEntry *FuncEntry) { + IndexOperandHashVecType IndexOperandHashes; + for (auto &[Indices, OpndHash] : *FuncEntry->IndexOperandHashMap) + IndexOperandHashes.emplace_back(Indices, OpndHash); + std::sort(IndexOperandHashes.begin(), IndexOperandHashes.end(), + [](auto &A, auto &B) { return A.first < B.first; }); + return IndexOperandHashes; +} + +void StableFunctionMapRecord::serialize(raw_ostream &OS) const { + support::endian::Writer Writer(OS, endianness::little); + + // Write Names. + auto &Names = FunctionMap->getNames(); + uint32_t ByteSize = 4; + Writer.write(Names.size()); + for (auto &Name : Names) { + Writer.OS << Name << '\0'; + ByteSize += Name.size() + 1; + } + // Align ByteSize to 4 bytes. + uint32_t Padding = offsetToAlignment(ByteSize, Align(4)); + for (uint32_t I = 0; I < Padding; ++I) + Writer.OS << '\0'; + + // Write StableFunctionEntries whose pointers are sorted. + auto FuncEntries = getStableFunctionEntries(*FunctionMap); + Writer.write(FuncEntries.size()); + + for (const auto *FuncRef : FuncEntries) { + Writer.write(FuncRef->Hash); + Writer.write(FuncRef->FunctionNameId); + Writer.write(FuncRef->ModuleNameId); + Writer.write(FuncRef->InstCount); + + // Emit IndexOperandHashes sorted from IndexOperandHashMap. + IndexOperandHashVecType IndexOperandHashes = + getStableIndexOperandHashes(FuncRef); + Writer.write(IndexOperandHashes.size()); + for (auto &IndexOperandHash : IndexOperandHashes) { + Writer.write(IndexOperandHash.first.first); + Writer.write(IndexOperandHash.first.second); + Writer.write(IndexOperandHash.second); + } + } +} + +void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr) { + // Assert that Ptr is 4-byte aligned + assert(((uintptr_t)Ptr % 4) == 0); + // Read Names. + auto NumNames = + endian::readNext(Ptr); + // Early exit if there is no name. + if (NumNames == 0) + return; + for (unsigned I = 0; I < NumNames; ++I) { + std::string Name(reinterpret_cast(Ptr)); + Ptr += Name.size() + 1; + FunctionMap->getIdOrCreateForName(Name); + } + // Align Ptr to 4 bytes. + Ptr = reinterpret_cast(alignAddr(Ptr, Align(4))); + + // Read StableFunctionEntries. + auto NumFuncs = + endian::readNext(Ptr); + for (unsigned I = 0; I < NumFuncs; ++I) { + auto Hash = + endian::readNext(Ptr); + auto FunctionNameId = + endian::readNext(Ptr); + assert(FunctionMap->getNameForId(FunctionNameId) && + "FunctionNameId out of range"); + auto ModuleNameId = + endian::readNext(Ptr); + assert(FunctionMap->getNameForId(ModuleNameId) && + "ModuleNameId out of range"); + auto InstCount = + endian::readNext(Ptr); + + // Read IndexOperandHashes to build IndexOperandHashMap + auto NumIndexOperandHashes = + endian::readNext(Ptr); + auto IndexOperandHashMap = std::make_unique(); + for (unsigned J = 0; J < NumIndexOperandHashes; ++J) { + auto InstIndex = + endian::readNext(Ptr); + auto OpndIndex = + endian::readNext(Ptr); + auto OpndHash = + endian::readNext(Ptr); + assert(InstIndex < InstCount && "InstIndex out of range"); + + auto Indices = std::make_pair(InstIndex, OpndIndex); + (*IndexOperandHashMap)[Indices] = OpndHash; + } + + // Insert a new StableFunctionEntry into the map. + auto FuncEntry = std::make_unique( + Hash, FunctionNameId, ModuleNameId, InstCount, + std::move(IndexOperandHashMap)); + + FunctionMap->insert(std::move(FuncEntry)); + } +} + +void StableFunctionMapRecord::serializeYAML(yaml::Output &YOS) const { + auto FuncEntries = getStableFunctionEntries(*FunctionMap); + SmallVector Functions; + for (const auto *FuncEntry : FuncEntries) { + auto IndexOperandHashes = getStableIndexOperandHashes(FuncEntry); + Functions.emplace_back( + FuncEntry->Hash, *FunctionMap->getNameForId(FuncEntry->FunctionNameId), + *FunctionMap->getNameForId(FuncEntry->ModuleNameId), + FuncEntry->InstCount, std::move(IndexOperandHashes)); + } + + YOS << Functions; +} + +void StableFunctionMapRecord::deserializeYAML(yaml::Input &YIS) { + std::vector Funcs; + YIS >> Funcs; + for (auto &Func : Funcs) + FunctionMap->insert(Func); + YIS.nextDocument(); +} diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp index fb4f33a021a96b..c88728c5f35719 100644 --- a/llvm/lib/IR/StructuralHash.cpp +++ b/llvm/lib/IR/StructuralHash.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/StructuralHash.h" -#include "llvm/ADT/Hashing.h" +#include "llvm/CodeGen/MachineStableHash.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InstrTypes.h" @@ -24,61 +24,222 @@ namespace { // by the MergeFunctions pass. class StructuralHashImpl { - uint64_t Hash = 4; + stable_hash Hash = 4; - void hash(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); } + bool DetailedHash; - // This will produce different values on 32-bit and 64-bit systens as - // hash_combine returns a size_t. However, this is only used for - // detailed hashing which, in-tree, only needs to distinguish between - // differences in functions. - template void hashArbitaryType(const T &V) { - hash(hash_combine(V)); - } + IgnoreOperandFunc IgnoreOp = nullptr; + std::unique_ptr IndexInstruction = nullptr; + std::unique_ptr IndexOperandHashMap = nullptr; + + DenseMap ValueToId; - void hashType(Type *ValueType) { - hash(ValueType->getTypeID()); - if (ValueType->isIntegerTy()) - hash(ValueType->getIntegerBitWidth()); + stable_hash hashType(Type *Ty) { + SmallVector Hashes; + Hashes.emplace_back(Ty->getTypeID()); + if (Ty->isIntegerTy()) + Hashes.emplace_back(Ty->getIntegerBitWidth()); + return stable_hash_combine(Hashes); } public: - StructuralHashImpl() = default; - - void updateOperand(Value *Operand) { - hashType(Operand->getType()); - - // The cases enumerated below are not exhaustive and are only aimed to - // get decent coverage over the function. - if (ConstantInt *ConstInt = dyn_cast(Operand)) { - hashArbitaryType(ConstInt->getValue()); - } else if (ConstantFP *ConstFP = dyn_cast(Operand)) { - hashArbitaryType(ConstFP->getValue()); - } else if (Argument *Arg = dyn_cast(Operand)) { - hash(Arg->getArgNo()); - } else if (Function *Func = dyn_cast(Operand)) { - // Hashing the name will be deterministic as LLVM's hashing infrastructure - // has explicit support for hashing strings and will not simply hash - // the pointer. - hashArbitaryType(Func->getName()); + StructuralHashImpl() = delete; + explicit StructuralHashImpl(bool DetailedHash, + IgnoreOperandFunc IgnoreOp = nullptr) + : DetailedHash(DetailedHash), IgnoreOp(IgnoreOp) { + if (IgnoreOp) { + IndexInstruction = std::make_unique(); + IndexOperandHashMap = std::make_unique(); } } - void updateInstruction(const Instruction &Inst, bool DetailedHash) { - hash(Inst.getOpcode()); + stable_hash hashAPInt(const APInt &I) { + SmallVector Hashes; + for (unsigned J = 0; J < I.getNumWords(); ++J) + Hashes.emplace_back((I.getRawData())[J]); + return stable_hash_combine(Hashes); + } + + stable_hash hashAPFloat(const APFloat &F) { + SmallVector Hashes; + const fltSemantics &S = F.getSemantics(); + Hashes.emplace_back(APFloat::semanticsPrecision(S)); + Hashes.emplace_back(APFloat::semanticsMaxExponent(S)); + Hashes.emplace_back(APFloat::semanticsMinExponent(S)); + Hashes.emplace_back(APFloat::semanticsSizeInBits(S)); + Hashes.emplace_back(hashAPInt(F.bitcastToAPInt())); + return stable_hash_combine(Hashes); + } + + stable_hash hashGlobalValue(const GlobalValue *GV) { + if (!GV->hasName()) + return 0; + return stable_hash_name(GV->getName()); + } + + stable_hash hashConstant(const Constant *C) { + SmallVector Hashes; + + Type *Ty = C->getType(); + Hashes.emplace_back(hashType(Ty)); + + if (C->isNullValue()) { + Hashes.emplace_back(static_cast('N')); + return stable_hash_combine(Hashes); + } + + auto *G = dyn_cast(C); + if (G) { + Hashes.emplace_back(hashGlobalValue(G)); + return stable_hash_combine(Hashes); + } + + if (const auto *Seq = dyn_cast(C)) { + Hashes.emplace_back(xxh3_64bits(Seq->getRawDataValues())); + return stable_hash_combine(Hashes); + } + + switch (C->getValueID()) { + case Value::UndefValueVal: + case Value::PoisonValueVal: + case Value::ConstantTokenNoneVal: { + return stable_hash_combine(Hashes); + } + case Value::ConstantIntVal: { + const APInt &Int = cast(C)->getValue(); + Hashes.emplace_back(hashAPInt(Int)); + return stable_hash_combine(Hashes); + } + case Value::ConstantFPVal: { + const APFloat &APF = cast(C)->getValueAPF(); + Hashes.emplace_back(hashAPFloat(APF)); + return stable_hash_combine(Hashes); + } + case Value::ConstantArrayVal: { + const ConstantArray *A = cast(C); + uint64_t NumElements = cast(Ty)->getNumElements(); + Hashes.emplace_back(NumElements); + for (uint64_t i = 0; i < NumElements; ++i) { + auto H = hashConstant(cast(A->getOperand(i))); + Hashes.emplace_back(H); + } + return stable_hash_combine(Hashes); + } + case Value::ConstantStructVal: { + const ConstantStruct *S = cast(C); + unsigned NumElements = cast(Ty)->getNumElements(); + Hashes.emplace_back(NumElements); + for (unsigned i = 0; i != NumElements; ++i) { + auto H = hashConstant(cast(S->getOperand(i))); + Hashes.emplace_back(H); + } + return stable_hash_combine(Hashes); + } + case Value::ConstantVectorVal: { + const ConstantVector *V = cast(C); + unsigned NumElements = cast(Ty)->getNumElements(); + Hashes.emplace_back(NumElements); + for (unsigned i = 0; i != NumElements; ++i) { + auto H = hashConstant(cast(V->getOperand(i))); + Hashes.emplace_back(H); + } + return stable_hash_combine(Hashes); + } + case Value::ConstantExprVal: { + const ConstantExpr *E = cast(C); + unsigned NumOperands = E->getNumOperands(); + Hashes.emplace_back(NumOperands); + for (unsigned i = 0; i < NumOperands; ++i) { + auto H = hashConstant(cast(E->getOperand(i))); + Hashes.emplace_back(H); + } + // TODO: GEPOperator + return stable_hash_combine(Hashes); + } + case Value::BlockAddressVal: { + const BlockAddress *BA = cast(C); + auto H = hashGlobalValue(BA->getFunction()); + Hashes.emplace_back(H); + // TODO: handle BBs in the same function. can we reference a block + // in another TU? + return stable_hash_combine(Hashes); + } + case Value::DSOLocalEquivalentVal: { + const auto *Equiv = cast(C); + auto H = hashGlobalValue(Equiv->getGlobalValue()); + Hashes.emplace_back(H); + return stable_hash_combine(Hashes); + } + default: // Unknown constant, abort. + llvm_unreachable("Constant ValueID not recognized."); + } + return Hash; + } + + /// Hash a value in order similar to FunctionCompartor::cmpValue(). + /// If this is the first time the value are seen, it's added to the mapping + /// so that we can use its index for hash computation. + stable_hash hashValue(Value *V) { + SmallVector Hashes; + + // Check constant and return its hash. + const Constant *C = dyn_cast(V); + if (C) { + Hashes.emplace_back(hashConstant(C)); + return stable_hash_combine(Hashes); + } + + // Hash argument number. + if (Argument *Arg = dyn_cast(V)) + Hashes.emplace_back(Arg->getArgNo()); + + // Get an index (an insertion order) for the non-constant value. + auto I = ValueToId.insert({V, ValueToId.size()}); + Hashes.emplace_back(I.first->second); + + return stable_hash_combine(Hashes); + } + + stable_hash hashOperand(Value *Operand) { + SmallVector Hashes; + Hashes.emplace_back(hashType(Operand->getType())); + Hashes.emplace_back(hashValue(Operand)); + + return stable_hash_combine(Hashes); + } + + stable_hash hashInstruction(const Instruction &Inst) { + SmallVector Hashes; + Hashes.emplace_back(Inst.getOpcode()); if (!DetailedHash) - return; + return stable_hash_combine(Hashes); - hashType(Inst.getType()); + Hashes.emplace_back(hashType(Inst.getType())); // Handle additional properties of specific instructions that cause // semantic differences in the IR. + // TODO: expand cmpOperations for different type of instructions if (const auto *ComparisonInstruction = dyn_cast(&Inst)) - hash(ComparisonInstruction->getPredicate()); + Hashes.emplace_back(ComparisonInstruction->getPredicate()); + + unsigned InstIdx = 0; + if (IndexInstruction) { + InstIdx = IndexInstruction->size(); + IndexInstruction->insert({InstIdx, const_cast(&Inst)}); + } + + for (unsigned OpndIdx = 0; OpndIdx < Inst.getNumOperands(); ++OpndIdx) { + auto *Op = Inst.getOperand(OpndIdx); + auto OpndHash = hashOperand(Op); + if (IgnoreOp && IgnoreOp(&Inst, OpndIdx)) { + assert(IndexOperandHashMap); + IndexOperandHashMap->insert({{InstIdx, OpndIdx}, OpndHash}); + } else + Hashes.emplace_back(OpndHash); + } - for (const auto &Op : Inst.operands()) - updateOperand(Op); + return stable_hash_combine(Hashes); } // A function hash is calculated by considering only the number of arguments @@ -97,15 +258,17 @@ class StructuralHashImpl { // expensive checks for pass modification status). When modifying this // function, most changes should be gated behind an option and enabled // selectively. - void update(const Function &F, bool DetailedHash) { + void update(const Function &F) { // Declarations don't affect analyses. if (F.isDeclaration()) return; - hash(0x62642d6b6b2d6b72); // Function header + SmallVector Hashes; + Hashes.emplace_back(Hash); + Hashes.emplace_back(0x62642d6b6b2d6b72); // Function header - hash(F.isVarArg()); - hash(F.arg_size()); + Hashes.emplace_back(F.isVarArg()); + Hashes.emplace_back(F.arg_size()); SmallVector BBs; SmallPtrSet VisitedBBs; @@ -121,14 +284,17 @@ class StructuralHashImpl { // This random value acts as a block header, as otherwise the partition of // opcodes into BBs wouldn't affect the hash, only the order of the // opcodes - hash(45798); + Hashes.emplace_back(45798); for (auto &Inst : *BB) - updateInstruction(Inst, DetailedHash); + Hashes.emplace_back(hashInstruction(Inst)); for (const BasicBlock *Succ : successors(BB)) if (VisitedBBs.insert(Succ).second) BBs.push_back(Succ); } + + // Update the combined hash in place. + Hash = stable_hash_combine(Hashes); } void update(const GlobalVariable &GV) { @@ -137,30 +303,50 @@ class StructuralHashImpl { // we ignore anything with the `.llvm` prefix if (GV.isDeclaration() || GV.getName().starts_with("llvm.")) return; - hash(23456); // Global header - hash(GV.getValueType()->getTypeID()); + SmallVector Hashes; + Hashes.emplace_back(Hash); + Hashes.emplace_back(23456); // Global header + Hashes.emplace_back(GV.getValueType()->getTypeID()); + + // Update the combined hash in place. + Hash = stable_hash_combine(Hashes); } - void update(const Module &M, bool DetailedHash) { + void update(const Module &M) { for (const GlobalVariable &GV : M.globals()) update(GV); for (const Function &F : M) - update(F, DetailedHash); + update(F); } uint64_t getHash() const { return Hash; } + std::unique_ptr getIndexInstrMap() { + return std::move(IndexInstruction); + } + std::unique_ptr getIndexPairOpndHashMap() { + return std::move(IndexOperandHashMap); + } }; } // namespace IRHash llvm::StructuralHash(const Function &F, bool DetailedHash) { - StructuralHashImpl H; - H.update(F, DetailedHash); + StructuralHashImpl H(DetailedHash); + H.update(F); return H.getHash(); } IRHash llvm::StructuralHash(const Module &M, bool DetailedHash) { - StructuralHashImpl H; - H.update(M, DetailedHash); + StructuralHashImpl H(DetailedHash); + H.update(M); return H.getHash(); } + +FunctionHashInfo +llvm::StructuralHashWithDifferences(const Function &F, + IgnoreOperandFunc IgnoreOp) { + StructuralHashImpl H(/*DetailedHash=*/true, IgnoreOp); + H.update(F); + return FunctionHashInfo(H.getHash(), H.getIndexInstrMap(), + H.getIndexPairOpndHashMap()); +} diff --git a/llvm/lib/LTO/CMakeLists.txt b/llvm/lib/LTO/CMakeLists.txt index 69ff08e1f374c4..057d73b6349cf1 100644 --- a/llvm/lib/LTO/CMakeLists.txt +++ b/llvm/lib/LTO/CMakeLists.txt @@ -21,6 +21,7 @@ add_llvm_component_library(LLVMLTO BinaryFormat BitReader BitWriter + CGData CodeGen CodeGenTypes Core diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index a88124dacfaefd..e2aa9dbc527da2 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -21,6 +21,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CGData/CodeGenData.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/AutoUpgrade.h" @@ -50,6 +51,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/GlobalMergeFunctions.h" #include "llvm/Transforms/IPO/MemProfContextDisambiguation.h" #include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" @@ -70,6 +72,8 @@ static cl::opt DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden, cl::desc("Dump the SCCs in the ThinLTO index's callgraph")); +extern cl::opt CodeGenDataThinLTOTwoRounds; + namespace llvm { /// Enable global value internalization in LTO. cl::opt EnableLTOInternalization( @@ -1458,7 +1462,7 @@ class InProcessThinBackend : public ThinBackendProc { GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); } - Error runThinLTOBackendThread( + virtual Error runThinLTOBackendThread( AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM, ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, @@ -1473,7 +1477,8 @@ class InProcessThinBackend : public ThinBackendProc { return MOrErr.takeError(); return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex, - ImportList, DefinedGlobals, &ModuleMap); + ImportList, DefinedGlobals, &ModuleMap, + Conf.CodeGenOnly); }; auto ModuleID = BM.getModuleIdentifier(); @@ -1558,6 +1563,69 @@ class InProcessThinBackend : public ThinBackendProc { return BackendThreadPool.getMaxConcurrency(); } }; + +/// This backend is utilized in the first round of a two-codegen round process. +/// It first saves optimized bitcode files to disk before the codegen process +/// begins. After codegen, it stores the resulting object files in a scratch +/// buffer. Note the codegen data stored in the scratch buffer will be extracted +/// and merged in the subsequent step. +class FirstRoundThinBackend : public InProcessThinBackend { +public: + FirstRoundThinBackend( + const Config &Conf, ModuleSummaryIndex &CombinedIndex, + ThreadPoolStrategy ThinLTOParallelism, + const DenseMap &ModuleToDefinedGVSummaries, + std::unique_ptr>> Scratch) + : InProcessThinBackend( + Conf, CombinedIndex, ThinLTOParallelism, ModuleToDefinedGVSummaries, + // Allocate a scratch buffer for each task to write output to. + [Allocation = &*Scratch](unsigned Task, const Twine &ModuleName) { + return std::make_unique( + std::make_unique((*Allocation)[Task])); + }, + FileCache(), /*OnWrite=*/nullptr, /*ShouldEmitIndexFiles=*/false, + /*ShouldEmitImportsFiles=*/false), + Scratch(std::move(Scratch)) {} + + /// Scratch space for writing output during the codegen. + std::unique_ptr>> Scratch; +}; + +/// This backend operates in the second round of a two-codegen round process. +/// It starts by reading the optimized bitcode files that were saved during the +/// first round. The backend then executes the codegen only to further optimize +/// the code, utilizing the codegen data merged from the first round. Finally, +/// it writes the resulting object files as usual. +class SecondRoundThinBackend : public InProcessThinBackend { +public: + SecondRoundThinBackend( + const Config &Conf, ModuleSummaryIndex &CombinedIndex, + ThreadPoolStrategy ThinLTOParallelism, + const DenseMap &ModuleToDefinedGVSummaries, + AddStreamFn AddStream) + : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism, + ModuleToDefinedGVSummaries, AddStream, FileCache(), + /*OnWrite=*/nullptr, + /*ShouldEmitIndexFiles=*/false, + /*ShouldEmitImportsFiles=*/false) {} + + virtual Error runThinLTOBackendThread( + AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM, + ModuleSummaryIndex &CombinedIndex, + const FunctionImporter::ImportMapTy &ImportList, + const FunctionImporter::ExportSetTy &ExportList, + const std::map &ResolvedODR, + const GVSummaryMapTy &DefinedGlobals, + MapVector &ModuleMap) override { + LTOLLVMContext BackendContext(Conf); + std::unique_ptr LoadedModule = + cgdata::loadModuleForTwoRounds(BM, Task, BackendContext); + + return thinBackend(Conf, Task, AddStream, *LoadedModule, CombinedIndex, + ImportList, DefinedGlobals, &ModuleMap, + /*CodeGenOnly=*/true); + } +}; } // end anonymous namespace ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism, @@ -1839,45 +1907,85 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, TimeTraceScopeExit.release(); - std::unique_ptr BackendProc = - ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, - AddStream, Cache); - auto &ModuleMap = ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap; - auto ProcessOneModule = [&](int I) -> Error { - auto &Mod = *(ModuleMap.begin() + I); - // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for - // combined module and parallel code generation partitions. - return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I, - Mod.second, ImportLists[Mod.first], - ExportLists[Mod.first], ResolvedODR[Mod.first], - ThinLTO.ModuleMap); + auto RunBackends = [&](ThinBackendProc *BackendProcess) -> Error { + auto ProcessOneModule = [&](int I) -> Error { + auto &Mod = *(ModuleMap.begin() + I); + // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for + // combined module and parallel code generation partitions. + return BackendProcess->start( + RegularLTO.ParallelCodeGenParallelismLevel + I, Mod.second, + ImportLists[Mod.first], ExportLists[Mod.first], + ResolvedODR[Mod.first], ThinLTO.ModuleMap); + }; + + if (BackendProcess->getThreadCount() == 1) { + // Process the modules in the order they were provided on the + // command-line. It is important for this codepath to be used for + // WriteIndexesThinBackend, to ensure the emitted LinkedObjectsFile lists + // ThinLTO objects in the same order as the inputs, which otherwise would + // affect the final link order. + for (int I = 0, E = ModuleMap.size(); I != E; ++I) + if (Error E = ProcessOneModule(I)) + return E; + } else { + // When executing in parallel, process largest bitsize modules first to + // improve parallelism, and avoid starving the thread pool near the end. + // This saves about 15 sec on a 36-core machine while link `clang.exe` + // (out of 100 sec). + std::vector ModulesVec; + ModulesVec.reserve(ModuleMap.size()); + for (auto &Mod : ModuleMap) + ModulesVec.push_back(&Mod.second); + for (int I : generateModulesOrdering(ModulesVec)) + if (Error E = ProcessOneModule(I)) + return E; + } + return BackendProcess->wait(); }; - if (BackendProc->getThreadCount() == 1) { - // Process the modules in the order they were provided on the command-line. - // It is important for this codepath to be used for WriteIndexesThinBackend, - // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same - // order as the inputs, which otherwise would affect the final link order. - for (int I = 0, E = ModuleMap.size(); I != E; ++I) - if (Error E = ProcessOneModule(I)) - return E; - } else { - // When executing in parallel, process largest bitsize modules first to - // improve parallelism, and avoid starving the thread pool near the end. - // This saves about 15 sec on a 36-core machine while link `clang.exe` (out - // of 100 sec). - std::vector ModulesVec; - ModulesVec.reserve(ModuleMap.size()); - for (auto &Mod : ModuleMap) - ModulesVec.push_back(&Mod.second); - for (int I : generateModulesOrdering(ModulesVec)) - if (Error E = ProcessOneModule(I)) - return E; + if (!CodeGenDataThinLTOTwoRounds) { + std::unique_ptr BackendProc = + ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, + AddStream, Cache); + return RunBackends(BackendProc.get()); } - return BackendProc->wait(); + + // Perform two rounds of code generation for ThinLTO: + // 1. First round: Run optimization and code generation with a scratch output. + // 2. Merge codegen data extracted from the scratch output. + // 3. Second round: Run code generation again using the merged data. + LLVM_DEBUG(dbgs() << "Running ThinLTO two-codegen rounds\n"); + + // Initialize a temporary path to store and retrieve optimized IRs for + // two-round code generation. + cgdata::initializeTwoCodegenRounds(); + + // Create a scratch output to hold intermediate results. + auto Outputs = + std::make_unique>>(getMaxTasks()); + auto FirstRoundLTO = std::make_unique( + Conf, ThinLTO.CombinedIndex, llvm::heavyweight_hardware_concurrency(), + ModuleToDefinedGVSummaries, std::move(Outputs)); + // First round: Run optimization and code generation with a scratch output. + // Before code generation, serialize modules. + if (Error E = RunBackends(FirstRoundLTO.get())) + return E; + + // Merge codegen data extracted from the scratch output. + if (Error E = cgdata::mergeCodeGenData(std::move(FirstRoundLTO->Scratch))) + return E; + + // Second round: Run code generation by reading IRs. + std::unique_ptr SecondRoundLTO = + std::make_unique( + Conf, ThinLTO.CombinedIndex, llvm::heavyweight_hardware_concurrency(), + ModuleToDefinedGVSummaries, AddStream); + Error E = RunBackends(SecondRoundLTO.get()); + + return E; } Expected> lto::setupLLVMOptimizationRemarks( diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp index 4e58cd369c3ac9..805300e920bfd7 100644 --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -20,6 +20,7 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CGData/CodeGenData.h" #include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/PassManager.h" @@ -74,6 +75,8 @@ static cl::opt ThinLTOAssumeMerged( cl::desc("Assume the input has already undergone ThinLTO function " "importing and the other pre-optimization pipeline changes.")); +extern cl::opt CodeGenDataThinLTOTwoRounds; + namespace llvm { extern cl::opt NoPGOWarnMismatch; } @@ -423,6 +426,7 @@ static void codegen(const Config &Conf, TargetMachine *TM, createImmutableModuleSummaryIndexWrapperPass(&CombinedIndex)); if (Conf.PreCodeGenPassesHook) Conf.PreCodeGenPassesHook(CodeGenPasses); + if (TM->addPassesToEmitFile(CodeGenPasses, *Stream->OS, DwoOut ? &DwoOut->os() : nullptr, Conf.CGFileType)) @@ -565,7 +569,7 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector *ModuleMap, - const std::vector &CmdArgs) { + bool CodeGenOnly, const std::vector &CmdArgs) { Expected TOrErr = initAndLookupTarget(Conf, Mod); if (!TOrErr) return TOrErr.takeError(); @@ -586,7 +590,9 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream, Mod.setPartialSampleProfileRatio(CombinedIndex); LLVM_DEBUG(dbgs() << "Running ThinLTO\n"); - if (Conf.CodeGenOnly) { + if (CodeGenOnly) { + // If CodeGenOnly is set, we only perform code generation and skip + // optimization. codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex); return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile)); } @@ -597,11 +603,19 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream, auto OptimizeAndCodegen = [&](Module &Mod, TargetMachine *TM, std::unique_ptr DiagnosticOutputFile) { + // Perform optimization and code generation for ThinLTO. if (!opt(Conf, TM, Task, Mod, /*IsThinLTO=*/true, /*ExportSummary=*/nullptr, /*ImportSummary=*/&CombinedIndex, CmdArgs)) return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile)); + // Save the current module before the first codegen round. + // Note that the second codegen round runs only `codegen()` without + // running `opt()`. We're not reaching here as it's bailed out earlier + // with `CodeGenOnly` which has been set in `SecondRoundThinBackend`. + if (CodeGenDataThinLTOTwoRounds) + cgdata::saveModuleForTwoRounds(Mod, Task); + codegen(Conf, TM, AddStream, Task, Mod, CombinedIndex); return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile)); }; diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 3dc7f185f330c5..01a284be78de46 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -75,6 +75,7 @@ MODULE_PASS("hipstdpar-interpose-alloc", HipStdParAllocationInterpositionPass()) MODULE_PASS("hipstdpar-select-accelerator-code", HipStdParAcceleratorCodeSelectionPass()) MODULE_PASS("hotcoldsplit", HotColdSplittingPass()) +//MODULE_PASS("global-merge-func", GlobalMergeFuncNewPM()) MODULE_PASS("inferattrs", InferFunctionAttrsPass()) MODULE_PASS("inliner-ml-advisor-release", ModuleInlinerWrapperPass(getInlineParams(), true, {}, diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt index 15cb57399d2460..6a36ed64149f93 100644 --- a/llvm/lib/Transforms/IPO/CMakeLists.txt +++ b/llvm/lib/Transforms/IPO/CMakeLists.txt @@ -21,6 +21,7 @@ add_llvm_component_library(LLVMipo GlobalDCE.cpp GlobalOpt.cpp GlobalSplit.cpp + GlobalMergeFunctions.cpp HotColdSplitting.cpp IPO.cpp IROutliner.cpp @@ -60,6 +61,7 @@ add_llvm_component_library(LLVMipo Analysis BitReader BitWriter + CGData Core FrontendOpenMP InstCombine diff --git a/llvm/lib/Transforms/IPO/GlobalMergeFunctions.cpp b/llvm/lib/Transforms/IPO/GlobalMergeFunctions.cpp new file mode 100644 index 00000000000000..ac90bda746bdeb --- /dev/null +++ b/llvm/lib/Transforms/IPO/GlobalMergeFunctions.cpp @@ -0,0 +1,731 @@ +//===---- GlobalMergeFunctions.cpp - Global merge functions -------*- C++ -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This implements a function merge using function hash. Like the pika merge +// functions, this can merge functions that differ by Constant operands thru +// parameterizing them. However, instead of directly comparing IR functions, +// this uses stable function hash to find potential merge candidates. +// This provides a flexible framework to implement a global function merge with +// ThinLTO two-codegen rounds. The first codegen round collects stable function +// hashes, and determines the merge candidates that match the stable function +// hashes. The set of parameters pointing to different Constants are also +// computed during the stable function merge. The second codegen round uses this +// global function info to optimistically create a merged function in each +// module context to guarantee correct transformation. Similar to the global +// outliner, the linker's deduplication (ICF) folds the identical merged +// functions to save the final binary size. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/IPO/GlobalMergeFunctions.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StableHashing.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Twine.h" +#include "llvm/CGData/CodeGenData.h" +#include "llvm/CGData/StableFunctionMap.h" +#include "llvm/CodeGen/MachineStableHash.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/StructuralHash.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include +#include +#include + +#define DEBUG_TYPE "global-merge-func" + +using namespace llvm; +using namespace llvm::support; + +cl::opt EnableGlobalMergeFunc( + "enable-global-merge-func", cl::init(false), cl::Hidden, + cl::desc("enable global merge functions (default = off)")); + +cl::opt GlobalMergeExtraThreshold( + "globalmergefunc-extra-threshold", + cl::desc("An extra cost threshold for merging. '0' disables the extra cost " + "and benefit analysis."), + cl::init(0), cl::Hidden); + +cl::opt DisableCrossModuleGlobalMergeFunc( + "disable-cross-module-global-merge-func", cl::init(false), cl::Hidden, + cl::desc("disable cross-module global merge functions. When this flag is " + "true, only local functions are merged by global merge func.")); + +STATISTIC(NumMismatchedFunctionHashGlobalMergeFunction, + "Number of mismatched function hash for global merge function"); +STATISTIC(NumMismatchedInstCountGlobalMergeFunction, + "Number of mismatched instruction count for global merge function"); +STATISTIC(NumMismatchedConstHashGlobalMergeFunction, + "Number of mismatched const hash for global merge function"); +STATISTIC(NumMismatchedIRGlobalMergeFunction, + "Number of mismatched IR for global merge function"); +STATISTIC(NumMismatchedModuleIdGlobalMergeFunction, + "Number of mismatched Module Id for global merge function"); +STATISTIC( + NumMismatchedGlobalMergeFunctionCandidates, + "Number of mismatched global merge function candidates that are skipped"); +STATISTIC(NumGlobalMergeFunctionCandidates, + "Number of global merge function candidates"); +STATISTIC(NumCrossModuleGlobalMergeFunctionCandidates, + "Number of cross-module global merge function candidates"); +STATISTIC(NumIdenticalGlobalMergeFunctionCandidates, + "Number of global merge function candidates that are identical (no " + "parameter)"); +STATISTIC(NumGlobalMergeFunctions, + "Number of functions that are actually merged using function hash"); +STATISTIC(NumAnalyzedModues, "Number of modules that are analyzed"); +STATISTIC(NumAnalyzedFunctions, "Number of functions that are analyzed"); +STATISTIC(NumEligibleFunctions, "Number of functions that are eligible"); + +// A singleton context for diagnostic output. +static LLVMContext Ctx; +class GlobalMergeFuncDiagnosticInfo : public DiagnosticInfo { + const Twine &Msg; + +public: + GlobalMergeFuncDiagnosticInfo(const Twine &DiagMsg, + DiagnosticSeverity Severity = DS_Error) + : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {} + void print(DiagnosticPrinter &DP) const override { DP << Msg; } +}; + +/// Returns true if the \opIdx operand of \p CI is the callee operand. +static bool isCalleeOperand(const CallBase *CI, unsigned opIdx) { + return &CI->getCalledOperandUse() == &CI->getOperandUse(opIdx); +} + +static bool canParameterizeCallOperand(const CallBase *CI, unsigned opIdx) { + if (CI->isInlineAsm()) + return false; + Function *Callee = CI->getCalledOperand() + ? dyn_cast_or_null( + CI->getCalledOperand()->stripPointerCasts()) + : nullptr; + if (Callee) { + if (Callee->isIntrinsic()) + return false; + // objc_msgSend stubs must be called, and can't have their address taken. + if (Callee->getName().starts_with("objc_msgSend$")) + return false; + } + if (isCalleeOperand(CI, opIdx) && + CI->getOperandBundle(LLVMContext::OB_ptrauth).has_value()) { + // The operand is the callee and it has already been signed. Ignore this + // because we cannot add another ptrauth bundle to the call instruction. + return false; + } + return true; +} + +bool isEligibleInstrunctionForConstantSharing(const Instruction *I) { + switch (I->getOpcode()) { + case Instruction::Load: + case Instruction::Store: + case Instruction::Call: + case Instruction::Invoke: + return true; + default: + return false; + } +} + +bool isEligibleOperandForConstantSharing(const Instruction *I, unsigned OpIdx) { + assert(OpIdx < I->getNumOperands() && "Invalid operand index"); + + if (!isEligibleInstrunctionForConstantSharing(I)) + return false; + + auto Opnd = I->getOperand(OpIdx); + if (!isa(Opnd)) + return false; + + if (const auto *CI = dyn_cast(I)) + return canParameterizeCallOperand(CI, OpIdx); + + return true; +} + +/// Returns true if function \p F is eligible for merging. +bool isEligibleFunction(Function *F) { + if (F->isDeclaration()) + return false; + + if (F->hasFnAttribute(llvm::Attribute::NoMerge)) + return false; + + if (F->hasAvailableExternallyLinkage()) { + return false; + } + + if (F->getFunctionType()->isVarArg()) { + return false; + } + + if (F->getCallingConv() == CallingConv::SwiftTail) + return false; + + // if function contains callsites with musttail, if we merge + // it, the merged function will have the musttail callsite, but + // the number of parameters can change, thus the parameter count + // of the callsite will mismatch with the function itself. + // if (IgnoreMusttailFunction) { + for (const BasicBlock &BB : *F) { + for (const Instruction &I : BB) { + const auto *CB = dyn_cast(&I); + if (CB && CB->isMustTailCall()) + return false; + } + } + + return true; +} + +static bool +isEligibleInstrunctionForConstantSharingLocal(const Instruction *I) { + switch (I->getOpcode()) { + case Instruction::Load: + case Instruction::Store: + case Instruction::Call: + case Instruction::Invoke: + return true; + default: + return false; + } +} + +static bool ignoreOp(const Instruction *I, unsigned OpIdx) { + assert(OpIdx < I->getNumOperands() && "Invalid operand index"); + + if (!isEligibleInstrunctionForConstantSharingLocal(I)) + return false; + + if (!isa(I->getOperand(OpIdx))) + return false; + + if (const auto *CI = dyn_cast(I)) + return canParameterizeCallOperand(CI, OpIdx); + + return true; +} + +// copy from merge functions.cpp +static Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy) { + Type *SrcTy = V->getType(); + if (SrcTy->isStructTy()) { + assert(DestTy->isStructTy()); + assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements()); + Value *Result = PoisonValue::get(DestTy); + for (unsigned int I = 0, E = SrcTy->getStructNumElements(); I < E; ++I) { + Value *Element = + createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)), + DestTy->getStructElementType(I)); + + Result = Builder.CreateInsertValue(Result, Element, ArrayRef(I)); + } + return Result; + } + assert(!DestTy->isStructTy()); + if (auto *SrcAT = dyn_cast(SrcTy)) { + auto *DestAT = dyn_cast(DestTy); + assert(DestAT); + assert(SrcAT->getNumElements() == DestAT->getNumElements()); + Value *Result = UndefValue::get(DestTy); + for (unsigned int I = 0, E = SrcAT->getNumElements(); I < E; ++I) { + Value *Element = + createCast(Builder, Builder.CreateExtractValue(V, ArrayRef(I)), + DestAT->getElementType()); + + Result = Builder.CreateInsertValue(Result, Element, ArrayRef(I)); + } + return Result; + } + assert(!DestTy->isArrayTy()); + if (SrcTy->isIntegerTy() && DestTy->isPointerTy()) + return Builder.CreateIntToPtr(V, DestTy); + else if (SrcTy->isPointerTy() && DestTy->isIntegerTy()) + return Builder.CreatePtrToInt(V, DestTy); + else + return Builder.CreateBitCast(V, DestTy); +} + +void GlobalMergeFunc::analyze(Module &M) { + ++NumAnalyzedModues; + for (Function &Func : M) { + ++NumAnalyzedFunctions; + if (isEligibleFunction(&Func)) { + ++NumEligibleFunctions; + + auto FI = llvm::StructuralHashWithDifferences(Func, ignoreOp); + + // TODO: move this vector implementation inside + // StructuralHashWithDifferences. + IndexOperandHashVecType IndexOperandHashes; + for (auto &Pair : *FI.IndexOperandHashMap) + IndexOperandHashes.emplace_back(Pair); + + StableFunction SF(FI.FunctionHash, get_stable_name(Func.getName()).str(), + M.getModuleIdentifier(), FI.IndexInstruction->size(), + std::move(IndexOperandHashes)); + + LocalFunctionMap->insert(SF); + } + } +} + +/// Tuple to hold function info to process merging. +struct FuncMergeInfo { + StableFunctionEntry *SF; + Function *F; + std::unique_ptr IndexInstruction; +}; + +// Given the func info, and the parameterized locations, create and return +// a new merged function by replacing the original constants with the new +// parameters. +static Function *createMergedFunction(FuncMergeInfo &FI, + ArrayRef ConstParamTypes, + const ParamLocsVecTy &ParamLocsVec) { + // Synthesize a new merged function name by appending ".Tgm" to the root + // function's name. + auto *MergedFunc = FI.F; + auto NewFunctionName = + MergedFunc->getName().str() + ".Tgm"; // TODO MergeFunctionInfo::Suffix; + auto *M = MergedFunc->getParent(); + assert(!M->getFunction(NewFunctionName)); + + FunctionType *OrigTy = MergedFunc->getFunctionType(); + // Get the original params' types. + SmallVector ParamTypes(OrigTy->param_begin(), OrigTy->param_end()); + // Append const parameter types that are passed in. + ParamTypes.append(ConstParamTypes.begin(), ConstParamTypes.end()); + FunctionType *FuncType = + FunctionType::get(OrigTy->getReturnType(), ParamTypes, false); + + // Declare a new function + Function *NewFunction = + Function::Create(FuncType, MergedFunc->getLinkage(), NewFunctionName); + if (auto *SP = MergedFunc->getSubprogram()) + NewFunction->setSubprogram(SP); + NewFunction->copyAttributesFrom(MergedFunc); + NewFunction->setDLLStorageClass(GlobalValue::DefaultStorageClass); + + NewFunction->setLinkage(GlobalValue::InternalLinkage); + NewFunction->addFnAttr(Attribute::NoInline); + + // Add the new function before the root function. + M->getFunctionList().insert(MergedFunc->getIterator(), NewFunction); + + // Move the body of MergedFunc into the NewFunction. + NewFunction->splice(NewFunction->begin(), MergedFunc); + + // Update the original args by the new args. + auto NewArgIter = NewFunction->arg_begin(); + for (Argument &OrigArg : MergedFunc->args()) { + Argument &NewArg = *NewArgIter++; + OrigArg.replaceAllUsesWith(&NewArg); + } + + // Replace the original Constants by the new args. + unsigned NumOrigArgs = MergedFunc->arg_size(); + for (unsigned ParamIdx = 0; ParamIdx < ParamLocsVec.size(); ++ParamIdx) { + Argument *NewArg = NewFunction->getArg(NumOrigArgs + ParamIdx); + for (auto [InstIndex, OpndIndex] : ParamLocsVec[ParamIdx]) { + auto *Inst = FI.IndexInstruction->lookup(InstIndex); + auto *OrigC = Inst->getOperand(OpndIndex); + if (OrigC->getType() != NewArg->getType()) { + IRBuilder<> Builder(Inst->getParent(), Inst->getIterator()); + Inst->setOperand(OpndIndex, + createCast(Builder, NewArg, OrigC->getType())); + } else + Inst->setOperand(OpndIndex, NewArg); + } + } + + return NewFunction; +} + +// Given the original function (Thunk) and the merged function (ToFunc), create +// a thunk to the merged function. + +static void createThunk(FuncMergeInfo &FI, ArrayRef Params, + Function *ToFunc) { + auto *Thunk = FI.F; + + assert(Thunk->arg_size() + Params.size() == + ToFunc->getFunctionType()->getNumParams()); + Thunk->dropAllReferences(); + + BasicBlock *BB = BasicBlock::Create(Thunk->getContext(), "", Thunk); + IRBuilder<> Builder(BB); + + SmallVector Args; + unsigned ParamIdx = 0; + FunctionType *ToFuncTy = ToFunc->getFunctionType(); + + // Add arguments which are passed through Thunk. + for (Argument &AI : Thunk->args()) { + Args.push_back(createCast(Builder, &AI, ToFuncTy->getParamType(ParamIdx))); + ++ParamIdx; + } + + // Add new arguments defined by Params. + for (auto *Param : Params) { + assert(ParamIdx < ToFuncTy->getNumParams()); + // FIXME: do not support signing + Args.push_back( + createCast(Builder, Param, ToFuncTy->getParamType(ParamIdx))); + ++ParamIdx; + } + + CallInst *CI = Builder.CreateCall(ToFunc, Args); + bool isSwiftTailCall = ToFunc->getCallingConv() == CallingConv::SwiftTail && + Thunk->getCallingConv() == CallingConv::SwiftTail; + CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail + : llvm::CallInst::TCK_Tail); + CI->setCallingConv(ToFunc->getCallingConv()); + CI->setAttributes(ToFunc->getAttributes()); + if (Thunk->getReturnType()->isVoidTy()) { + Builder.CreateRetVoid(); + } else { + Builder.CreateRet(createCast(Builder, CI, Thunk->getReturnType())); + } +} + +// Check if the old merged/optimized IndexOperandHashMap is compatible with +// the current IndexOperandHashMap. OpndHash may not be stable across +// different builds due to varying modules combined. To address this, one +// solution could be to relax the hash computation for Const in +// PikaFunctionHash. However, instead of doing so, we relax the hash check +// condition by comparing Const hash patterns instead of absolute hash values. +// For example, let's assume we have three Consts located at idx1, idx3, and +// idx6, where their corresponding hashes are hash1, hash2, and hash1 in the old +// merged map below: +// Old (Merged): [(idx1, hash1), (idx3, hash2), (idx6, hash1)] +// Current: [(idx1, hash1'), (idx3, hash2'), (idx6, hash1')] +// If the current function also has three Consts in the same locations, +// with hash sequences hash1', hash2', and hash1' where the first and third +// are the same as the old hash sequences, we consider them matched. +static bool checkConstHashCompatible( + const DenseMap &OldInstOpndIndexToConstHash, + const DenseMap &CurrInstOpndIndexToConstHash) { + + DenseMap OldHashToCurrHash; + for (const auto &[Index, OldHash] : OldInstOpndIndexToConstHash) { + auto It = CurrInstOpndIndexToConstHash.find(Index); + if (It == CurrInstOpndIndexToConstHash.end()) + return false; + + auto CurrHash = It->second; + auto J = OldHashToCurrHash.find(OldHash); + if (J == OldHashToCurrHash.end()) + OldHashToCurrHash.insert({OldHash, CurrHash}); + else if (J->second != CurrHash) + return false; + } + + return true; +} + +// Validate the locations pointed by a param has the same hash and Constant. +static bool checkConstLocationCompatible(const StableFunctionEntry &SF, + const IndexInstrMap &IndexInstruction, + const ParamLocsVecTy &ParamLocsVec) { + for (auto &ParamLocs : ParamLocsVec) { + std::optional OldHash; + std::optional OldConst; + for (auto &Loc : ParamLocs) { + assert(SF.IndexOperandHashMap->count(Loc)); + auto CurrHash = SF.IndexOperandHashMap.get()->at(Loc); + auto [InstIndex, OpndIndex] = Loc; + assert(InstIndex < IndexInstruction.size()); + const auto *Inst = IndexInstruction.lookup(InstIndex); + auto *CurrConst = cast(Inst->getOperand(OpndIndex)); + if (!OldHash) { + OldHash = CurrHash; + OldConst = CurrConst; + } else if (CurrConst != *OldConst || CurrHash != *OldHash) + return false; + } + } + return true; +} + +static ParamLocsVecTy +computeParamInfo(const SmallVector> &SFS) { + std::map, ParamLocs> HashSeqToLocs; + auto &RSF = *SFS[0]; + unsigned StableFunctionCount = SFS.size(); + + for (auto &[IndexPair, Hash] : *RSF.IndexOperandHashMap) { + // Const hash sequence across stable functions. + // We will allocate a parameter per unique hash squence. + // can't use SmallVector as key + std::vector ConstHashSeq; + ConstHashSeq.push_back(Hash); + bool Identical = true; + for (unsigned J = 1; J < StableFunctionCount; ++J) { + auto &SF = SFS[J]; + assert(SF->IndexOperandHashMap->count(IndexPair)); + auto SHash = (*SF->IndexOperandHashMap)[IndexPair]; + if (Hash != SHash) + Identical = false; + ConstHashSeq.push_back(SHash); + } + +// we've already minimized the Const hash sequence. +#if 0 + (void)(Identical); + assert(!Identical && + "Function Map has not been finalized or minimized before"); +#endif + if (Identical) + continue; + + // For each unique Const hash sequence (parameter), add the locations. + HashSeqToLocs[ConstHashSeq].push_back(IndexPair); + } + + ParamLocsVecTy ParamLocsVec; + for (auto &[HashSeq, Locs] : HashSeqToLocs) { + ParamLocsVec.push_back(std::move(Locs)); + std::sort( + ParamLocsVec.begin(), ParamLocsVec.end(), + [&](const ParamLocs &L, const ParamLocs &R) { return L[0] < R[0]; }); + } + return ParamLocsVec; +} + +bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) { + bool Changed = false; + + // Build a map from stable function name to function. + StringMap StableNameToFuncMap; + for (auto &F : M) + StableNameToFuncMap[get_stable_name(F.getName())] = &F; + // Track merged functions + DenseSet MergedFunctions; + + auto ModId = M.getModuleIdentifier(); + for (auto &[Hash, SFS] : FunctionMap->getFunctionMap()) { + // No interest if the number of candidates are less than 2. + if (SFS.size() < 2) + continue; + // Compute the parameter locations based on the unique hash sequences + // across the candidates. + auto ParamLocsVec = computeParamInfo(SFS); + LLVM_DEBUG({ + dbgs() << "[GlobalMergeFunc] Merging hash: " << Hash << " with Params " + << ParamLocsVec.size() << "\n"; + }); + + Function *MergedFunc = nullptr; + std::string MergedModId; + SmallVector FuncMergeInfos; + for (auto &SF : SFS) { + // Get the function from the stable name. + auto I = StableNameToFuncMap.find( + *FunctionMap->getNameForId(SF->FunctionNameId)); + if (I == StableNameToFuncMap.end()) + continue; + Function *F = I->second; + assert(F); + // Skip if the function has been merged before. + if (MergedFunctions.count(F)) + continue; + // Consider the function if it is eligible for merging. + if (!isEligibleFunction(F)) + continue; + + auto FI = llvm::StructuralHashWithDifferences(*F, ignoreOp); + uint64_t FuncHash = FI.FunctionHash; + if (Hash != FuncHash) { + ++NumMismatchedFunctionHashGlobalMergeFunction; + continue; + } + + if (SF->InstCount != FI.IndexInstruction->size()) { + ++NumMismatchedInstCountGlobalMergeFunction; + continue; + } + bool HasValidSharedConst = true; + for (auto &[Index, Hash] : *SF->IndexOperandHashMap) { + auto [InstIndex, OpndIndex] = Index; + assert(InstIndex < FI.IndexInstruction->size()); + auto *Inst = FI.IndexInstruction->lookup(InstIndex); + if (!isEligibleOperandForConstantSharing(Inst, OpndIndex)) { + HasValidSharedConst = false; + break; + } + } + if (!HasValidSharedConst) { + ++NumMismatchedConstHashGlobalMergeFunction; + continue; + } + if (!checkConstHashCompatible(*SF->IndexOperandHashMap, + *FI.IndexOperandHashMap)) { + ++NumMismatchedConstHashGlobalMergeFunction; + continue; + } + if (!checkConstLocationCompatible(*SF, *FI.IndexInstruction, + ParamLocsVec)) { + ++NumMismatchedConstHashGlobalMergeFunction; + continue; + } + + if (MergedFunc) { + // Check if the matched functions fall into the same (first) module. + // This module check is not strictly necessary as the functions can move + // around. We just want to avoid merging functions from different + // modules than the first one in the functon map, as they may not end up + // with not being ICFed. + if (MergedModId != *FunctionMap->getNameForId(SF->ModuleNameId)) { + ++NumMismatchedModuleIdGlobalMergeFunction; + continue; + } + } else { + MergedFunc = F; + MergedModId = *FunctionMap->getNameForId(SF->ModuleNameId); + } + + FuncMergeInfos.push_back({SF.get(), F, std::move(FI.IndexInstruction)}); + MergedFunctions.insert(F); + } + unsigned FuncMergeInfoSize = FuncMergeInfos.size(); + if (FuncMergeInfoSize == 0) + continue; + + LLVM_DEBUG({ + dbgs() << "[GlobalMergeFunc] Merging function count " << FuncMergeInfoSize + << " in " << ModId << "\n"; + }); + for (auto &FMI : FuncMergeInfos) { + Changed = true; + + // We've already validated all locations of constant operands pointed by + // the parameters. Just use the first one to bookkeep the original + // constants for each parameter + SmallVector Params; + SmallVector ParamTypes; + for (auto &ParamLocs : ParamLocsVec) { + assert(!ParamLocs.empty()); + auto &[InstIndex, OpndIndex] = ParamLocs[0]; + auto *Inst = FMI.IndexInstruction->lookup(InstIndex); + auto *Opnd = cast(Inst->getOperand(OpndIndex)); + Params.push_back(Opnd); + ParamTypes.push_back(Opnd->getType()); + } + + // Create a merged function derived from the first function in the current + // module context. + Function *MergedFunc = + createMergedFunction(FMI, ParamTypes, ParamLocsVec); + + LLVM_DEBUG({ + dbgs() << "[GlobalMergeFunc] Merged function (hash:" << FMI.SF->Hash + << ") " << MergedFunc->getName() << " generated from " + << FMI.F->getName() << ":\n"; + MergedFunc->dump(); + }); + + // Create a thunk to the merged function. + createThunk(FMI, Params, MergedFunc); + LLVM_DEBUG({ + dbgs() << "[GlobalMergeFunc] Thunk generated: \n"; + FMI.F->dump(); + }); + ++NumGlobalMergeFunctions; + } + } + + return Changed; +} + +char GlobalMergeFunc::ID = 0; +INITIALIZE_PASS_BEGIN(GlobalMergeFunc, "global-merge-func", + "Global merge function pass", false, false) +INITIALIZE_PASS_END(GlobalMergeFunc, "global-merge-func", + "Global merge function pass", false, false) + +GlobalMergeFunc::GlobalMergeFunc() : ModulePass(ID) { + initializeGlobalMergeFuncPass(*llvm::PassRegistry::getPassRegistry()); +} + +namespace llvm { +Pass *createGlobalMergeFuncPass() { return new GlobalMergeFunc(); } +} // namespace llvm + +void GlobalMergeFunc::initializeMergerMode(const Module &M) { + // When codegen data write is enabled, we want to write the local outlined + // hash tree to the custom section, `__llvm_outline`. + // When the outlined hash tree is available from the previous codegen data, + // we want to read it to optimistically create global outlining candidates. + LocalFunctionMap = std::make_unique(); + if (cgdata::emitCGData()) + MergerMode = HashFunctionMode::BuildingHashFuncion; + else if (cgdata::hasStableFunctionMap()) + MergerMode = HashFunctionMode::UsingHashFunction; +} + +void GlobalMergeFunc::emitFunctionMap(Module &M) { + if (!LocalFunctionMap->empty()) { + LLVM_DEBUG({ + dbgs() << "Emit function map. Size: " << LocalFunctionMap->size() << "\n"; + }); + SmallVector Buf; + raw_svector_ostream OS(Buf); + + StableFunctionMapRecord SFR(std::move(LocalFunctionMap)); + SFR.serialize(OS); + + llvm::StringRef Data(Buf.data(), Buf.size()); + std::unique_ptr Buffer = MemoryBuffer::getMemBuffer( + Data, "in-memory stable function map", false); + + Triple TT(M.getTargetTriple()); + embedBufferInModule( + M, *Buffer.get(), + getCodeGenDataSectionName(CG_merge, TT.getObjectFormat()), Align(4)); + } +} + +bool GlobalMergeFunc::runOnModule(Module &M) { + bool Changed = false; + initializeMergerMode(M); + + switch (MergerMode) { + case HashFunctionMode::BuildingHashFuncion: { + analyze(M); + emitFunctionMap(M); + break; + } + case HashFunctionMode::UsingHashFunction: { + // Merge with the global function map read from CG data. + Changed = merge(M, cgdata::getStableFunctionMap()); + break; + } + default: { + // Local merge case (within a module) in one-pass without explicitly using + // CGData. + analyze(M); + LocalFunctionMap->finalize(); + Changed = merge(M, LocalFunctionMap.get()); + } + } + + return Changed; +} diff --git a/llvm/test/ThinLTO/AArch64/cgdata-two-rounds.ll b/llvm/test/ThinLTO/AArch64/cgdata-two-rounds.ll new file mode 100644 index 00000000000000..0e082cf4e55e54 --- /dev/null +++ b/llvm/test/ThinLTO/AArch64/cgdata-two-rounds.ll @@ -0,0 +1,94 @@ +; This test verifies whether we can outline a singleton instance (i.e., an instance that does not repeat) +; by running two codegen rounds. + +; RUN: split-file %s %t + +; Verify each outlining instance is singleton with the global outlining for thinlto. +; They will be identical, which can be folded by the linker with ICF. +; RUN: opt -module-summary %t/thin-one.ll -o %t/thin-one.bc +; RUN: opt -module-summary %t/thin-two.ll -o %t/thin-two.bc +; RUN: llvm-lto2 run %t/thin-one.bc %t/thin-two.bc -o %t/thinlto \ +; RUN: -r %t/thin-one.bc,_f3,px -r %t/thin-one.bc,_g,x \ +; RUN: -r %t/thin-two.bc,_f1,px -r %t/thin-two.bc,_f2,px -r %t/thin-two.bc,_g,x \ +; RUN: -codegen-data-thinlto-two-rounds + +; thin-one.ll will have one outlining instance (matched in the global outlined hash tree) +; RUN: llvm-objdump -d %t/thinlto.1 | FileCheck %s --check-prefix=THINLTO-1 +; THINLTO-1: _OUTLINED_FUNCTION{{.*}}>: +; THINLTO-1-NEXT: mov +; THINLTO-1-NEXT: mov +; THINLTO-1-NEXT: b + +; thin-two.ll will have two outlining instances (matched in the global outlined hash tree) +; RUN: llvm-objdump -d %t/thinlto.2 | FileCheck %s --check-prefix=THINLTO-2 +; THINLTO-2: _OUTLINED_FUNCTION{{.*}}>: +; THINLTO-2-NEXT: mov +; THINLTO-2-NEXT: mov +; THINLTO-2-NEXT: b +; THINLTO-2: _OUTLINED_FUNCTION{{.*}}>: +; THINLTO-2-NEXT: mov +; THINLTO-2-NEXT: mov +; THINLTO-2-NEXT: b + +; Now add a lto module to the above thinlto modules. +; Verify the lto module is optimized independent of the global outlining for thinlto. +; RUN: opt %t/lto.ll -o %t/lto.bc +; RUN: llvm-lto2 run %t/thin-one.bc %t/thin-two.bc %t/lto.bc -o %t/out \ +; RUN: -r %t/thin-one.bc,_f3,px -r %t/thin-one.bc,_g,x \ +; RUN: -r %t/thin-two.bc,_f1,px -r %t/thin-two.bc,_f2,px -r %t/thin-two.bc,_g,x \ +; RUN: -r %t/lto.bc,_f4,px -r %t/lto.bc,_f5,px -r %t/lto.bc,_f6,px -r %t/lto.bc,_g,x \ +; RUN: -codegen-data-thinlto-two-rounds + +; lto.ll will have one outlining instance within the lto module itself (no global outlining). +; RUN: llvm-objdump -d %t/out.0 | FileCheck %s --check-prefix=LTO-0 +; LTO-0: _OUTLINED_FUNCTION{{.*}}>: +; LTO-0-NEXT: mov +; LTO-0-NEXT: b + +; thin-one.ll will have one outlining instance (matched in the global outlined hash tree) +; RUN: llvm-objdump -d %t/out.1 | FileCheck %s --check-prefix=THINLTO-1 + +; thin-two.ll will have two outlining instances (matched in the global outlined hash tree) +; RUN: llvm-objdump -d %t/out.2 | FileCheck %s --check-prefix=THINLTO-2 + +;--- thin-one.ll +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-darwin" + +declare i32 @g(i32, i32, i32) +define i32 @f3() minsize { + %1 = call i32 @g(i32 30, i32 1, i32 2); + ret i32 %1 +} + +;--- thin-two.ll +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-darwin" + +declare i32 @g(i32, i32, i32) +define i32 @f1() minsize { + %1 = call i32 @g(i32 10, i32 1, i32 2); + ret i32 %1 +} +define i32 @f2() minsize { + %1 = call i32 @g(i32 20, i32 1, i32 2); + ret i32 %1 +} + +;--- lto.ll +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-darwin" + +declare i32 @g(i32, i32, i32) +define i32 @f4() minsize { + %1 = call i32 @g(i32 10, i32 30, i32 2); + ret i32 %1 +} +define i32 @f5() minsize { + %1 = call i32 @g(i32 20, i32 40, i32 2); + ret i32 %1 +} +define i32 @f6() minsize { + %1 = call i32 @g(i32 50, i32 60, i32 2); + ret i32 %1 +} diff --git a/llvm/test/ThinLTO/AArch64/lit.local.cfg b/llvm/test/ThinLTO/AArch64/lit.local.cfg new file mode 100644 index 00000000000000..10d4a0e953ed47 --- /dev/null +++ b/llvm/test/ThinLTO/AArch64/lit.local.cfg @@ -0,0 +1,2 @@ +if not "AArch64" in config.root.targets: + config.unsupported = True diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll index e7718ca84d3165..3c18a3f7bdefa3 100644 --- a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll +++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll @@ -63,6 +63,14 @@ lpad: resume { ptr, i32 } zeroinitializer } +define i8 @call_with_same_range() { +; CHECK-LABEL: @call_with_same_range +; CHECK: tail call i8 @call_with_range + bitcast i8 0 to i8 + %out = call i8 @dummy(), !range !0 + ret i8 %out +} + define i8 @invoke_with_same_range() personality ptr undef { ; CHECK-LABEL: @invoke_with_same_range() ; CHECK: tail call i8 @invoke_with_range() @@ -76,14 +84,6 @@ lpad: resume { ptr, i32 } zeroinitializer } -define i8 @call_with_same_range() { -; CHECK-LABEL: @call_with_same_range -; CHECK: tail call i8 @call_with_range - bitcast i8 0 to i8 - %out = call i8 @dummy(), !range !0 - ret i8 %out -} - declare i8 @dummy(); declare i32 @__gxx_personality_v0(...) diff --git a/llvm/test/tools/llvm-cgdata/empty.test b/llvm/test/tools/llvm-cgdata/empty.test index 70d5ea4b800630..bea78d512a6db7 100644 --- a/llvm/test/tools/llvm-cgdata/empty.test +++ b/llvm/test/tools/llvm-cgdata/empty.test @@ -16,7 +16,7 @@ RUN: llvm-cgdata --show %t_emptyheader.cgdata | count 0 # The version number appears when asked, as it's in the header RUN: llvm-cgdata --show --cgdata-version %t_emptyheader.cgdata | FileCheck %s --check-prefix=VERSION -VERSION: Version: 1 +VERSION: Version: 2 # When converting a binary file (w/ the header only) to a text file, it's an empty file as the text format does not have an explicit header. RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0 @@ -27,9 +27,11 @@ RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0 # uint32_t Version; # uint32_t DataKind; # uint64_t OutlinedHashTreeOffset; +# uint64_t StableFunctionMapOffset; # } RUN: printf '\xffcgdata\x81' > %t_header.cgdata -RUN: printf '\x01\x00\x00\x00' >> %t_header.cgdata +RUN: printf '\x02\x00\x00\x00' >> %t_header.cgdata RUN: printf '\x00\x00\x00\x00' >> %t_header.cgdata -RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata +RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata +RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata RUN: diff %t_header.cgdata %t_emptyheader.cgdata diff --git a/llvm/test/tools/llvm-cgdata/error.test b/llvm/test/tools/llvm-cgdata/error.test index c992174505c1ad..2caa3aef403950 100644 --- a/llvm/test/tools/llvm-cgdata/error.test +++ b/llvm/test/tools/llvm-cgdata/error.test @@ -6,6 +6,7 @@ # uint32_t Version; # uint32_t DataKind; # uint64_t OutlinedHashTreeOffset; +# uint64_t StableFunctionMapOffset; # } RUN: touch %t_empty.cgdata RUN: not llvm-cgdata --show %t_empty.cgdata 2>&1 | FileCheck %s --check-prefix=EMPTY @@ -21,18 +22,20 @@ RUN: printf '\xffcgdata\x81' > %t_corrupt.cgdata RUN: not llvm-cgdata --show %t_corrupt.cgdata 2>&1 | FileCheck %s --check-prefix=CORRUPT CORRUPT: {{.}}cgdata: invalid codegen data (file header is corrupt) -# The current version 1 while the header says 2. +# The current version 2 while the header says 3. RUN: printf '\xffcgdata\x81' > %t_version.cgdata -RUN: printf '\x02\x00\x00\x00' >> %t_version.cgdata +RUN: printf '\x03\x00\x00\x00' >> %t_version.cgdata RUN: printf '\x00\x00\x00\x00' >> %t_version.cgdata -RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata +RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata +RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata RUN: not llvm-cgdata --show %t_version.cgdata 2>&1 | FileCheck %s --check-prefix=BAD_VERSION BAD_VERSION: {{.}}cgdata: unsupported codegen data version # Header says an outlined hash tree, but the file ends after the header. RUN: printf '\xffcgdata\x81' > %t_eof.cgdata +RUN: printf '\x02\x00\x00\x00' >> %t_eof.cgdata RUN: printf '\x01\x00\x00\x00' >> %t_eof.cgdata -RUN: printf '\x01\x00\x00\x00' >> %t_eof.cgdata -RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_eof.cgdata +RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_eof.cgdata +RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_eof.cgdata RUN: not llvm-cgdata --show %t_eof.cgdata 2>&1 | FileCheck %s --check-prefix=EOF EOF: {{.}}cgdata: end of File diff --git a/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test b/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test new file mode 100644 index 00000000000000..b9bf067d3771c5 --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test @@ -0,0 +1,66 @@ +# REQUIRES: shell, aarch64-registered-target +# UNSUPPORTED: system-windows + +# Test merge a single object file having both __llvm_outline and __llvm_merge into a cgdata. +# Effectively, this test combines merge-hashtree.test and merge-funcmap.test. + +RUN: split-file %s %t + +# Synthesize raw hashtree bytes without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-hashtree.cgtext -o %t/raw-hashtree.cgdata +RUN: od -t x1 -j 32 -An %t/raw-hashtree.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-hashtree-bytes.txt + +# Synthesize raw funcmap bytes without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-funcmap.cgtext -o %t/raw-funcmap.cgdata +RUN: od -t x1 -j 32 -An %t/raw-funcmap.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-funcmap-bytes.txt + +# Synthesize a bitcode file by creating two sections for the hash tree and the function map, respectively. +RUN: sed "s//$(cat %t/raw-hashtree-bytes.txt)/g" %t/merge-both-template.ll > %t/merge-both-hashtree-template.ll +RUN: sed "s//$(cat %t/raw-funcmap-bytes.txt)/g" %t/merge-both-hashtree-template.ll > %t/merge-both-hashtree-funcmap.ll + +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-both-hashtree-funcmap.ll -o %t/merge-both-hashtree-funcmap.o + +# Merge an object file having cgdata (__llvm_outline and __llvm_merge) +RUN: llvm-cgdata -m %t/merge-both-hashtree-funcmap.o -o %t/merge-both-hashtree-funcmap.cgdata +RUN: llvm-cgdata -s %t/merge-both-hashtree-funcmap.cgdata | FileCheck %s + +CHECK: Outlined hash tree: +CHECK-NEXT: Total Node Count: 3 +CHECK-NEXT: Terminal Node Count: 1 +CHECK-NEXT: Depth: 2 +CHECK-NEXT: Stable function map: +CHECK-NEXT: Unique hash Count: 1 +CHECK-NEXT: Total function Count: 1 +CHECK-NEXT: Mergeable function Count: 0 + +;--- raw-hashtree.cgtext +:outlined_hash_tree +0: + Hash: 0x0 + Terminals: 0 + SuccessorIds: [ 1 ] +1: + Hash: 0x1 + Terminals: 0 + SuccessorIds: [ 2 ] +2: + Hash: 0x2 + Terminals: 4 + SuccessorIds: [ ] +... + +;--- raw-funcmap.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func1 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 3 +... + +;--- merge-both-template.ll +@.data1 = private unnamed_addr constant [72 x i8] c"", section "__DATA,__llvm_outline" +@.data2 = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test new file mode 100644 index 00000000000000..f643c8d92073e3 --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test @@ -0,0 +1,83 @@ +# REQUIRES: shell, aarch64-registered-target +# UNSUPPORTED: system-windows + +# Merge an archive that has two object files having cgdata (__llvm_merge) + +RUN: split-file %s %t + +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata +RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt +RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-1-template.ll > %t/merge-1.ll +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o + +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata +RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt +RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-2-template.ll > %t/merge-2.ll +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o + +# Make an archive from two object files +RUN: llvm-ar rcs %t/merge-archive.a %t/merge-1.o %t/merge-2.o + +# Merge the archive into the codegen data file. +RUN: llvm-cgdata --merge %t/merge-archive.a -o %t/merge-archive.cgdata +RUN: llvm-cgdata --show %t/merge-archive.cgdata | FileCheck %s + +RUN: llvm-cgdata --show %t/merge-archive.cgdata| FileCheck %s +CHECK: Stable function map: +CHECK-NEXT: Unique hash Count: 1 +CHECK-NEXT: Total function Count: 2 +CHECK-NEXT: Mergeable function Count: 2 + +RUN: llvm-cgdata --convert %t/merge-archive.cgdata| FileCheck %s --check-prefix=MAP +MAP: # Stable function map +MAP-NEXT: :stable_function_map +MAP-NEXT: --- +MAP-NEXT: - Hash: 1 +MAP-NEXT: FunctionName: Func1 +MAP-NEXT: ModuleName: Mod1 +MAP-NEXT: InstCount: 2 +MAP-NEXT: IndexOperandHashes: +MAP-NEXT: - InstIndex: 0 +MAP-NEXT: OpndIndex: 1 +MAP-NEXT: OpndHash: 3 +MAP-NEXT: - Hash: 1 +MAP-NEXT: FunctionName: Func2 +MAP-NEXT: ModuleName: Mod1 +MAP-NEXT: InstCount: 2 +MAP-NEXT: IndexOperandHashes: +MAP-NEXT: - InstIndex: 0 +MAP-NEXT: OpndIndex: 1 +MAP-NEXT: OpndHash: 4 +MAP-NEXT: ... + +;--- raw-1.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func2 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 4 +... + +;--- merge-1-template.ll +@.data = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" + +;--- raw-2.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func1 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 3 +... + +;--- merge-2-template.ll +@.data = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test new file mode 100644 index 00000000000000..c8acf1f3916e5a --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test @@ -0,0 +1,78 @@ +# REQUIRES: shell, aarch64-registered-target +# UNSUPPORTED: system-windows + +# Merge a binary file (e.g., a linked executable) having concatenated cgdata (__llvm_merge) + +RUN: split-file %s %t + +# Synthesize two sets of raw cgdata without the header (32 byte) from the indexed cgdata. +# Concatenate them in merge-concat.ll +RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata +RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt +RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-concat-template.ll > %t/merge-concat-template-2.ll +RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata +RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt +RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-concat-template-2.ll > %t/merge-concat.ll + +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-concat.ll -o %t/merge-concat.o +RUN: llvm-cgdata --merge %t/merge-concat.o -o %t/merge-concat.cgdata +RUN: llvm-cgdata --show %t/merge-concat.cgdata | FileCheck %s + +CHECK: Stable function map: +CHECK-NEXT: Unique hash Count: 1 +CHECK-NEXT: Total function Count: 2 +CHECK-NEXT: Mergeable function Count: 2 + +RUN: llvm-cgdata --convert %t/merge-concat.cgdata| FileCheck %s --check-prefix=MAP +MAP: # Stable function map +MAP-NEXT: :stable_function_map +MAP-NEXT: --- +MAP-NEXT: - Hash: 1 +MAP-NEXT: FunctionName: Func1 +MAP-NEXT: ModuleName: Mod1 +MAP-NEXT: InstCount: 2 +MAP-NEXT: IndexOperandHashes: +MAP-NEXT: - InstIndex: 0 +MAP-NEXT: OpndIndex: 1 +MAP-NEXT: OpndHash: 3 +MAP-NEXT: - Hash: 1 +MAP-NEXT: FunctionName: Func2 +MAP-NEXT: ModuleName: Mod1 +MAP-NEXT: InstCount: 2 +MAP-NEXT: IndexOperandHashes: +MAP-NEXT: - InstIndex: 0 +MAP-NEXT: OpndIndex: 1 +MAP-NEXT: OpndHash: 4 +MAP-NEXT: ... + +;--- raw-1.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func2 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 4 +... + +;--- raw-2.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func1 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 3 +... + +;--- merge-concat-template.ll + +; In an linked executable (as opposed to an object file), cgdata in __llvm_merge might be concatenated. +; Although this is not a typical workflow, we simply support this case to parse cgdata that is concatenated. +; In other words, the following two trees are encoded back-to-back in a binary format. +@.data1 = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" +@.data2 = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test new file mode 100644 index 00000000000000..3ae67f062f820f --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test @@ -0,0 +1,79 @@ +# REQUIRES: shell, aarch64-registered-target +# UNSUPPORTED: system-windows + +# Merge two object files having cgdata (__llvm_merge) + +RUN: split-file %s %t + +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata +RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt +RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-1-template.ll > %t/merge-1.ll +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o + +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata +RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt +RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-2-template.ll > %t/merge-2.ll +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o + +# Merge two object files into the codegen data file. +RUN: llvm-cgdata --merge %t/merge-1.o %t/merge-2.o -o %t/merge.cgdata + +RUN: llvm-cgdata --show %t/merge.cgdata | FileCheck %s +CHECK: Stable function map: +CHECK-NEXT: Unique hash Count: 1 +CHECK-NEXT: Total function Count: 2 +CHECK-NEXT: Mergeable function Count: 2 + +RUN: llvm-cgdata --convert %t/merge.cgdata | FileCheck %s --check-prefix=MAP +MAP: # Stable function map +MAP-NEXT: :stable_function_map +MAP-NEXT: --- +MAP-NEXT: - Hash: 1 +MAP-NEXT: FunctionName: Func1 +MAP-NEXT: ModuleName: Mod1 +MAP-NEXT: InstCount: 2 +MAP-NEXT: IndexOperandHashes: +MAP-NEXT: - InstIndex: 0 +MAP-NEXT: OpndIndex: 1 +MAP-NEXT: OpndHash: 3 +MAP-NEXT: - Hash: 1 +MAP-NEXT: FunctionName: Func2 +MAP-NEXT: ModuleName: Mod1 +MAP-NEXT: InstCount: 2 +MAP-NEXT: IndexOperandHashes: +MAP-NEXT: - InstIndex: 0 +MAP-NEXT: OpndIndex: 1 +MAP-NEXT: OpndHash: 4 +MAP-NEXT: ... + +;--- raw-1.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func2 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 4 +... + +;--- merge-1-template.ll +@.data = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" + +;--- raw-2.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func1 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 3 +... + +;--- merge-2-template.ll +@.data = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test new file mode 100644 index 00000000000000..6a4e635f638657 --- /dev/null +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test @@ -0,0 +1,36 @@ +# REQUIRES: shell, aarch64-registered-target +# UNSUPPORTED: system-windows + +# Test merge a single object file into a cgdata + +RUN: split-file %s %t + +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. +RUN: llvm-cgdata --convert --format binary %t/raw-single.cgtext -o %t/raw-single.cgdata +RUN: od -t x1 -j 32 -An %t/raw-single.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-single-bytes.txt + +RUN: sed "s//$(cat %t/raw-single-bytes.txt)/g" %t/merge-single-template.ll > %t/merge-single.ll +RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-single.ll -o %t/merge-single.o + +# Merge an object file having cgdata (__llvm_merge) +RUN: llvm-cgdata -m %t/merge-single.o -o %t/merge-single.cgdata +RUN: llvm-cgdata -s %t/merge-single.cgdata | FileCheck %s +CHECK: Stable function map: +CHECK-NEXT: Unique hash Count: 1 +CHECK-NEXT: Total function Count: 1 +CHECK-NEXT: Mergeable function Count: 0 + +;--- raw-single.cgtext +:stable_function_map +- Hash: 1 + FunctionName: Func1 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 3 +... + +;--- merge-single-template.ll +@.data = private unnamed_addr constant [60 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-archive.test b/llvm/test/tools/llvm-cgdata/merge-hashtree-archive.test similarity index 91% rename from llvm/test/tools/llvm-cgdata/merge-archive.test rename to llvm/test/tools/llvm-cgdata/merge-hashtree-archive.test index 03eb9106b54562..ee6345247c5be6 100644 --- a/llvm/test/tools/llvm-cgdata/merge-archive.test +++ b/llvm/test/tools/llvm-cgdata/merge-hashtree-archive.test @@ -5,15 +5,15 @@ RUN: split-file %s %t -# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata -RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-1-template.ll > %t/merge-1.ll RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o -# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata -RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-2-template.ll > %t/merge-2.ll RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o diff --git a/llvm/test/tools/llvm-cgdata/merge-concat.test b/llvm/test/tools/llvm-cgdata/merge-hashtree-concat.test similarity index 93% rename from llvm/test/tools/llvm-cgdata/merge-concat.test rename to llvm/test/tools/llvm-cgdata/merge-hashtree-concat.test index ac0e7a6e29e878..5a3ece05a3f990 100644 --- a/llvm/test/tools/llvm-cgdata/merge-concat.test +++ b/llvm/test/tools/llvm-cgdata/merge-hashtree-concat.test @@ -5,13 +5,13 @@ RUN: split-file %s %t -# Synthesize two sets of raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize two sets of raw cgdata without the header (32 byte) from the indexed cgdata. # Concatenate them in merge-concat.ll RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata -RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-concat-template.ll > %t/merge-concat-template-2.ll RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata -RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-concat-template-2.ll > %t/merge-concat.ll RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-concat.ll -o %t/merge-concat.o diff --git a/llvm/test/tools/llvm-cgdata/merge-double.test b/llvm/test/tools/llvm-cgdata/merge-hashtree-double.test similarity index 90% rename from llvm/test/tools/llvm-cgdata/merge-double.test rename to llvm/test/tools/llvm-cgdata/merge-hashtree-double.test index 1ae8064291019e..044a8649cf4adf 100644 --- a/llvm/test/tools/llvm-cgdata/merge-double.test +++ b/llvm/test/tools/llvm-cgdata/merge-hashtree-double.test @@ -5,15 +5,15 @@ RUN: split-file %s %t -# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata -RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-1-bytes.txt RUN: sed "s//$(cat %t/raw-1-bytes.txt)/g" %t/merge-1-template.ll > %t/merge-1.ll RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o -# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata -RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-2-bytes.txt RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-2-template.ll > %t/merge-2.ll RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o diff --git a/llvm/test/tools/llvm-cgdata/merge-single.test b/llvm/test/tools/llvm-cgdata/merge-hashtree-single.test similarity index 92% rename from llvm/test/tools/llvm-cgdata/merge-single.test rename to llvm/test/tools/llvm-cgdata/merge-hashtree-single.test index 47e3cb3f4f50fb..829c63f0f17a2c 100644 --- a/llvm/test/tools/llvm-cgdata/merge-single.test +++ b/llvm/test/tools/llvm-cgdata/merge-hashtree-single.test @@ -11,9 +11,9 @@ RUN: llvm-cgdata --merge %t/merge-empty.o --output %t/merge-empty.cgdata # No summary appear with the header only cgdata. RUN: llvm-cgdata --show %t/merge-empty.cgdata | count 0 -# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata. +# Synthesize raw cgdata without the header (32 byte) from the indexed cgdata. RUN: llvm-cgdata --convert --format binary %t/raw-single.cgtext -o %t/raw-single.cgdata -RUN: od -t x1 -j 24 -An %t/raw-single.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-single-bytes.txt +RUN: od -t x1 -j 32 -An %t/raw-single.cgdata | tr -d '\n\r\t' | sed 's/[ ]*$//' | sed 's/[ ][ ]*/\\\\/g' > %t/raw-single-bytes.txt RUN: sed "s//$(cat %t/raw-single-bytes.txt)/g" %t/merge-single-template.ll > %t/merge-single.ll RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-single.ll -o %t/merge-single.o diff --git a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp index 483f4662631284..0931cad4bcb7ed 100644 --- a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp +++ b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp @@ -80,8 +80,6 @@ static CGDataAction Action; static std::optional OutputFormat; static std::vector InputFilenames; -// TODO: Add a doc, https://llvm.org/docs/CommandGuide/llvm-cgdata.html - static void exitWithError(Twine Message, std::string Whence = "", std::string Hint = "") { WithColor::error(); @@ -128,6 +126,10 @@ static int convert_main(int argc, const char *argv[]) { OutlinedHashTreeRecord Record(Reader->releaseOutlinedHashTree()); Writer.addRecord(Record); } + if (Reader->hasStableFunctionMap()) { + StableFunctionMapRecord Record(Reader->releaseStableFunctionMap()); + Writer.addRecord(Record); + } if (OutputFormat == CGDataFormat::Text) { if (Error E = Writer.writeText(OS)) @@ -141,10 +143,12 @@ static int convert_main(int argc, const char *argv[]) { } static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, - OutlinedHashTreeRecord &GlobalOutlineRecord); + OutlinedHashTreeRecord &GlobalOutlineRecord, + StableFunctionMapRecord &GlobalFunctionMapRecord); static bool handleArchive(StringRef Filename, Archive &Arch, - OutlinedHashTreeRecord &GlobalOutlineRecord) { + OutlinedHashTreeRecord &GlobalOutlineRecord, + StableFunctionMapRecord &GlobalFunctionMapRecord) { bool Result = true; Error Err = Error::success(); for (const auto &Child : Arch.children(Err)) { @@ -155,7 +159,8 @@ static bool handleArchive(StringRef Filename, Archive &Arch, if (Error E = NameOrErr.takeError()) exitWithError(std::move(E), Filename); std::string Name = (Filename + "(" + NameOrErr.get() + ")").str(); - Result &= handleBuffer(Name, BuffOrErr.get(), GlobalOutlineRecord); + Result &= handleBuffer(Name, BuffOrErr.get(), GlobalOutlineRecord, + GlobalFunctionMapRecord); } if (Err) exitWithError(std::move(Err), Filename); @@ -163,7 +168,8 @@ static bool handleArchive(StringRef Filename, Archive &Arch, } static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, - OutlinedHashTreeRecord &GlobalOutlineRecord) { + OutlinedHashTreeRecord &GlobalOutlineRecord, + StableFunctionMapRecord &GlobalFunctionMapRecord) { Expected> BinOrErr = object::createBinary(Buffer); if (Error E = BinOrErr.takeError()) @@ -171,11 +177,12 @@ static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, bool Result = true; if (auto *Obj = dyn_cast(BinOrErr->get())) { - if (Error E = - CodeGenDataReader::mergeFromObjectFile(Obj, GlobalOutlineRecord)) + if (Error E = CodeGenDataReader::mergeFromObjectFile( + Obj, GlobalOutlineRecord, GlobalFunctionMapRecord)) exitWithError(std::move(E), Filename); } else if (auto *Arch = dyn_cast(BinOrErr->get())) { - Result &= handleArchive(Filename, *Arch, GlobalOutlineRecord); + Result &= handleArchive(Filename, *Arch, GlobalOutlineRecord, + GlobalFunctionMapRecord); } else { // TODO: Support for the MachO universal binary format. errs() << "Error: unsupported binary file: " << Filename << "\n"; @@ -186,26 +193,34 @@ static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer, } static bool handleFile(StringRef Filename, - OutlinedHashTreeRecord &GlobalOutlineRecord) { + OutlinedHashTreeRecord &GlobalOutlineRecord, + StableFunctionMapRecord &GlobalFunctionMapRecord) { ErrorOr> BuffOrErr = MemoryBuffer::getFileOrSTDIN(Filename); if (std::error_code EC = BuffOrErr.getError()) exitWithErrorCode(EC, Filename); - return handleBuffer(Filename, *BuffOrErr.get(), GlobalOutlineRecord); + return handleBuffer(Filename, *BuffOrErr.get(), GlobalOutlineRecord, + GlobalFunctionMapRecord); } static int merge_main(int argc, const char *argv[]) { bool Result = true; OutlinedHashTreeRecord GlobalOutlineRecord; + StableFunctionMapRecord GlobalFunctionMapRecord; for (auto &Filename : InputFilenames) - Result &= handleFile(Filename, GlobalOutlineRecord); + Result &= + handleFile(Filename, GlobalOutlineRecord, GlobalFunctionMapRecord); if (!Result) exitWithError("failed to merge codegen data files."); + GlobalFunctionMapRecord.finalize(); + CodeGenDataWriter Writer; if (!GlobalOutlineRecord.empty()) Writer.addRecord(GlobalOutlineRecord); + if (!GlobalFunctionMapRecord.empty()) + Writer.addRecord(GlobalFunctionMapRecord); std::error_code EC; raw_fd_ostream OS(OutputFilename, EC, @@ -249,6 +264,15 @@ static int show_main(int argc, const char *argv[]) { << "\n"; OS << " Depth: " << Tree->depth() << "\n"; } + if (Reader->hasStableFunctionMap()) { + auto Map = Reader->releaseStableFunctionMap(); + OS << "Stable function map:\n"; + OS << " Unique hash Count: " << Map->size() << "\n"; + OS << " Total function Count: " + << Map->size(StableFunctionMap::TotalFunctionCount) << "\n"; + OS << " Mergeable function Count: " + << Map->size(StableFunctionMap::MergeableFunctionCount) << "\n"; + } return 0; } diff --git a/llvm/unittests/CGData/CMakeLists.txt b/llvm/unittests/CGData/CMakeLists.txt index 792b323130b474..0bdb9e1f08c702 100644 --- a/llvm/unittests/CGData/CMakeLists.txt +++ b/llvm/unittests/CGData/CMakeLists.txt @@ -9,6 +9,8 @@ set(LLVM_LINK_COMPONENTS add_llvm_unittest(CGDataTests OutlinedHashTreeRecordTest.cpp OutlinedHashTreeTest.cpp + StableFunctionMapRecordTest.cpp + StableFunctionMapTest.cpp ) target_link_libraries(CGDataTests PRIVATE LLVMTestingSupport) diff --git a/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp b/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp new file mode 100644 index 00000000000000..77d7e85bb7973b --- /dev/null +++ b/llvm/unittests/CGData/StableFunctionMapRecordTest.cpp @@ -0,0 +1,131 @@ +//===- StableFunctionMapRecordTest.cpp ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CGData/StableFunctionMapRecord.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +TEST(StableFunctionMapRecordTest, Print) { + StableFunctionMapRecord MapRecord; + StableFunction Func1{1, "Func1", "Mod1", 2, {{{0, 1}, 3}}}; + MapRecord.FunctionMap->insert(Func1); + + const char *ExpectedMapStr = R"(--- +- Hash: 1 + FunctionName: Func1 + ModuleName: Mod1 + InstCount: 2 + IndexOperandHashes: + - InstIndex: 0 + OpndIndex: 1 + OpndHash: 3 +... +)"; + std::string MapDump; + raw_string_ostream OS(MapDump); + MapRecord.print(OS); + EXPECT_EQ(ExpectedMapStr, MapDump); +} + +TEST(OutlinedHashTreeRecordTest, Stable) { + StableFunction Func1{1, "Func2", "Mod1", 1, {}}; + StableFunction Func2{1, "Func3", "Mod1", 1, {}}; + StableFunction Func3{1, "Func1", "Mod2", 1, {}}; + StableFunction Func4{2, "Func4", "Mod3", 1, {}}; + + StableFunctionMapRecord MapRecord1; + MapRecord1.FunctionMap->insert(Func1); + MapRecord1.FunctionMap->insert(Func2); + MapRecord1.FunctionMap->insert(Func3); + MapRecord1.FunctionMap->insert(Func4); + + StableFunctionMapRecord MapRecord2; + MapRecord2.FunctionMap->insert(Func4); + MapRecord2.FunctionMap->insert(Func3); + MapRecord2.FunctionMap->insert(Func2); + MapRecord2.FunctionMap->insert(Func1); + + // Output is sorted by hash (1 < 2), module name (Mod1 < Mod2), and function + // name (Func2 < Func3). + std::string MapDump1; + raw_string_ostream OS1(MapDump1); + MapRecord1.print(OS1); + std::string MapDump2; + raw_string_ostream OS2(MapDump2); + MapRecord2.print(OS2); + EXPECT_EQ(MapDump1, MapDump2); +} + +TEST(StableFunctionMapRecordTest, Serialize) { + StableFunctionMapRecord MapRecord1; + StableFunction Func1{1, "Func1", "Mod1", 2, {{{0, 1}, 3}, {{1, 2}, 4}}}; + StableFunction Func2{2, "Func2", "Mod1", 3, {{{0, 1}, 2}}}; + StableFunction Func3{2, "Func3", "Mod1", 3, {{{0, 1}, 3}}}; + MapRecord1.FunctionMap->insert(Func1); + MapRecord1.FunctionMap->insert(Func2); + MapRecord1.FunctionMap->insert(Func3); + + // Serialize and deserialize the tree. + SmallVector Out; + raw_svector_ostream OS(Out); + MapRecord1.serialize(OS); + + StableFunctionMapRecord MapRecord2; + const uint8_t *Data = reinterpret_cast(Out.data()); + MapRecord2.deserialize(Data); + + // Two maps should be identical. + std::string MapDump1; + raw_string_ostream OS1(MapDump1); + MapRecord1.print(OS1); + MapRecord1.print(); + std::string MapDump2; + raw_string_ostream OS2(MapDump2); + MapRecord2.print(OS2); + MapRecord2.print(); + + EXPECT_EQ(MapDump1, MapDump2); +} + +TEST(StableFunctionMapRecordTest, SerializeYAML) { + StableFunctionMapRecord MapRecord1; + StableFunction Func1{1, "Func1", "Mod1", 2, {{{0, 1}, 3}, {{1, 2}, 4}}}; + StableFunction Func2{2, "Func2", "Mod1", 3, {{{0, 1}, 2}}}; + StableFunction Func3{2, "Func3", "Mod1", 3, {{{0, 1}, 3}}}; + MapRecord1.FunctionMap->insert(Func1); + MapRecord1.FunctionMap->insert(Func2); + MapRecord1.FunctionMap->insert(Func3); + + // Serialize and deserialize the map in a YAML format. + std::string Out; + raw_string_ostream OS(Out); + yaml::Output YOS(OS); + MapRecord1.serializeYAML(YOS); + + StableFunctionMapRecord MapRecord2; + yaml::Input YIS(StringRef(Out.data(), Out.size())); + MapRecord2.deserializeYAML(YIS); + + // Two maps should be identical. + std::string MapDump1; + raw_string_ostream OS1(MapDump1); + MapRecord1.print(OS1); + MapRecord1.print(); + std::string MapDump2; + raw_string_ostream OS2(MapDump2); + MapRecord2.print(OS2); + MapRecord2.print(); + + EXPECT_EQ(MapDump1, MapDump2); +} + +} // end namespace diff --git a/llvm/unittests/CGData/StableFunctionMapTest.cpp b/llvm/unittests/CGData/StableFunctionMapTest.cpp new file mode 100644 index 00000000000000..e9bdbcd91b0f2a --- /dev/null +++ b/llvm/unittests/CGData/StableFunctionMapTest.cpp @@ -0,0 +1,104 @@ +//===- StableFunctionMapTest.cpp ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CGData/StableFunctionMap.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +TEST(StableFunctionMap, Name) { + StableFunctionMap Map; + EXPECT_TRUE(Map.empty()); + EXPECT_TRUE(Map.getNames().empty()); + unsigned ID1 = Map.getIdOrCreateForName("Func1"); + unsigned ID2 = Map.getIdOrCreateForName("Func2"); + unsigned ID3 = Map.getIdOrCreateForName("Func1"); + + EXPECT_EQ(Map.getNames().size(), 2u); + // The different names should return different IDs. + EXPECT_NE(ID1, ID2); + // The same name should return the same ID. + EXPECT_EQ(ID1, ID3); + // The IDs should be valid. + EXPECT_EQ(*Map.getNameForId(ID1), "Func1"); + EXPECT_EQ(*Map.getNameForId(ID2), "Func2"); +} + +TEST(StableFunctionMap, Insert) { + StableFunctionMap Map; + + StableFunction Func1{1, "Func1", "Mod1", 2, {{{0, 1}, 3}}}; + StableFunction Func2{1, "Func2", "Mod1", 2, {{{0, 1}, 2}}}; + Map.insert(Func1); + Map.insert(Func2); + // We only have a unique hash, 1 + EXPECT_EQ(Map.size(), 1u); + // We have two functions with the same hash which are potentially mergeable. + EXPECT_EQ(Map.size(StableFunctionMap::SizeType::TotalFunctionCount), 2u); + EXPECT_EQ(Map.size(StableFunctionMap::SizeType::MergeableFunctionCount), 2u); +} + +TEST(StableFunctionMap, InsertEntry) { + StableFunctionMap Map; + + unsigned ID1 = Map.getIdOrCreateForName("Func1"); + unsigned ID2 = Map.getIdOrCreateForName("Mod1"); + unsigned ID3 = Map.getIdOrCreateForName("Func2"); + + // Create a function entry and insert it into the map. + auto IndexOperandHashMap1 = std::make_unique(); + IndexOperandHashMap1->insert({{0, 1}, 3}); + auto FuncEntry1 = std::make_unique( + 1, ID1, ID2, 2, std::move(IndexOperandHashMap1)); + Map.insert(std::move(FuncEntry1)); + + // Create another function entry and insert it into the map. + auto IndexOperandHashMap2 = std::make_unique(); + IndexOperandHashMap2->insert({{0, 1}, 2}); + auto FuncEntry2 = std::make_unique( + 1, ID3, ID2, 2, std::move(IndexOperandHashMap2)); + Map.insert(std::move(FuncEntry2)); + + // We only have a unique hash, 1 + EXPECT_EQ(Map.size(), 1u); + // We have two functions with the same hash which are potentially mergeable. + EXPECT_EQ(Map.size(StableFunctionMap::SizeType::TotalFunctionCount), 2u); +} + +TEST(OutlinedHashTreeTest, Merge) { + StableFunctionMap Map1; + StableFunction Func1{1, "Func1", "Mod1", 2, {{{0, 1}, 3}}}; + StableFunction Func2{1, "Func2", "Mod1", 2, {{{0, 1}, 2}}}; + StableFunction Func3{2, "Func3", "Mod1", 2, {{{1, 1}, 2}}}; + Map1.insert(Func1); + Map1.insert(Func2); + Map1.insert(Func3); + + StableFunctionMap Map2; + StableFunction Func4{1, "Func4", "Mod2", 2, {{{0, 1}, 4}}}; + StableFunction Func5{2, "Func5", "Mod2", 2, {{{1, 1}, 5}}}; + StableFunction Func6{3, "Func6", "Mod2", 2, {{{1, 1}, 6}}}; + Map2.insert(Func4); + Map2.insert(Func5); + Map2.insert(Func6); + + // Merge two maps. + Map1.merge(Map2); + + // We only have two unique hashes, 1, 2 and 3 + EXPECT_EQ(Map1.size(), 3u); + // We have total 6 functions. + EXPECT_EQ(Map1.size(StableFunctionMap::SizeType::TotalFunctionCount), 6u); + // We have 5 mergeable functions. Func6 only has a unique hash, 3. + EXPECT_EQ(Map1.size(StableFunctionMap::SizeType::MergeableFunctionCount), 5u); +} + +} // end namespace diff --git a/llvm/unittests/IR/StructuralHashTest.cpp b/llvm/unittests/IR/StructuralHashTest.cpp index 64e66aa5f97a6d..e9f18ed40bc65b 100644 --- a/llvm/unittests/IR/StructuralHashTest.cpp +++ b/llvm/unittests/IR/StructuralHashTest.cpp @@ -239,4 +239,59 @@ TEST(StructuralHashTest, ArgumentNumber) { EXPECT_EQ(StructuralHash(*M1), StructuralHash(*M2)); EXPECT_NE(StructuralHash(*M1, true), StructuralHash(*M2, true)); } + +TEST(StructuralHashTest, Differences) { + LLVMContext Ctx; + std::unique_ptr M1 = parseIR(Ctx, "define i64 @f(i64 %a) {\n" + " %c = add i64 %a, 1\n" + " %b = call i64 @f1(i64 %c)\n" + " ret i64 %b\n" + "}\n" + "declare i64 @f1(i64)"); + auto *F1 = M1->getFunction("f"); + std::unique_ptr M2 = parseIR(Ctx, "define i64 @g(i64 %a) {\n" + " %c = add i64 %a, 1\n" + " %b = call i64 @f2(i64 %c)\n" + " ret i64 %b\n" + "}\n" + "declare i64 @f2(i64)"); + auto *F2 = M2->getFunction("g"); + + // They are originally different when not ignoring any operand. + EXPECT_NE(StructuralHash(*F1, true), StructuralHash(*F2, true)); + EXPECT_NE(StructuralHashWithDifferences(*F1, nullptr).FunctionHash, + StructuralHashWithDifferences(*F2, nullptr).FunctionHash); + + // When we ignore the call target f1 vs f2, they have the same hash. + auto IgnoreOp = [&](const Instruction *I, unsigned OpndIdx) { + return I->getOpcode() == Instruction::Call && OpndIdx == 1; + }; + auto FuncHashInfo1 = StructuralHashWithDifferences(*F1, IgnoreOp); + auto FuncHashInfo2 = StructuralHashWithDifferences(*F2, IgnoreOp); + EXPECT_EQ(FuncHashInfo1.FunctionHash, FuncHashInfo2.FunctionHash); + + // There are total 3 instructions. + EXPECT_EQ(FuncHashInfo1.IndexInstruction->size(), 3u); + EXPECT_EQ(FuncHashInfo2.IndexInstruction->size(), 3u); + + // The only 1 operand (the call target) has been ignored. + EXPECT_EQ(FuncHashInfo1.IndexOperandHashMap->size(), 1u); + EXPECT_EQ(FuncHashInfo2.IndexOperandHashMap->size(), 1u); + + // The index pair of instruction and operand (1, 1) is a key in the map. + ASSERT_TRUE(FuncHashInfo1.IndexOperandHashMap->count({1, 1})); + ASSERT_TRUE(FuncHashInfo2.IndexOperandHashMap->count({1, 1})); + + // The indexed instruciton must be the call instruction as shown in the + // IgnoreOp above. + EXPECT_EQ(FuncHashInfo1.IndexInstruction->lookup(1)->getOpcode(), + Instruction::Call); + EXPECT_EQ(FuncHashInfo2.IndexInstruction->lookup(1)->getOpcode(), + Instruction::Call); + + // The ignored operand hashes (for f1 vs. f2) are different. + EXPECT_NE(FuncHashInfo1.IndexOperandHashMap->lookup({1, 1}), + FuncHashInfo2.IndexOperandHashMap->lookup({1, 1})); +} + } // end anonymous namespace