From 3a0d894fafddace75f03fa7df25022cadbe2dffc Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Mon, 29 Apr 2024 14:44:04 -0700 Subject: [PATCH] [BOLT] Add support for BOLT-reserved space in a binary (#90300) Allow the user to allocate space in a binary that could be used by BOLT for allocating new sections. The reservation is specified by two special symbols recognizable by BOLT: __bolt_reserved_{start,end}. The reserved space will be useful for optimizing the Linux kernel where we cannot allocate a new executable segment. However, the support is not limited to kernel binaries as some user-space application may find it useful too. --- bolt/include/bolt/Rewrite/RewriteInstance.h | 4 + bolt/lib/Rewrite/RewriteInstance.cpp | 88 +++++++++++++++------ 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h index d37e6f5ed8595b..41a92e7ba01e84 100644 --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -422,6 +422,10 @@ class RewriteInstance { /// Section name used for extra BOLT code in addition to .text. static StringRef getBOLTTextSectionName() { return ".bolt.text"; } + /// Symbol markers for BOLT reserved area. + static StringRef getBOLTReservedStart() { return "__bolt_reserved_start"; } + static StringRef getBOLTReservedEnd() { return "__bolt_reserved_end"; } + /// Common section names. static StringRef getEHFrameSectionName() { return ".eh_frame"; } static StringRef getEHFrameHdrSectionName() { return ".eh_frame_hdr"; } diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 329649c1ca67c5..1c5ce5d0a6cd0b 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -1063,6 +1063,11 @@ void RewriteInstance::discoverFileObjects() { continue; } + if (SymName == getBOLTReservedStart() || SymName == getBOLTReservedEnd()) { + registerName(SymbolSize); + continue; + } + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName << " for function\n"); @@ -3594,6 +3599,26 @@ void RewriteInstance::updateMetadata() { void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) { BC->deregisterUnusedSections(); + // Check if the input has a space reserved for BOLT. + BinaryData *StartBD = BC->getBinaryDataByName(getBOLTReservedStart()); + BinaryData *EndBD = BC->getBinaryDataByName(getBOLTReservedEnd()); + if (!StartBD != !EndBD) { + BC->errs() << "BOLT-ERROR: one of the symbols is missing from the binary: " + << getBOLTReservedStart() << ", " << getBOLTReservedEnd() + << '\n'; + exit(1); + } + + if (StartBD) { + PHDRTableOffset = 0; + PHDRTableAddress = 0; + NewTextSegmentAddress = 0; + NewTextSegmentOffset = 0; + NextAvailableAddress = StartBD->getAddress(); + BC->outs() + << "BOLT-INFO: using reserved space for allocating new sections\n"; + } + // If no new .eh_frame was written, remove relocated original .eh_frame. BinarySection *RelocatedEHFrameSection = getSection(".relocated" + getEHFrameSectionName()); @@ -3613,6 +3638,18 @@ void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) { // Map the rest of the sections. mapAllocatableSections(MapSection); + + if (StartBD) { + const uint64_t ReservedSpace = EndBD->getAddress() - StartBD->getAddress(); + const uint64_t AllocatedSize = NextAvailableAddress - StartBD->getAddress(); + if (ReservedSpace < AllocatedSize) { + BC->errs() << "BOLT-ERROR: reserved space (" << ReservedSpace << " byte" + << (ReservedSpace == 1 ? "" : "s") + << ") is smaller than required for new allocations (" + << AllocatedSize << " bytes)\n"; + exit(1); + } + } } std::vector RewriteInstance::getCodeSections() { @@ -3854,7 +3891,7 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) { // Add the new text section aggregating all existing code sections. // This is pseudo-section that serves a purpose of creating a corresponding // entry in section header table. - int64_t NewTextSectionSize = + const uint64_t NewTextSectionSize = NextAvailableAddress - NewTextSectionStartAddress; if (NewTextSectionSize) { const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true, @@ -3937,7 +3974,7 @@ void RewriteInstance::mapAllocatableSections( if (PHDRTableAddress) { // Segment size includes the size of the PHDR area. NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; - } else { + } else if (NewTextSegmentAddress) { // Existing PHDR table would be updated. NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; } @@ -3976,7 +4013,7 @@ void RewriteInstance::patchELFPHDRTable() { assert(!PHDRTableAddress && "unexpected address for program header table"); PHDRTableOffset = Obj.getHeader().e_phoff; if (NewWritableSegmentSize) { - BC->errs() << "Unable to add writable segment with UseGnuStack option\n"; + BC->errs() << "BOLT-ERROR: unable to add writable segment\n"; exit(1); } } @@ -3986,7 +4023,7 @@ void RewriteInstance::patchELFPHDRTable() { if (!NewWritableSegmentSize) { if (PHDRTableAddress) NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; - else + else if (NewTextSegmentAddress) NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; } else { NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress; @@ -4020,8 +4057,10 @@ void RewriteInstance::patchELFPHDRTable() { }; auto writeNewSegmentPhdrs = [&]() { - ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); - OS.write(reinterpret_cast(&NewTextPhdr), sizeof(NewTextPhdr)); + if (PHDRTableAddress || NewTextSegmentSize) { + ELF64LE::Phdr NewPhdr = createNewTextPhdr(); + OS.write(reinterpret_cast(&NewPhdr), sizeof(NewPhdr)); + } if (NewWritableSegmentSize) { ELF64LEPhdrTy NewPhdr; @@ -4119,9 +4158,8 @@ void RewriteInstance::rewriteNoteSections() { const ELFFile &Obj = ELF64LEFile->getELFFile(); raw_fd_ostream &OS = Out->os(); - uint64_t NextAvailableOffset = getFileOffsetForAddress(NextAvailableAddress); - assert(NextAvailableOffset >= FirstNonAllocatableOffset && - "next available offset calculation failure"); + uint64_t NextAvailableOffset = std::max( + getFileOffsetForAddress(NextAvailableAddress), FirstNonAllocatableOffset); OS.seek(NextAvailableOffset); // Copy over non-allocatable section contents and update file offsets. @@ -4860,7 +4898,7 @@ void RewriteInstance::updateELFSymbolTable( ++NumHotDataSymsUpdated; } - if (*SymbolName == "_end") + if (*SymbolName == "_end" && NextAvailableAddress > Symbol.st_value) updateSymbolValue(*SymbolName, NextAvailableAddress); if (IsDynSym) @@ -4974,13 +5012,6 @@ void RewriteInstance::patchELFSymTabs(ELFObjectFile *File) { std::vector NewSectionIndex; getOutputSections(File, NewSectionIndex); - // Set pointer at the end of the output file, so we can pwrite old symbol - // tables if we need to. - uint64_t NextAvailableOffset = getFileOffsetForAddress(NextAvailableAddress); - assert(NextAvailableOffset >= FirstNonAllocatableOffset && - "next available offset calculation failure"); - Out->os().seek(NextAvailableOffset); - // Update dynamic symbol table. const ELFShdrTy *DynSymSection = nullptr; for (const ELFShdrTy &Section : cantFail(Obj.sections())) { @@ -4992,6 +5023,10 @@ void RewriteInstance::patchELFSymTabs(ELFObjectFile *File) { assert((DynSymSection || BC->IsStaticExecutable) && "dynamic symbol table expected"); if (DynSymSection) { + // Set pointer to the end of the section, so we can use pwrite to update + // the dynamic symbol table. + Out->os().seek(DynSymSection->sh_offset + DynSymSection->sh_size); + updateELFSymbolTable( File, /*IsDynSym=*/true, @@ -5545,10 +5580,10 @@ void RewriteInstance::rewriteFile() { auto Streamer = BC->createStreamer(OS); // Make sure output stream has enough reserved space, otherwise // pwrite() will fail. - uint64_t Offset = OS.seek(getFileOffsetForAddress(NextAvailableAddress)); - (void)Offset; - assert(Offset == getFileOffsetForAddress(NextAvailableAddress) && - "error resizing output file"); + uint64_t Offset = std::max(getFileOffsetForAddress(NextAvailableAddress), + FirstNonAllocatableOffset); + Offset = OS.seek(Offset); + assert((Offset != (uint64_t)-1) && "Error resizing output file"); // Overwrite functions with fixed output address. This is mostly used by // non-relocation mode, with one exception: injected functions are covered @@ -5780,7 +5815,7 @@ void RewriteInstance::writeEHFrameHeader() { std::vector NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader( RelocatedEHFrame, NewEHFrame, EHFrameHdrOutputAddress, FailedAddresses); - assert(Out->os().tell() == EHFrameHdrFileOffset && "offset mismatch"); + Out->os().seek(EHFrameHdrFileOffset); Out->os().write(NewEHFrameHdr.data(), NewEHFrameHdr.size()); const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true, @@ -5800,6 +5835,15 @@ void RewriteInstance::writeEHFrameHeader() { NextAvailableAddress += EHFrameHdrSec.getOutputSize(); + if (const BinaryData *ReservedEnd = + BC->getBinaryDataByName(getBOLTReservedEnd())) { + if (NextAvailableAddress > ReservedEnd->getAddress()) { + BC->errs() << "BOLT-ERROR: unable to fit " << getEHFrameHdrSectionName() + << " into reserved space\n"; + exit(1); + } + } + // Merge new .eh_frame with the relocated original so that gdb can locate all // FDEs. if (RelocatedEHFrameSection) {