Skip to content

Commit

Permalink
[BOLT] Add support for BOLT-reserved space in a binary (llvm#90300)
Browse files Browse the repository at this point in the history 
Allow the user to allocate space in a binary that could be used by BOLT
for allocating new sections. The reservation is specified by two special
symbols recognizable by BOLT: __bolt_reserved_{start,end}.

The reserved space will be useful for optimizing the Linux kernel where
we cannot allocate a new executable segment. However, the support is not
limited to kernel binaries as some user-space application may find it
useful too.
  • Loading branch information
@maksfb
maksfb committed Apr 29, 2024
1 parent 5bbf1ea commit 3a0d894
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 22 deletions.
4 changes: 4 additions & 0 deletions bolt/include/bolt/Rewrite/RewriteInstance.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,10 @@ class RewriteInstance {
/// Section name used for extra BOLT code in addition to .text.
static StringRef getBOLTTextSectionName() { return ".bolt.text"; }

/// Symbol markers for BOLT reserved area.
static StringRef getBOLTReservedStart() { return "__bolt_reserved_start"; }
static StringRef getBOLTReservedEnd() { return "__bolt_reserved_end"; }

/// Common section names.
static StringRef getEHFrameSectionName() { return ".eh_frame"; }
static StringRef getEHFrameHdrSectionName() { return ".eh_frame_hdr"; }
Expand Down
88 changes: 66 additions & 22 deletions bolt/lib/Rewrite/RewriteInstance.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1063,6 +1063,11 @@ void RewriteInstance::discoverFileObjects() {
continue;
}

if (SymName == getBOLTReservedStart() || SymName == getBOLTReservedEnd()) {
registerName(SymbolSize);
continue;
}

LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName
<< " for function\n");

Expand Down Expand Up @@ -3594,6 +3599,26 @@ void RewriteInstance::updateMetadata() {
void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {
BC->deregisterUnusedSections();

// Check if the input has a space reserved for BOLT.
BinaryData *StartBD = BC->getBinaryDataByName(getBOLTReservedStart());
BinaryData *EndBD = BC->getBinaryDataByName(getBOLTReservedEnd());
if (!StartBD != !EndBD) {
BC->errs() << "BOLT-ERROR: one of the symbols is missing from the binary: "
<< getBOLTReservedStart() << ", " << getBOLTReservedEnd()
<< '\n';
exit(1);
}

if (StartBD) {
PHDRTableOffset = 0;
PHDRTableAddress = 0;
NewTextSegmentAddress = 0;
NewTextSegmentOffset = 0;
NextAvailableAddress = StartBD->getAddress();
BC->outs()
<< "BOLT-INFO: using reserved space for allocating new sections\n";
}

// If no new .eh_frame was written, remove relocated original .eh_frame.
BinarySection *RelocatedEHFrameSection =
getSection(".relocated" + getEHFrameSectionName());
Expand All @@ -3613,6 +3638,18 @@ void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {

// Map the rest of the sections.
mapAllocatableSections(MapSection);

if (StartBD) {
const uint64_t ReservedSpace = EndBD->getAddress() - StartBD->getAddress();
const uint64_t AllocatedSize = NextAvailableAddress - StartBD->getAddress();
if (ReservedSpace < AllocatedSize) {
BC->errs() << "BOLT-ERROR: reserved space (" << ReservedSpace << " byte"
<< (ReservedSpace == 1 ? "" : "s")
<< ") is smaller than required for new allocations ("
<< AllocatedSize << " bytes)\n";
exit(1);
}
}
}

std::vector<BinarySection *> RewriteInstance::getCodeSections() {
Expand Down Expand Up @@ -3854,7 +3891,7 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
// Add the new text section aggregating all existing code sections.
// This is pseudo-section that serves a purpose of creating a corresponding
// entry in section header table.
int64_t NewTextSectionSize =
const uint64_t NewTextSectionSize =
NextAvailableAddress - NewTextSectionStartAddress;
if (NewTextSectionSize) {
const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
Expand Down Expand Up @@ -3937,7 +3974,7 @@ void RewriteInstance::mapAllocatableSections(
if (PHDRTableAddress) {
// Segment size includes the size of the PHDR area.
NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
} else {
} else if (NewTextSegmentAddress) {
// Existing PHDR table would be updated.
NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
}
Expand Down Expand Up @@ -3976,7 +4013,7 @@ void RewriteInstance::patchELFPHDRTable() {
assert(!PHDRTableAddress && "unexpected address for program header table");
PHDRTableOffset = Obj.getHeader().e_phoff;
if (NewWritableSegmentSize) {
BC->errs() << "Unable to add writable segment with UseGnuStack option\n";
BC->errs() << "BOLT-ERROR: unable to add writable segment\n";
exit(1);
}
}
Expand All @@ -3986,7 +4023,7 @@ void RewriteInstance::patchELFPHDRTable() {
if (!NewWritableSegmentSize) {
if (PHDRTableAddress)
NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
else
else if (NewTextSegmentAddress)
NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
} else {
NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
Expand Down Expand Up @@ -4020,8 +4057,10 @@ void RewriteInstance::patchELFPHDRTable() {
};

auto writeNewSegmentPhdrs = [&]() {
ELF64LE::Phdr NewTextPhdr = createNewTextPhdr();
OS.write(reinterpret_cast<const char *>(&NewTextPhdr), sizeof(NewTextPhdr));
if (PHDRTableAddress || NewTextSegmentSize) {
ELF64LE::Phdr NewPhdr = createNewTextPhdr();
OS.write(reinterpret_cast<const char *>(&NewPhdr), sizeof(NewPhdr));
}

if (NewWritableSegmentSize) {
ELF64LEPhdrTy NewPhdr;
Expand Down Expand Up @@ -4119,9 +4158,8 @@ void RewriteInstance::rewriteNoteSections() {
const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
raw_fd_ostream &OS = Out->os();

uint64_t NextAvailableOffset = getFileOffsetForAddress(NextAvailableAddress);
assert(NextAvailableOffset >= FirstNonAllocatableOffset &&
"next available offset calculation failure");
uint64_t NextAvailableOffset = std::max(
getFileOffsetForAddress(NextAvailableAddress), FirstNonAllocatableOffset);
OS.seek(NextAvailableOffset);

// Copy over non-allocatable section contents and update file offsets.
Expand Down Expand Up @@ -4860,7 +4898,7 @@ void RewriteInstance::updateELFSymbolTable(
++NumHotDataSymsUpdated;
}

if (*SymbolName == "_end")
if (*SymbolName == "_end" && NextAvailableAddress > Symbol.st_value)
updateSymbolValue(*SymbolName, NextAvailableAddress);

if (IsDynSym)
Expand Down Expand Up @@ -4974,13 +5012,6 @@ void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {
std::vector<uint32_t> NewSectionIndex;
getOutputSections(File, NewSectionIndex);

// Set pointer at the end of the output file, so we can pwrite old symbol
// tables if we need to.
uint64_t NextAvailableOffset = getFileOffsetForAddress(NextAvailableAddress);
assert(NextAvailableOffset >= FirstNonAllocatableOffset &&
"next available offset calculation failure");
Out->os().seek(NextAvailableOffset);

// Update dynamic symbol table.
const ELFShdrTy *DynSymSection = nullptr;
for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
Expand All @@ -4992,6 +5023,10 @@ void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {
assert((DynSymSection || BC->IsStaticExecutable) &&
"dynamic symbol table expected");
if (DynSymSection) {
// Set pointer to the end of the section, so we can use pwrite to update
// the dynamic symbol table.
Out->os().seek(DynSymSection->sh_offset + DynSymSection->sh_size);

updateELFSymbolTable(
File,
/*IsDynSym=*/true,
Expand Down Expand Up @@ -5545,10 +5580,10 @@ void RewriteInstance::rewriteFile() {
auto Streamer = BC->createStreamer(OS);
// Make sure output stream has enough reserved space, otherwise
// pwrite() will fail.
uint64_t Offset = OS.seek(getFileOffsetForAddress(NextAvailableAddress));
(void)Offset;
assert(Offset == getFileOffsetForAddress(NextAvailableAddress) &&
"error resizing output file");
uint64_t Offset = std::max(getFileOffsetForAddress(NextAvailableAddress),
FirstNonAllocatableOffset);
Offset = OS.seek(Offset);
assert((Offset != (uint64_t)-1) && "Error resizing output file");

// Overwrite functions with fixed output address. This is mostly used by
// non-relocation mode, with one exception: injected functions are covered
Expand Down Expand Up @@ -5780,7 +5815,7 @@ void RewriteInstance::writeEHFrameHeader() {
std::vector<char> NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(
RelocatedEHFrame, NewEHFrame, EHFrameHdrOutputAddress, FailedAddresses);

assert(Out->os().tell() == EHFrameHdrFileOffset && "offset mismatch");
Out->os().seek(EHFrameHdrFileOffset);
Out->os().write(NewEHFrameHdr.data(), NewEHFrameHdr.size());

const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
Expand All @@ -5800,6 +5835,15 @@ void RewriteInstance::writeEHFrameHeader() {

NextAvailableAddress += EHFrameHdrSec.getOutputSize();

if (const BinaryData *ReservedEnd =
BC->getBinaryDataByName(getBOLTReservedEnd())) {
if (NextAvailableAddress > ReservedEnd->getAddress()) {
BC->errs() << "BOLT-ERROR: unable to fit " << getEHFrameHdrSectionName()
<< " into reserved space\n";
exit(1);
}
}

// Merge new .eh_frame with the relocated original so that gdb can locate all
// FDEs.
if (RelocatedEHFrameSection) {
Expand Down

0 comments on commit 3a0d894

Please sign in to comment.