From 4ff164610836666804416600b708fb0fc9713911 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Fri, 10 Nov 2023 10:55:05 +0000 Subject: [PATCH 1/8] Copy SectionMemoryManager from LLVM 14.0.6 Copied verbatim from llvm/llvm-project@f28c006a5895, files: ``` llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h llvm/lib/ExecutionEngine/SectionMemoryManager.cpp ``` --- ffi/memorymanager.cpp | 273 ++++++++++++++++++++++++++++++++++++++++++ ffi/memorymanager.h | 193 +++++++++++++++++++++++++++++ 2 files changed, 466 insertions(+) create mode 100644 ffi/memorymanager.cpp create mode 100644 ffi/memorymanager.h diff --git a/ffi/memorymanager.cpp b/ffi/memorymanager.cpp new file mode 100644 index 000000000..56b232b9d --- /dev/null +++ b/ffi/memorymanager.cpp @@ -0,0 +1,273 @@ +//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the section-based memory manager used by the MCJIT +// execution engine and RuntimeDyld +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/Config/config.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Process.h" + +namespace llvm { + +uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size, + unsigned Alignment, + unsigned SectionID, + StringRef SectionName, + bool IsReadOnly) { + if (IsReadOnly) + return allocateSection(SectionMemoryManager::AllocationPurpose::ROData, + Size, Alignment); + return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size, + Alignment); +} + +uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size, + unsigned Alignment, + unsigned SectionID, + StringRef SectionName) { + return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size, + Alignment); +} + +uint8_t *SectionMemoryManager::allocateSection( + SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size, + unsigned Alignment) { + if (!Alignment) + Alignment = 16; + + assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two."); + + uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1); + uintptr_t Addr = 0; + + MemoryGroup &MemGroup = [&]() -> MemoryGroup & { + switch (Purpose) { + case AllocationPurpose::Code: + return CodeMem; + case AllocationPurpose::ROData: + return RODataMem; + case AllocationPurpose::RWData: + return RWDataMem; + } + llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose"); + }(); + + // Look in the list of free memory regions and use a block there if one + // is available. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.allocatedSize() >= RequiredSize) { + Addr = (uintptr_t)FreeMB.Free.base(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); + // Align the address. + Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + + if (FreeMB.PendingPrefixIndex == (unsigned)-1) { + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + + // Remember this pending block, such that future allocations can just + // modify it rather than creating a new one + FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; + } else { + sys::MemoryBlock &PendingMB = + MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; + PendingMB = sys::MemoryBlock(PendingMB.base(), + Addr + Size - (uintptr_t)PendingMB.base()); + } + + // Remember how much free space is now left in this block + FreeMB.Free = + sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); + return (uint8_t *)Addr; + } + } + + // No pre-allocated free block was large enough. Allocate a new memory region. + // Note that all sections get allocated as read-write. The permissions will + // be updated later based on memory group. + // + // FIXME: It would be useful to define a default allocation size (or add + // it as a constructor parameter) to minimize the number of allocations. + // + // FIXME: Initialize the Near member for each memory group to avoid + // interleaving. + std::error_code ec; + sys::MemoryBlock MB = MMapper.allocateMappedMemory( + Purpose, RequiredSize, &MemGroup.Near, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); + if (ec) { + // FIXME: Add error propagation to the interface. + return nullptr; + } + + // Save this address as the basis for our next request + MemGroup.Near = MB; + + // Copy the address to all the other groups, if they have not + // been initialized. + if (CodeMem.Near.base() == nullptr) + CodeMem.Near = MB; + if (RODataMem.Near.base() == nullptr) + RODataMem.Near = MB; + if (RWDataMem.Near.base() == nullptr) + RWDataMem.Near = MB; + + // Remember that we allocated this memory + MemGroup.AllocatedMem.push_back(MB); + Addr = (uintptr_t)MB.base(); + uintptr_t EndOfBlock = Addr + MB.allocatedSize(); + + // Align the address. + Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + + // The allocateMappedMemory may allocate much more memory than we need. In + // this case, we store the unused memory as a free memory block. + unsigned FreeSize = EndOfBlock - Addr - Size; + if (FreeSize > 16) { + FreeMemBlock FreeMB; + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize); + FreeMB.PendingPrefixIndex = (unsigned)-1; + MemGroup.FreeMem.push_back(FreeMB); + } + + // Return aligned address + return (uint8_t *)Addr; +} + +bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { + // FIXME: Should in-progress permissions be reverted if an error occurs? + std::error_code ec; + + // Make code memory executable. + ec = applyMemoryGroupPermissions(CodeMem, + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + if (ec) { + if (ErrMsg) { + *ErrMsg = ec.message(); + } + return true; + } + + // Make read-only data memory read-only. + ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ); + if (ec) { + if (ErrMsg) { + *ErrMsg = ec.message(); + } + return true; + } + + // Read-write data memory already has the correct permissions + + // Some platforms with separate data cache and instruction cache require + // explicit cache flush, otherwise JIT code manipulations (like resolved + // relocations) will get to the data cache but not to the instruction cache. + invalidateInstructionCache(); + + return false; +} + +static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { + static const size_t PageSize = sys::Process::getPageSizeEstimate(); + + size_t StartOverlap = + (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; + + size_t TrimmedSize = M.allocatedSize(); + TrimmedSize -= StartOverlap; + TrimmedSize -= TrimmedSize % PageSize; + + sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), + TrimmedSize); + + assert(((uintptr_t)Trimmed.base() % PageSize) == 0); + assert((Trimmed.allocatedSize() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && + Trimmed.allocatedSize() <= M.allocatedSize()); + + return Trimmed; +} + +std::error_code +SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions) { + for (sys::MemoryBlock &MB : MemGroup.PendingMem) + if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions)) + return EC; + + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the entire + // page because one of the pending blocks may have overlapped it. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + FreeMB.Free = trimBlockToPageSize(FreeMB.Free); + // We cleared the PendingMem list, so all these pointers are now invalid + FreeMB.PendingPrefixIndex = (unsigned)-1; + } + + // Remove all blocks which are now empty + erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) { + return FreeMB.Free.allocatedSize() == 0; + }); + + return std::error_code(); +} + +void SectionMemoryManager::invalidateInstructionCache() { + for (sys::MemoryBlock &Block : CodeMem.PendingMem) + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); +} + +SectionMemoryManager::~SectionMemoryManager() { + for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { + for (sys::MemoryBlock &Block : Group->AllocatedMem) + MMapper.releaseMappedMemory(Block); + } +} + +SectionMemoryManager::MemoryMapper::~MemoryMapper() {} + +void SectionMemoryManager::anchor() {} + +namespace { +// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls +// into sys::Memory. +class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { +public: + sys::MemoryBlock + allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose, + size_t NumBytes, const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) override { + return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC); + } + + std::error_code protectMappedMemory(const sys::MemoryBlock &Block, + unsigned Flags) override { + return sys::Memory::protectMappedMemory(Block, Flags); + } + + std::error_code releaseMappedMemory(sys::MemoryBlock &M) override { + return sys::Memory::releaseMappedMemory(M); + } +}; + +DefaultMMapper DefaultMMapperInstance; +} // namespace + +SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM) + : MMapper(MM ? *MM : DefaultMMapperInstance) {} + +} // namespace llvm diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h new file mode 100644 index 000000000..455efc9f9 --- /dev/null +++ b/ffi/memorymanager.h @@ -0,0 +1,193 @@ +//===- SectionMemoryManager.h - Memory manager for MCJIT/RtDyld -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the declaration of a section-based memory manager used by +// the MCJIT execution engine and RuntimeDyld. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/Support/Memory.h" +#include +#include +#include + +namespace llvm { + +/// This is a simple memory manager which implements the methods called by +/// the RuntimeDyld class to allocate memory for section-based loading of +/// objects, usually those generated by the MCJIT execution engine. +/// +/// This memory manager allocates all section memory as read-write. The +/// RuntimeDyld will copy JITed section memory into these allocated blocks +/// and perform any necessary linking and relocations. +/// +/// Any client using this memory manager MUST ensure that section-specific +/// page permissions have been applied before attempting to execute functions +/// in the JITed object. Permissions can be applied either by calling +/// MCJIT::finalizeObject or by calling SectionMemoryManager::finalizeMemory +/// directly. Clients of MCJIT should call MCJIT::finalizeObject. +class SectionMemoryManager : public RTDyldMemoryManager { +public: + /// This enum describes the various reasons to allocate pages from + /// allocateMappedMemory. + enum class AllocationPurpose { + Code, + ROData, + RWData, + }; + + /// Implementations of this interface are used by SectionMemoryManager to + /// request pages from the operating system. + class MemoryMapper { + public: + /// This method attempts to allocate \p NumBytes bytes of virtual memory for + /// \p Purpose. \p NearBlock may point to an existing allocation, in which + /// case an attempt is made to allocate more memory near the existing block. + /// The actual allocated address is not guaranteed to be near the requested + /// address. \p Flags is used to set the initial protection flags for the + /// block of the memory. \p EC [out] returns an object describing any error + /// that occurs. + /// + /// This method may allocate more than the number of bytes requested. The + /// actual number of bytes allocated is indicated in the returned + /// MemoryBlock. + /// + /// The start of the allocated block must be aligned with the system + /// allocation granularity (64K on Windows, page size on Linux). If the + /// address following \p NearBlock is not so aligned, it will be rounded up + /// to the next allocation granularity boundary. + /// + /// \r a non-null MemoryBlock if the function was successful, otherwise a + /// null MemoryBlock with \p EC describing the error. + virtual sys::MemoryBlock + allocateMappedMemory(AllocationPurpose Purpose, size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) = 0; + + /// This method sets the protection flags for a block of memory to the state + /// specified by \p Flags. The behavior is not specified if the memory was + /// not allocated using the allocateMappedMemory method. + /// \p Block describes the memory block to be protected. + /// \p Flags specifies the new protection state to be assigned to the block. + /// + /// If \p Flags is MF_WRITE, the actual behavior varies with the operating + /// system (i.e. MF_READ | MF_WRITE on Windows) and the target architecture + /// (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code protectMappedMemory(const sys::MemoryBlock &Block, + unsigned Flags) = 0; + + /// This method releases a block of memory that was allocated with the + /// allocateMappedMemory method. It should not be used to release any memory + /// block allocated any other way. + /// \p Block describes the memory to be released. + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code releaseMappedMemory(sys::MemoryBlock &M) = 0; + + virtual ~MemoryMapper(); + }; + + /// Creates a SectionMemoryManager instance with \p MM as the associated + /// memory mapper. If \p MM is nullptr then a default memory mapper is used + /// that directly calls into the operating system. + SectionMemoryManager(MemoryMapper *MM = nullptr); + SectionMemoryManager(const SectionMemoryManager &) = delete; + void operator=(const SectionMemoryManager &) = delete; + ~SectionMemoryManager() override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. + /// + /// The value of \p Alignment must be a power of two. If \p Alignment is zero + /// a default alignment of 16 will be used. + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. + /// + /// The value of \p Alignment must be a power of two. If \p Alignment is zero + /// a default alignment of 16 will be used. + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool isReadOnly) override; + + /// Update section-specific memory permissions and other attributes. + /// + /// This method is called when object loading is complete and section page + /// permissions can be applied. It is up to the memory manager implementation + /// to decide whether or not to act on this method. The memory manager will + /// typically allocate all sections as read-write and then apply specific + /// permissions when this method is called. Code sections cannot be executed + /// until this function has been called. In addition, any cache coherency + /// operations needed to reliably use the memory are also performed. + /// + /// \returns true if an error occurred, false otherwise. + bool finalizeMemory(std::string *ErrMsg = nullptr) override; + + /// Invalidate instruction cache for code sections. + /// + /// Some platforms with separate data cache and instruction cache require + /// explicit cache flush, otherwise JIT code manipulations (like resolved + /// relocations) will get to the data cache but not to the instruction cache. + /// + /// This method is called from finalizeMemory. + virtual void invalidateInstructionCache(); + +private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right before + // this block, store it's index in PendingMem, to be able to update the + // pending region if part of this block is allocated, rather than having to + // create a new one + unsigned PendingPrefixIndex; + }; + + struct MemoryGroup { + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the user. + SmallVector PendingMem; + SmallVector FreeMem; + + // All memory blocks that have been requested from the system + SmallVector AllocatedMem; + + sys::MemoryBlock Near; + }; + + uint8_t *allocateSection(AllocationPurpose Purpose, uintptr_t Size, + unsigned Alignment); + + std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions); + + void anchor() override; + + MemoryGroup CodeMem; + MemoryGroup RWDataMem; + MemoryGroup RODataMem; + MemoryMapper &MMapper; +}; + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H From 441320ff783dec1279be058b5b5bd71ad7b1df90 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Wed, 15 Nov 2023 13:50:56 +0000 Subject: [PATCH 2/8] Run clang-format-14 over memory manager files This makes them compliant with our C++ style check. --- ffi/memorymanager.cpp | 383 +++++++++++++++++++++--------------------- ffi/memorymanager.h | 287 +++++++++++++++---------------- 2 files changed, 340 insertions(+), 330 deletions(-) diff --git a/ffi/memorymanager.cpp b/ffi/memorymanager.cpp index 56b232b9d..232e848c6 100644 --- a/ffi/memorymanager.cpp +++ b/ffi/memorymanager.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Process.h" @@ -23,219 +23,224 @@ uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size, unsigned SectionID, StringRef SectionName, bool IsReadOnly) { - if (IsReadOnly) - return allocateSection(SectionMemoryManager::AllocationPurpose::ROData, + if (IsReadOnly) + return allocateSection(SectionMemoryManager::AllocationPurpose::ROData, + Size, Alignment); + return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size, Alignment); - return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size, - Alignment); } uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size, unsigned Alignment, unsigned SectionID, StringRef SectionName) { - return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size, - Alignment); + return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size, + Alignment); } uint8_t *SectionMemoryManager::allocateSection( SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size, unsigned Alignment) { - if (!Alignment) - Alignment = 16; - - assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two."); - - uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1); - uintptr_t Addr = 0; - - MemoryGroup &MemGroup = [&]() -> MemoryGroup & { - switch (Purpose) { - case AllocationPurpose::Code: - return CodeMem; - case AllocationPurpose::ROData: - return RODataMem; - case AllocationPurpose::RWData: - return RWDataMem; - } - llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose"); - }(); - - // Look in the list of free memory regions and use a block there if one - // is available. - for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { - if (FreeMB.Free.allocatedSize() >= RequiredSize) { - Addr = (uintptr_t)FreeMB.Free.base(); - uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); - // Align the address. - Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); - - if (FreeMB.PendingPrefixIndex == (unsigned)-1) { - // The part of the block we're giving out to the user is now pending - MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); - - // Remember this pending block, such that future allocations can just - // modify it rather than creating a new one - FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; - } else { - sys::MemoryBlock &PendingMB = - MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; - PendingMB = sys::MemoryBlock(PendingMB.base(), + if (!Alignment) + Alignment = 16; + + assert(!(Alignment & (Alignment - 1)) && + "Alignment must be a power of two."); + + uintptr_t RequiredSize = + Alignment * ((Size + Alignment - 1) / Alignment + 1); + uintptr_t Addr = 0; + + MemoryGroup &MemGroup = [&]() -> MemoryGroup & { + switch (Purpose) { + case AllocationPurpose::Code: + return CodeMem; + case AllocationPurpose::ROData: + return RODataMem; + case AllocationPurpose::RWData: + return RWDataMem; + } + llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose"); + }(); + + // Look in the list of free memory regions and use a block there if one + // is available. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.allocatedSize() >= RequiredSize) { + Addr = (uintptr_t)FreeMB.Free.base(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); + // Align the address. + Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + + if (FreeMB.PendingPrefixIndex == (unsigned)-1) { + // The part of the block we're giving out to the user is now + // pending + MemGroup.PendingMem.push_back( + sys::MemoryBlock((void *)Addr, Size)); + + // Remember this pending block, such that future allocations can + // just modify it rather than creating a new one + FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; + } else { + sys::MemoryBlock &PendingMB = + MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; + PendingMB = + sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base()); - } + } + + // Remember how much free space is now left in this block + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), + EndOfBlock - Addr - Size); + return (uint8_t *)Addr; + } + } - // Remember how much free space is now left in this block - FreeMB.Free = - sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); - return (uint8_t *)Addr; + // No pre-allocated free block was large enough. Allocate a new memory + // region. Note that all sections get allocated as read-write. The + // permissions will be updated later based on memory group. + // + // FIXME: It would be useful to define a default allocation size (or add + // it as a constructor parameter) to minimize the number of allocations. + // + // FIXME: Initialize the Near member for each memory group to avoid + // interleaving. + std::error_code ec; + sys::MemoryBlock MB = MMapper.allocateMappedMemory( + Purpose, RequiredSize, &MemGroup.Near, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); + if (ec) { + // FIXME: Add error propagation to the interface. + return nullptr; } - } - - // No pre-allocated free block was large enough. Allocate a new memory region. - // Note that all sections get allocated as read-write. The permissions will - // be updated later based on memory group. - // - // FIXME: It would be useful to define a default allocation size (or add - // it as a constructor parameter) to minimize the number of allocations. - // - // FIXME: Initialize the Near member for each memory group to avoid - // interleaving. - std::error_code ec; - sys::MemoryBlock MB = MMapper.allocateMappedMemory( - Purpose, RequiredSize, &MemGroup.Near, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); - if (ec) { - // FIXME: Add error propagation to the interface. - return nullptr; - } - - // Save this address as the basis for our next request - MemGroup.Near = MB; - - // Copy the address to all the other groups, if they have not - // been initialized. - if (CodeMem.Near.base() == nullptr) - CodeMem.Near = MB; - if (RODataMem.Near.base() == nullptr) - RODataMem.Near = MB; - if (RWDataMem.Near.base() == nullptr) - RWDataMem.Near = MB; - - // Remember that we allocated this memory - MemGroup.AllocatedMem.push_back(MB); - Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.allocatedSize(); - - // Align the address. - Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); - - // The part of the block we're giving out to the user is now pending - MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); - - // The allocateMappedMemory may allocate much more memory than we need. In - // this case, we store the unused memory as a free memory block. - unsigned FreeSize = EndOfBlock - Addr - Size; - if (FreeSize > 16) { - FreeMemBlock FreeMB; - FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize); - FreeMB.PendingPrefixIndex = (unsigned)-1; - MemGroup.FreeMem.push_back(FreeMB); - } - - // Return aligned address - return (uint8_t *)Addr; + + // Save this address as the basis for our next request + MemGroup.Near = MB; + + // Copy the address to all the other groups, if they have not + // been initialized. + if (CodeMem.Near.base() == nullptr) + CodeMem.Near = MB; + if (RODataMem.Near.base() == nullptr) + RODataMem.Near = MB; + if (RWDataMem.Near.base() == nullptr) + RWDataMem.Near = MB; + + // Remember that we allocated this memory + MemGroup.AllocatedMem.push_back(MB); + Addr = (uintptr_t)MB.base(); + uintptr_t EndOfBlock = Addr + MB.allocatedSize(); + + // Align the address. + Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + + // The allocateMappedMemory may allocate much more memory than we need. In + // this case, we store the unused memory as a free memory block. + unsigned FreeSize = EndOfBlock - Addr - Size; + if (FreeSize > 16) { + FreeMemBlock FreeMB; + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize); + FreeMB.PendingPrefixIndex = (unsigned)-1; + MemGroup.FreeMem.push_back(FreeMB); + } + + // Return aligned address + return (uint8_t *)Addr; } bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { - // FIXME: Should in-progress permissions be reverted if an error occurs? - std::error_code ec; - - // Make code memory executable. - ec = applyMemoryGroupPermissions(CodeMem, - sys::Memory::MF_READ | sys::Memory::MF_EXEC); - if (ec) { - if (ErrMsg) { - *ErrMsg = ec.message(); + // FIXME: Should in-progress permissions be reverted if an error occurs? + std::error_code ec; + + // Make code memory executable. + ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + if (ec) { + if (ErrMsg) { + *ErrMsg = ec.message(); + } + return true; } - return true; - } - - // Make read-only data memory read-only. - ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ); - if (ec) { - if (ErrMsg) { - *ErrMsg = ec.message(); + + // Make read-only data memory read-only. + ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ); + if (ec) { + if (ErrMsg) { + *ErrMsg = ec.message(); + } + return true; } - return true; - } - // Read-write data memory already has the correct permissions + // Read-write data memory already has the correct permissions - // Some platforms with separate data cache and instruction cache require - // explicit cache flush, otherwise JIT code manipulations (like resolved - // relocations) will get to the data cache but not to the instruction cache. - invalidateInstructionCache(); + // Some platforms with separate data cache and instruction cache require + // explicit cache flush, otherwise JIT code manipulations (like resolved + // relocations) will get to the data cache but not to the instruction cache. + invalidateInstructionCache(); - return false; + return false; } static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { - static const size_t PageSize = sys::Process::getPageSizeEstimate(); + static const size_t PageSize = sys::Process::getPageSizeEstimate(); - size_t StartOverlap = - (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; + size_t StartOverlap = + (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; - size_t TrimmedSize = M.allocatedSize(); - TrimmedSize -= StartOverlap; - TrimmedSize -= TrimmedSize % PageSize; + size_t TrimmedSize = M.allocatedSize(); + TrimmedSize -= StartOverlap; + TrimmedSize -= TrimmedSize % PageSize; - sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), - TrimmedSize); + sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), + TrimmedSize); - assert(((uintptr_t)Trimmed.base() % PageSize) == 0); - assert((Trimmed.allocatedSize() % PageSize) == 0); - assert(M.base() <= Trimmed.base() && - Trimmed.allocatedSize() <= M.allocatedSize()); + assert(((uintptr_t)Trimmed.base() % PageSize) == 0); + assert((Trimmed.allocatedSize() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && + Trimmed.allocatedSize() <= M.allocatedSize()); - return Trimmed; + return Trimmed; } std::error_code SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions) { - for (sys::MemoryBlock &MB : MemGroup.PendingMem) - if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions)) - return EC; - - MemGroup.PendingMem.clear(); - - // Now go through free blocks and trim any of them that don't span the entire - // page because one of the pending blocks may have overlapped it. - for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { - FreeMB.Free = trimBlockToPageSize(FreeMB.Free); - // We cleared the PendingMem list, so all these pointers are now invalid - FreeMB.PendingPrefixIndex = (unsigned)-1; - } - - // Remove all blocks which are now empty - erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) { - return FreeMB.Free.allocatedSize() == 0; - }); - - return std::error_code(); + for (sys::MemoryBlock &MB : MemGroup.PendingMem) + if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions)) + return EC; + + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the + // entire page because one of the pending blocks may have overlapped it. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + FreeMB.Free = trimBlockToPageSize(FreeMB.Free); + // We cleared the PendingMem list, so all these pointers are now invalid + FreeMB.PendingPrefixIndex = (unsigned)-1; + } + + // Remove all blocks which are now empty + erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) { + return FreeMB.Free.allocatedSize() == 0; + }); + + return std::error_code(); } void SectionMemoryManager::invalidateInstructionCache() { - for (sys::MemoryBlock &Block : CodeMem.PendingMem) - sys::Memory::InvalidateInstructionCache(Block.base(), - Block.allocatedSize()); + for (sys::MemoryBlock &Block : CodeMem.PendingMem) + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); } SectionMemoryManager::~SectionMemoryManager() { - for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { - for (sys::MemoryBlock &Block : Group->AllocatedMem) - MMapper.releaseMappedMemory(Block); - } + for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { + for (sys::MemoryBlock &Block : Group->AllocatedMem) + MMapper.releaseMappedMemory(Block); + } } SectionMemoryManager::MemoryMapper::~MemoryMapper() {} @@ -246,22 +251,24 @@ namespace { // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls // into sys::Memory. class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { -public: - sys::MemoryBlock - allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose, - size_t NumBytes, const sys::MemoryBlock *const NearBlock, - unsigned Flags, std::error_code &EC) override { - return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC); - } - - std::error_code protectMappedMemory(const sys::MemoryBlock &Block, - unsigned Flags) override { - return sys::Memory::protectMappedMemory(Block, Flags); - } - - std::error_code releaseMappedMemory(sys::MemoryBlock &M) override { - return sys::Memory::releaseMappedMemory(M); - } + public: + sys::MemoryBlock + allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose, + size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) override { + return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, + EC); + } + + std::error_code protectMappedMemory(const sys::MemoryBlock &Block, + unsigned Flags) override { + return sys::Memory::protectMappedMemory(Block, Flags); + } + + std::error_code releaseMappedMemory(sys::MemoryBlock &M) override { + return sys::Memory::releaseMappedMemory(M); + } }; DefaultMMapper DefaultMMapperInstance; diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h index 455efc9f9..a60d31552 100644 --- a/ffi/memorymanager.h +++ b/ffi/memorymanager.h @@ -37,155 +37,158 @@ namespace llvm { /// MCJIT::finalizeObject or by calling SectionMemoryManager::finalizeMemory /// directly. Clients of MCJIT should call MCJIT::finalizeObject. class SectionMemoryManager : public RTDyldMemoryManager { -public: - /// This enum describes the various reasons to allocate pages from - /// allocateMappedMemory. - enum class AllocationPurpose { - Code, - ROData, - RWData, - }; - - /// Implementations of this interface are used by SectionMemoryManager to - /// request pages from the operating system. - class MemoryMapper { public: - /// This method attempts to allocate \p NumBytes bytes of virtual memory for - /// \p Purpose. \p NearBlock may point to an existing allocation, in which - /// case an attempt is made to allocate more memory near the existing block. - /// The actual allocated address is not guaranteed to be near the requested - /// address. \p Flags is used to set the initial protection flags for the - /// block of the memory. \p EC [out] returns an object describing any error - /// that occurs. + /// This enum describes the various reasons to allocate pages from + /// allocateMappedMemory. + enum class AllocationPurpose { + Code, + ROData, + RWData, + }; + + /// Implementations of this interface are used by SectionMemoryManager to + /// request pages from the operating system. + class MemoryMapper { + public: + /// This method attempts to allocate \p NumBytes bytes of virtual memory + /// for \p Purpose. \p NearBlock may point to an existing allocation, + /// in which case an attempt is made to allocate more memory near the + /// existing block. The actual allocated address is not guaranteed to be + /// near the requested address. \p Flags is used to set the initial + /// protection flags for the block of the memory. \p EC [out] returns + /// an object describing any error that occurs. + /// + /// This method may allocate more than the number of bytes requested. + /// The actual number of bytes allocated is indicated in the returned + /// MemoryBlock. + /// + /// The start of the allocated block must be aligned with the system + /// allocation granularity (64K on Windows, page size on Linux). If the + /// address following \p NearBlock is not so aligned, it will be rounded + /// up to the next allocation granularity boundary. + /// + /// \r a non-null MemoryBlock if the function was successful, otherwise + /// a null MemoryBlock with \p EC describing the error. + virtual sys::MemoryBlock + allocateMappedMemory(AllocationPurpose Purpose, size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) = 0; + + /// This method sets the protection flags for a block of memory to the + /// state specified by \p Flags. The behavior is not specified if the + /// memory was not allocated using the allocateMappedMemory method. \p + /// Block describes the memory block to be protected. \p Flags specifies + /// the new protection state to be assigned to the block. + /// + /// If \p Flags is MF_WRITE, the actual behavior varies with the + /// operating system (i.e. MF_READ | MF_WRITE on Windows) and the target + /// architecture (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code + protectMappedMemory(const sys::MemoryBlock &Block, unsigned Flags) = 0; + + /// This method releases a block of memory that was allocated with the + /// allocateMappedMemory method. It should not be used to release any + /// memory block allocated any other way. \p Block describes the memory + /// to be released. + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code releaseMappedMemory(sys::MemoryBlock &M) = 0; + + virtual ~MemoryMapper(); + }; + + /// Creates a SectionMemoryManager instance with \p MM as the associated + /// memory mapper. If \p MM is nullptr then a default memory mapper is used + /// that directly calls into the operating system. + SectionMemoryManager(MemoryMapper *MM = nullptr); + SectionMemoryManager(const SectionMemoryManager &) = delete; + void operator=(const SectionMemoryManager &) = delete; + ~SectionMemoryManager() override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. /// - /// This method may allocate more than the number of bytes requested. The - /// actual number of bytes allocated is indicated in the returned - /// MemoryBlock. + /// The value of \p Alignment must be a power of two. If \p Alignment is + /// zero a default alignment of 16 will be used. + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. /// - /// The start of the allocated block must be aligned with the system - /// allocation granularity (64K on Windows, page size on Linux). If the - /// address following \p NearBlock is not so aligned, it will be rounded up - /// to the next allocation granularity boundary. + /// The value of \p Alignment must be a power of two. If \p Alignment is + /// zero a default alignment of 16 will be used. + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool isReadOnly) override; + + /// Update section-specific memory permissions and other attributes. /// - /// \r a non-null MemoryBlock if the function was successful, otherwise a - /// null MemoryBlock with \p EC describing the error. - virtual sys::MemoryBlock - allocateMappedMemory(AllocationPurpose Purpose, size_t NumBytes, - const sys::MemoryBlock *const NearBlock, - unsigned Flags, std::error_code &EC) = 0; - - /// This method sets the protection flags for a block of memory to the state - /// specified by \p Flags. The behavior is not specified if the memory was - /// not allocated using the allocateMappedMemory method. - /// \p Block describes the memory block to be protected. - /// \p Flags specifies the new protection state to be assigned to the block. + /// This method is called when object loading is complete and section page + /// permissions can be applied. It is up to the memory manager + /// implementation to decide whether or not to act on this method. The + /// memory manager will typically allocate all sections as read-write and + /// then apply specific permissions when this method is called. Code + /// sections cannot be executed until this function has been called. In + /// addition, any cache coherency operations needed to reliably use the + /// memory are also performed. /// - /// If \p Flags is MF_WRITE, the actual behavior varies with the operating - /// system (i.e. MF_READ | MF_WRITE on Windows) and the target architecture - /// (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). + /// \returns true if an error occurred, false otherwise. + bool finalizeMemory(std::string *ErrMsg = nullptr) override; + + /// Invalidate instruction cache for code sections. /// - /// \r error_success if the function was successful, or an error_code - /// describing the failure if an error occurred. - virtual std::error_code protectMappedMemory(const sys::MemoryBlock &Block, - unsigned Flags) = 0; - - /// This method releases a block of memory that was allocated with the - /// allocateMappedMemory method. It should not be used to release any memory - /// block allocated any other way. - /// \p Block describes the memory to be released. + /// Some platforms with separate data cache and instruction cache require + /// explicit cache flush, otherwise JIT code manipulations (like resolved + /// relocations) will get to the data cache but not to the instruction + /// cache. /// - /// \r error_success if the function was successful, or an error_code - /// describing the failure if an error occurred. - virtual std::error_code releaseMappedMemory(sys::MemoryBlock &M) = 0; - - virtual ~MemoryMapper(); - }; - - /// Creates a SectionMemoryManager instance with \p MM as the associated - /// memory mapper. If \p MM is nullptr then a default memory mapper is used - /// that directly calls into the operating system. - SectionMemoryManager(MemoryMapper *MM = nullptr); - SectionMemoryManager(const SectionMemoryManager &) = delete; - void operator=(const SectionMemoryManager &) = delete; - ~SectionMemoryManager() override; - - /// Allocates a memory block of (at least) the given size suitable for - /// executable code. - /// - /// The value of \p Alignment must be a power of two. If \p Alignment is zero - /// a default alignment of 16 will be used. - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override; - - /// Allocates a memory block of (at least) the given size suitable for - /// executable code. - /// - /// The value of \p Alignment must be a power of two. If \p Alignment is zero - /// a default alignment of 16 will be used. - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool isReadOnly) override; - - /// Update section-specific memory permissions and other attributes. - /// - /// This method is called when object loading is complete and section page - /// permissions can be applied. It is up to the memory manager implementation - /// to decide whether or not to act on this method. The memory manager will - /// typically allocate all sections as read-write and then apply specific - /// permissions when this method is called. Code sections cannot be executed - /// until this function has been called. In addition, any cache coherency - /// operations needed to reliably use the memory are also performed. - /// - /// \returns true if an error occurred, false otherwise. - bool finalizeMemory(std::string *ErrMsg = nullptr) override; - - /// Invalidate instruction cache for code sections. - /// - /// Some platforms with separate data cache and instruction cache require - /// explicit cache flush, otherwise JIT code manipulations (like resolved - /// relocations) will get to the data cache but not to the instruction cache. - /// - /// This method is called from finalizeMemory. - virtual void invalidateInstructionCache(); - -private: - struct FreeMemBlock { - // The actual block of free memory - sys::MemoryBlock Free; - // If there is a pending allocation from the same reservation right before - // this block, store it's index in PendingMem, to be able to update the - // pending region if part of this block is allocated, rather than having to - // create a new one - unsigned PendingPrefixIndex; - }; - - struct MemoryGroup { - // PendingMem contains all blocks of memory (subblocks of AllocatedMem) - // which have not yet had their permissions applied, but have been given - // out to the user. FreeMem contains all block of memory, which have - // neither had their permissions applied, nor been given out to the user. - SmallVector PendingMem; - SmallVector FreeMem; - - // All memory blocks that have been requested from the system - SmallVector AllocatedMem; - - sys::MemoryBlock Near; - }; - - uint8_t *allocateSection(AllocationPurpose Purpose, uintptr_t Size, - unsigned Alignment); - - std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, - unsigned Permissions); - - void anchor() override; - - MemoryGroup CodeMem; - MemoryGroup RWDataMem; - MemoryGroup RODataMem; - MemoryMapper &MMapper; + /// This method is called from finalizeMemory. + virtual void invalidateInstructionCache(); + + private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right + // before this block, store it's index in PendingMem, to be able to + // update the pending region if part of this block is allocated, rather + // than having to create a new one + unsigned PendingPrefixIndex; + }; + + struct MemoryGroup { + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the + // user. + SmallVector PendingMem; + SmallVector FreeMem; + + // All memory blocks that have been requested from the system + SmallVector AllocatedMem; + + sys::MemoryBlock Near; + }; + + uint8_t *allocateSection(AllocationPurpose Purpose, uintptr_t Size, + unsigned Alignment); + + std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions); + + void anchor() override; + + MemoryGroup CodeMem; + MemoryGroup RWDataMem; + MemoryGroup RODataMem; + MemoryMapper &MMapper; }; } // end namespace llvm From 88046679bc94a468afbad1544da9bc303f0fa555 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Wed, 15 Nov 2023 13:57:15 +0000 Subject: [PATCH 3/8] Rename the memory manager and use it in llvmlite Notes on the changes: - The memory manager is added to the build system. - The `LlvmliteMemoryManager` class is exported as a public interface. - When creating an execution engine, we set it to use our memory manager. --- ffi/CMakeLists.txt | 2 +- ffi/Makefile.freebsd | 2 +- ffi/Makefile.linux | 2 +- ffi/Makefile.osx | 2 +- ffi/executionengine.cpp | 5 ++++ ffi/memorymanager.cpp | 57 ++++++++++++++++++++--------------------- ffi/memorymanager.h | 24 ++++++++--------- 7 files changed, 48 insertions(+), 46 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index e4ce2c12c..3aab3927c 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -40,7 +40,7 @@ endif() add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp linker.cpp object_file.cpp - custom_passes.cpp orcjit.cpp) + custom_passes.cpp orcjit.cpp memorymanager.cpp) # Find the libraries that correspond to the LLVM components # that we wish to use. diff --git a/ffi/Makefile.freebsd b/ffi/Makefile.freebsd index a55349a34..150cbb285 100644 --- a/ffi/Makefile.freebsd +++ b/ffi/Makefile.freebsd @@ -11,7 +11,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp orcjit.cpp custom_passes.cpp + linker.cpp object_file.cpp orcjit.cpp custom_passes.cpp memorymanager.cpp OUTPUT = libllvmlite.so all: $(OUTPUT) diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index c1df42dbb..c101510f2 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -13,7 +13,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h OBJ = assembly.o bitcode.o core.o initfini.o module.o value.o \ executionengine.o transforms.o passmanagers.o targets.o dylib.o \ - linker.o object_file.o custom_passes.o orcjit.o + linker.o object_file.o custom_passes.o orcjit.o memorymanager.o OUTPUT = libllvmlite.so all: $(OUTPUT) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index a0d06db4f..9668dd356 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -8,7 +8,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp custom_passes.cpp orcjit.cpp + linker.cpp object_file.cpp custom_passes.cpp orcjit.cpp memorymanager.cpp OUTPUT = libllvmlite.dylib MACOSX_DEPLOYMENT_TARGET ?= 10.9 diff --git a/ffi/executionengine.cpp b/ffi/executionengine.cpp index 79f1ec9a3..54a6f9bb2 100644 --- a/ffi/executionengine.cpp +++ b/ffi/executionengine.cpp @@ -1,4 +1,5 @@ #include "core.h" +#include "memorymanager.h" #include "llvm-c/ExecutionEngine.h" #include "llvm-c/Object.h" @@ -73,6 +74,10 @@ static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, eb.setErrorStr(&err); eb.setEngineKind(llvm::EngineKind::JIT); + std::unique_ptr mm = + std::make_unique(); + eb.setMCJITMemoryManager(std::move(mm)); + /* EngineBuilder::create loads the current process symbols */ llvm::ExecutionEngine *engine = eb.create(llvm::unwrap(TM)); diff --git a/ffi/memorymanager.cpp b/ffi/memorymanager.cpp index 232e848c6..0db2fdd49 100644 --- a/ffi/memorymanager.cpp +++ b/ffi/memorymanager.cpp @@ -1,4 +1,4 @@ -//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==// +//===---- memorymanager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*----===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,35 +11,34 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Config/config.h" -#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "memorymanager.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Process.h" namespace llvm { -uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size, - unsigned Alignment, - unsigned SectionID, - StringRef SectionName, - bool IsReadOnly) { +uint8_t *LlvmliteMemoryManager::allocateDataSection(uintptr_t Size, + unsigned Alignment, + unsigned SectionID, + StringRef SectionName, + bool IsReadOnly) { if (IsReadOnly) - return allocateSection(SectionMemoryManager::AllocationPurpose::ROData, + return allocateSection(LlvmliteMemoryManager::AllocationPurpose::ROData, Size, Alignment); - return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, + return allocateSection(LlvmliteMemoryManager::AllocationPurpose::RWData, Size, Alignment); } -uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size, - unsigned Alignment, - unsigned SectionID, - StringRef SectionName) { - return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size, +uint8_t *LlvmliteMemoryManager::allocateCodeSection(uintptr_t Size, + unsigned Alignment, + unsigned SectionID, + StringRef SectionName) { + return allocateSection(LlvmliteMemoryManager::AllocationPurpose::Code, Size, Alignment); } -uint8_t *SectionMemoryManager::allocateSection( - SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size, +uint8_t *LlvmliteMemoryManager::allocateSection( + LlvmliteMemoryManager::AllocationPurpose Purpose, uintptr_t Size, unsigned Alignment) { if (!Alignment) Alignment = 16; @@ -60,7 +59,7 @@ uint8_t *SectionMemoryManager::allocateSection( case AllocationPurpose::RWData: return RWDataMem; } - llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose"); + llvm_unreachable("Unknown LlvmliteMemoryManager::AllocationPurpose"); }(); // Look in the list of free memory regions and use a block there if one @@ -151,7 +150,7 @@ uint8_t *SectionMemoryManager::allocateSection( return (uint8_t *)Addr; } -bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { +bool LlvmliteMemoryManager::finalizeMemory(std::string *ErrMsg) { // FIXME: Should in-progress permissions be reverted if an error occurs? std::error_code ec; @@ -206,8 +205,8 @@ static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { } std::error_code -SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, - unsigned Permissions) { +LlvmliteMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions) { for (sys::MemoryBlock &MB : MemGroup.PendingMem) if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions)) return EC; @@ -230,30 +229,30 @@ SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, return std::error_code(); } -void SectionMemoryManager::invalidateInstructionCache() { +void LlvmliteMemoryManager::invalidateInstructionCache() { for (sys::MemoryBlock &Block : CodeMem.PendingMem) sys::Memory::InvalidateInstructionCache(Block.base(), Block.allocatedSize()); } -SectionMemoryManager::~SectionMemoryManager() { +LlvmliteMemoryManager::~LlvmliteMemoryManager() { for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { for (sys::MemoryBlock &Block : Group->AllocatedMem) MMapper.releaseMappedMemory(Block); } } -SectionMemoryManager::MemoryMapper::~MemoryMapper() {} +LlvmliteMemoryManager::MemoryMapper::~MemoryMapper() {} -void SectionMemoryManager::anchor() {} +void LlvmliteMemoryManager::anchor() {} namespace { -// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls +// Trivial implementation of LlvmliteMemoryManager::MemoryMapper that just calls // into sys::Memory. -class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { +class DefaultMMapper final : public LlvmliteMemoryManager::MemoryMapper { public: sys::MemoryBlock - allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose, + allocateMappedMemory(LlvmliteMemoryManager::AllocationPurpose Purpose, size_t NumBytes, const sys::MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC) override { @@ -274,7 +273,7 @@ class DefaultMMapper final : public SectionMemoryManager::MemoryMapper { DefaultMMapper DefaultMMapperInstance; } // namespace -SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM) +LlvmliteMemoryManager::LlvmliteMemoryManager(MemoryMapper *MM) : MMapper(MM ? *MM : DefaultMMapperInstance) {} } // namespace llvm diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h index a60d31552..ed00a2cee 100644 --- a/ffi/memorymanager.h +++ b/ffi/memorymanager.h @@ -1,4 +1,4 @@ -//===- SectionMemoryManager.h - Memory manager for MCJIT/RtDyld -*- C++ -*-===// +//===----- memorymanager.h - Memory manager for MCJIT/RtDyld -*- C++ -*----===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,9 +11,9 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H -#define LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H +#pragma once +#include "core.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/Support/Memory.h" @@ -34,9 +34,9 @@ namespace llvm { /// Any client using this memory manager MUST ensure that section-specific /// page permissions have been applied before attempting to execute functions /// in the JITed object. Permissions can be applied either by calling -/// MCJIT::finalizeObject or by calling SectionMemoryManager::finalizeMemory +/// MCJIT::finalizeObject or by calling LlvmliteMemoryManager::finalizeMemory /// directly. Clients of MCJIT should call MCJIT::finalizeObject. -class SectionMemoryManager : public RTDyldMemoryManager { +class API_EXPORT(LlvmliteMemoryManager : public RTDyldMemoryManager) { public: /// This enum describes the various reasons to allocate pages from /// allocateMappedMemory. @@ -46,7 +46,7 @@ class SectionMemoryManager : public RTDyldMemoryManager { RWData, }; - /// Implementations of this interface are used by SectionMemoryManager to + /// Implementations of this interface are used by LlvmliteMemoryManager to /// request pages from the operating system. class MemoryMapper { public: @@ -101,13 +101,13 @@ class SectionMemoryManager : public RTDyldMemoryManager { virtual ~MemoryMapper(); }; - /// Creates a SectionMemoryManager instance with \p MM as the associated + /// Creates a LlvmliteMemoryManager instance with \p MM as the associated /// memory mapper. If \p MM is nullptr then a default memory mapper is used /// that directly calls into the operating system. - SectionMemoryManager(MemoryMapper *MM = nullptr); - SectionMemoryManager(const SectionMemoryManager &) = delete; - void operator=(const SectionMemoryManager &) = delete; - ~SectionMemoryManager() override; + LlvmliteMemoryManager(MemoryMapper *MM = nullptr); + LlvmliteMemoryManager(const LlvmliteMemoryManager &) = delete; + void operator=(const LlvmliteMemoryManager &) = delete; + ~LlvmliteMemoryManager() override; /// Allocates a memory block of (at least) the given size suitable for /// executable code. @@ -192,5 +192,3 @@ class SectionMemoryManager : public RTDyldMemoryManager { }; } // end namespace llvm - -#endif // LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H From 6aa56ebc52318dcae950e57e6b491c5ff386ba02 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Wed, 15 Nov 2023 14:14:25 +0000 Subject: [PATCH 4/8] Implement preallocation strategy in memory manager Based on changes submitted / discussed in: - https://github.com/llvm/llvm-project/pull/71968 - https://github.com/MikaelSmith/impala/blob/ac8561b6b69530f9fa2ff2ae65ec7415aa4395c6/be/src/codegen/mcjit-mem-mgr.cc - https://discourse.llvm.org/t/llvm-rtdyld-aarch64-abi-relocation-restrictions/74616 --- ffi/memorymanager.cpp | 120 ++++++++++++++++++++++++++++++++++-------- ffi/memorymanager.h | 9 ++++ 2 files changed, 107 insertions(+), 22 deletions(-) diff --git a/ffi/memorymanager.cpp b/ffi/memorymanager.cpp index 0db2fdd49..b472f3ecc 100644 --- a/ffi/memorymanager.cpp +++ b/ffi/memorymanager.cpp @@ -15,6 +15,8 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/Process.h" +#define DEBUG_TYPE "llvmlite-memory-manager" + namespace llvm { uint8_t *LlvmliteMemoryManager::allocateDataSection(uintptr_t Size, @@ -43,6 +45,10 @@ uint8_t *LlvmliteMemoryManager::allocateSection( if (!Alignment) Alignment = 16; + LLVM_DEBUG(dbgs() << "Requested size / alignment: " + << format_hex(Size, 2, true) << " / " << Alignment + << "\n"); + assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two."); @@ -50,13 +56,19 @@ uint8_t *LlvmliteMemoryManager::allocateSection( Alignment * ((Size + Alignment - 1) / Alignment + 1); uintptr_t Addr = 0; + LLVM_DEBUG(dbgs() << "Allocating " << format_hex(RequiredSize, 2, true) + << " bytes for "); + MemoryGroup &MemGroup = [&]() -> MemoryGroup & { switch (Purpose) { case AllocationPurpose::Code: + LLVM_DEBUG(dbgs() << "CodeMem at "); return CodeMem; case AllocationPurpose::ROData: + LLVM_DEBUG(dbgs() << "RODataMem at "); return RODataMem; case AllocationPurpose::RWData: + LLVM_DEBUG(dbgs() << "RWDataMem at "); return RWDataMem; } llvm_unreachable("Unknown LlvmliteMemoryManager::AllocationPurpose"); @@ -91,14 +103,68 @@ uint8_t *LlvmliteMemoryManager::allocateSection( // Remember how much free space is now left in this block FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); + LLVM_DEBUG(dbgs() << format_hex(Addr, 18, true) << "\n"); return (uint8_t *)Addr; } } - // No pre-allocated free block was large enough. Allocate a new memory - // region. Note that all sections get allocated as read-write. The - // permissions will be updated later based on memory group. - // + assert(false && "All memory must be pre-allocated"); + + return nullptr; +} + +static uintptr_t requiredPageSize(uintptr_t Size, uint32_t Alignment) { + static const size_t PageSize = sys::Process::getPageSizeEstimate(); + // Use the same calculation as allocateSection because we need to be able to + // satisfy it. + uintptr_t RequiredSize = + Alignment * ((Size + Alignment - 1) / Alignment + 1); + // Round up to the nearest page size. Blocks must be page-aligned. + return PageSize * ((RequiredSize + PageSize - 1) / PageSize); +} + +void LlvmliteMemoryManager::reserveAllocationSpace( + uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize, + uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) { + MemoryGroup &MemGroup = CodeMem; + + // MemoryManager does not have functions for releasing memory after it's + // allocated. Normally it tries to use any excess blocks that were + // allocated due to page alignment, but if we have insufficient free memory + // for the request this can lead to allocating disparate memory that can + // violate the ARM ABI. Clear free memory so only the new allocations are + // used, but do not release allocated memory as it may still be in-use. + CodeMem.FreeMem.clear(); + RODataMem.FreeMem.clear(); + RWDataMem.FreeMem.clear(); + + LLVM_DEBUG(dbgs() << "Code size / align: " << format_hex(CodeSize, 2, true) + << " / " << CodeAlign << "\n"); + LLVM_DEBUG(dbgs() << "ROData size / align: " + << format_hex(RODataSize, 2, true) << " / " << RODataAlign + << "\n"); + LLVM_DEBUG(dbgs() << "RWData size / align: " + << format_hex(RWDataSize, 2, true) << " / " << RWDataAlign + << "\n"); + + // Code alignment needs to be at least the stub alignment - however, we + // don't have an easy way to get that here so as a workaround, we assume + // it's 8, which is the largest value I observed across all platforms. + CodeAlign = CodeAlign ? CodeAlign : 16; + uint32_t StubAlign = 8; + CodeAlign = std::max(CodeAlign, StubAlign); + + RODataAlign = RODataAlign ? RODataAlign : 16; + RWDataAlign = RWDataAlign ? RWDataAlign : 16; + uintptr_t RequiredCodeSize = requiredPageSize(CodeSize, CodeAlign); + uintptr_t RequiredRODataSize = requiredPageSize(RODataSize, RODataAlign); + uintptr_t RequiredRWDataSize = requiredPageSize(RWDataSize, RWDataAlign); + uint64_t TotalSize = + RequiredCodeSize + RequiredRODataSize + RequiredRWDataSize; + + LLVM_DEBUG(dbgs() << "Reserving " << format_hex(TotalSize, 2, true) + << " bytes\n"); + // FIXME: It would be useful to define a default allocation size (or add // it as a constructor parameter) to minimize the number of allocations. // @@ -106,11 +172,11 @@ uint8_t *LlvmliteMemoryManager::allocateSection( // interleaving. std::error_code ec; sys::MemoryBlock MB = MMapper.allocateMappedMemory( - Purpose, RequiredSize, &MemGroup.Near, + AllocationPurpose::Code, TotalSize, &MemGroup.Near, sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); if (ec) { // FIXME: Add error propagation to the interface. - return nullptr; + assert(false && "Failed to allocate mapped memory"); } // Save this address as the basis for our next request @@ -127,27 +193,37 @@ uint8_t *LlvmliteMemoryManager::allocateSection( // Remember that we allocated this memory MemGroup.AllocatedMem.push_back(MB); - Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.allocatedSize(); - // Align the address. - Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + uintptr_t Addr = (uintptr_t)MB.base(); - // The part of the block we're giving out to the user is now pending - MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + FreeMemBlock FreeMB; + FreeMB.PendingPrefixIndex = (unsigned)-1; - // The allocateMappedMemory may allocate much more memory than we need. In - // this case, we store the unused memory as a free memory block. - unsigned FreeSize = EndOfBlock - Addr - Size; - if (FreeSize > 16) { - FreeMemBlock FreeMB; - FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize); - FreeMB.PendingPrefixIndex = (unsigned)-1; - MemGroup.FreeMem.push_back(FreeMB); + if (CodeSize > 0) { + LLVM_DEBUG(dbgs() << "Code mem starts at " << format_hex(Addr, 18, true) + << ", size " << format_hex(RequiredCodeSize, 2, true) + << "\n"); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredCodeSize); + CodeMem.FreeMem.push_back(FreeMB); + Addr += RequiredCodeSize; + } + + if (RODataSize > 0) { + LLVM_DEBUG(dbgs() << "Rodata mem starts at 0x" + << format_hex(Addr, 18, true) << ", size " + << format_hex(RequiredRODataSize, 2, true) << "\n"); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRODataSize); + RODataMem.FreeMem.push_back(FreeMB); + Addr += RequiredRODataSize; } - // Return aligned address - return (uint8_t *)Addr; + if (RWDataSize > 0) { + LLVM_DEBUG(dbgs() << "Rwdata mem starts at 0x" + << format_hex(Addr, 18, true) << ", size " + << format_hex(RequiredRWDataSize, 2, true) << "\n"); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRWDataSize); + RWDataMem.FreeMem.push_back(FreeMB); + } } bool LlvmliteMemoryManager::finalizeMemory(std::string *ErrMsg) { diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h index ed00a2cee..f8e44ef89 100644 --- a/ffi/memorymanager.h +++ b/ffi/memorymanager.h @@ -16,6 +16,7 @@ #include "core.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Memory.h" #include #include @@ -151,6 +152,14 @@ class API_EXPORT(LlvmliteMemoryManager : public RTDyldMemoryManager) { /// This method is called from finalizeMemory. virtual void invalidateInstructionCache(); + virtual bool needsToReserveAllocationSpace() override { return true; } + + virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, + uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override; + private: struct FreeMemBlock { // The actual block of free memory From 75b103cd44a4fab2439bd1e56a8fddea8cbd5031 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Wed, 22 Nov 2023 13:06:37 +0000 Subject: [PATCH 5/8] Memory manager: align changes with upstream The implementation of `reserveAllocationSpace()` now more closely follows that in https://github.com/llvm/llvm-project/pull/71968, following some changes made there. The changes here include: - Improved readability of debugging output - Using a default alignment of 8 in `allocateSection()` to match the default alignment provided by the stub alignment during preallocation. - Replacing the "bespoke" `requiredPageSize()` function with computations using the LLVM `alignTo()` function. - Returning early from preallocation when no space is requested. - Reusing existing preallocations if there is enough space left over from the previous preallocation for all the required segments - this can happen quite frequently because allocations for each segment get rounded up to page sizes, which are usually either 4K or 16K, and many Numba-jitted functions require a lot less than this. - Removal of setting the near hints for memory blocks - this doesn't really have any use when all memory is preallocated, and forced to be "near" to other memory. - Addition of extra asserts to validate alignment of allocated sections. --- ffi/memorymanager.cpp | 130 +++++++++++++++++++++++++----------------- ffi/memorymanager.h | 2 + 2 files changed, 79 insertions(+), 53 deletions(-) diff --git a/ffi/memorymanager.cpp b/ffi/memorymanager.cpp index b472f3ecc..3163e2430 100644 --- a/ffi/memorymanager.cpp +++ b/ffi/memorymanager.cpp @@ -42,13 +42,17 @@ uint8_t *LlvmliteMemoryManager::allocateCodeSection(uintptr_t Size, uint8_t *LlvmliteMemoryManager::allocateSection( LlvmliteMemoryManager::AllocationPurpose Purpose, uintptr_t Size, unsigned Alignment) { - if (!Alignment) - Alignment = 16; + LLVM_DEBUG( + dbgs() << "\nLlvmliteMemoryManager::allocateSection() request:\n"); LLVM_DEBUG(dbgs() << "Requested size / alignment: " << format_hex(Size, 2, true) << " / " << Alignment << "\n"); + // Chosen to match the stub alignment value used in reserveAllocationSpace() + if (!Alignment) + Alignment = 8; + assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two."); @@ -110,34 +114,27 @@ uint8_t *LlvmliteMemoryManager::allocateSection( assert(false && "All memory must be pre-allocated"); + // If asserts are turned off, returning a null pointer in the event of a + // failure to find a preallocated block large enough should at least lead + // to a quick crash. return nullptr; } -static uintptr_t requiredPageSize(uintptr_t Size, uint32_t Alignment) { - static const size_t PageSize = sys::Process::getPageSizeEstimate(); - // Use the same calculation as allocateSection because we need to be able to - // satisfy it. - uintptr_t RequiredSize = - Alignment * ((Size + Alignment - 1) / Alignment + 1); - // Round up to the nearest page size. Blocks must be page-aligned. - return PageSize * ((RequiredSize + PageSize - 1) / PageSize); +bool LlvmliteMemoryManager::hasSpace(const MemoryGroup &MemGroup, + uintptr_t Size) const { + for (const FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.allocatedSize() >= Size) + return true; + } + return false; } void LlvmliteMemoryManager::reserveAllocationSpace( uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize, uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) { - MemoryGroup &MemGroup = CodeMem; - - // MemoryManager does not have functions for releasing memory after it's - // allocated. Normally it tries to use any excess blocks that were - // allocated due to page alignment, but if we have insufficient free memory - // for the request this can lead to allocating disparate memory that can - // violate the ARM ABI. Clear free memory so only the new allocations are - // used, but do not release allocated memory as it may still be in-use. - CodeMem.FreeMem.clear(); - RODataMem.FreeMem.clear(); - RWDataMem.FreeMem.clear(); - + LLVM_DEBUG( + dbgs() + << "\nLlvmliteMemoryManager::reserveAllocationSpace() request:\n\n"); LLVM_DEBUG(dbgs() << "Code size / align: " << format_hex(CodeSize, 2, true) << " / " << CodeAlign << "\n"); LLVM_DEBUG(dbgs() << "ROData size / align: " @@ -147,55 +144,77 @@ void LlvmliteMemoryManager::reserveAllocationSpace( << format_hex(RWDataSize, 2, true) << " / " << RWDataAlign << "\n"); + if (CodeSize == 0 && RODataSize == 0 && RWDataSize == 0) { + LLVM_DEBUG(dbgs() << "No memory requested - returning early.\n"); + return; + } + // Code alignment needs to be at least the stub alignment - however, we // don't have an easy way to get that here so as a workaround, we assume // it's 8, which is the largest value I observed across all platforms. - CodeAlign = CodeAlign ? CodeAlign : 16; - uint32_t StubAlign = 8; + constexpr uint32_t StubAlign = 8; CodeAlign = std::max(CodeAlign, StubAlign); - RODataAlign = RODataAlign ? RODataAlign : 16; - RWDataAlign = RWDataAlign ? RWDataAlign : 16; - uintptr_t RequiredCodeSize = requiredPageSize(CodeSize, CodeAlign); - uintptr_t RequiredRODataSize = requiredPageSize(RODataSize, RODataAlign); - uintptr_t RequiredRWDataSize = requiredPageSize(RWDataSize, RWDataAlign); + // ROData and RWData may not need to be aligned to the StubAlign, but the + // stub alignment seems like a reasonable (if slightly arbitrary) minimum + // alignment for them that should not cause any issues on all (i.e. 64-bit) + // platforms. + RODataAlign = std::max(RODataAlign, StubAlign); + RWDataAlign = std::max(RWDataAlign, StubAlign); + + // Get space required for each section. Use the same calculation as + // allocateSection because we need to be able to satisfy it. + uintptr_t RequiredCodeSize = alignTo(CodeSize, CodeAlign) + CodeAlign; + uintptr_t RequiredRODataSize = + alignTo(RODataSize, RODataAlign) + RODataAlign; + uintptr_t RequiredRWDataSize = + alignTo(RWDataSize, RWDataAlign) + RWDataAlign; uint64_t TotalSize = RequiredCodeSize + RequiredRODataSize + RequiredRWDataSize; + if (hasSpace(CodeMem, RequiredCodeSize) && + hasSpace(RODataMem, RequiredRODataSize) && + hasSpace(RWDataMem, RequiredRWDataSize)) { + // Sufficient space in contiguous block already available. + LLVM_DEBUG( + dbgs() << "Previous preallocation sufficient; reusing it.\n"); + return; + } + + // MemoryManager does not have functions for releasing memory after it's + // allocated. Normally it tries to use any excess blocks that were + // allocated due to page alignment, but if we have insufficient free memory + // for the request this can lead to allocating disparate memory that can + // violate the ARM ABI. Clear free memory so only the new allocations are + // used, but do not release allocated memory as it may still be in-use. + CodeMem.FreeMem.clear(); + RODataMem.FreeMem.clear(); + RWDataMem.FreeMem.clear(); + + // Round up to the nearest page size. Blocks must be page-aligned. + static const size_t PageSize = sys::Process::getPageSizeEstimate(); + RequiredCodeSize = alignTo(RequiredCodeSize, PageSize); + RequiredRODataSize = alignTo(RequiredRODataSize, PageSize); + RequiredRWDataSize = alignTo(RequiredRWDataSize, PageSize); + uintptr_t RequiredSize = + RequiredCodeSize + RequiredRODataSize + RequiredRWDataSize; + LLVM_DEBUG(dbgs() << "Reserving " << format_hex(TotalSize, 2, true) << " bytes\n"); - // FIXME: It would be useful to define a default allocation size (or add - // it as a constructor parameter) to minimize the number of allocations. - // - // FIXME: Initialize the Near member for each memory group to avoid - // interleaving. std::error_code ec; + const sys::MemoryBlock *near = nullptr; sys::MemoryBlock MB = MMapper.allocateMappedMemory( - AllocationPurpose::Code, TotalSize, &MemGroup.Near, + AllocationPurpose::RWData, RequiredSize, near, sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); if (ec) { - // FIXME: Add error propagation to the interface. assert(false && "Failed to allocate mapped memory"); } - // Save this address as the basis for our next request - MemGroup.Near = MB; - - // Copy the address to all the other groups, if they have not - // been initialized. - if (CodeMem.Near.base() == nullptr) - CodeMem.Near = MB; - if (RODataMem.Near.base() == nullptr) - RODataMem.Near = MB; - if (RWDataMem.Near.base() == nullptr) - RWDataMem.Near = MB; - - // Remember that we allocated this memory - MemGroup.AllocatedMem.push_back(MB); + // CodeMem will arbitrarily own this MemoryBlock to handle cleanup. + CodeMem.AllocatedMem.push_back(MB); uintptr_t Addr = (uintptr_t)MB.base(); - FreeMemBlock FreeMB; FreeMB.PendingPrefixIndex = (unsigned)-1; @@ -203,27 +222,32 @@ void LlvmliteMemoryManager::reserveAllocationSpace( LLVM_DEBUG(dbgs() << "Code mem starts at " << format_hex(Addr, 18, true) << ", size " << format_hex(RequiredCodeSize, 2, true) << "\n"); + assert(isAddrAligned(Align(CodeAlign), (void *)Addr)); FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredCodeSize); CodeMem.FreeMem.push_back(FreeMB); Addr += RequiredCodeSize; } if (RODataSize > 0) { - LLVM_DEBUG(dbgs() << "Rodata mem starts at 0x" + LLVM_DEBUG(dbgs() << "ROData mem starts at " << format_hex(Addr, 18, true) << ", size " << format_hex(RequiredRODataSize, 2, true) << "\n"); + assert(isAddrAligned(Align(RODataAlign), (void *)Addr)); FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRODataSize); RODataMem.FreeMem.push_back(FreeMB); Addr += RequiredRODataSize; } if (RWDataSize > 0) { - LLVM_DEBUG(dbgs() << "Rwdata mem starts at 0x" + LLVM_DEBUG(dbgs() << "RWData mem starts at " << format_hex(Addr, 18, true) << ", size " << format_hex(RequiredRWDataSize, 2, true) << "\n"); + assert(isAddrAligned(Align(RWDataAlign), (void *)Addr)); FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRWDataSize); RWDataMem.FreeMem.push_back(FreeMB); } + + LLVM_DEBUG(dbgs() << "\n"); } bool LlvmliteMemoryManager::finalizeMemory(std::string *ErrMsg) { diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h index f8e44ef89..e5162490c 100644 --- a/ffi/memorymanager.h +++ b/ffi/memorymanager.h @@ -192,6 +192,8 @@ class API_EXPORT(LlvmliteMemoryManager : public RTDyldMemoryManager) { std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions); + bool hasSpace(const MemoryGroup &MemGroup, uintptr_t Size) const; + void anchor() override; MemoryGroup CodeMem; From 5e92f877bbd7d0f358979f5940784678935fe6a4 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Wed, 6 Dec 2023 12:24:41 +0000 Subject: [PATCH 6/8] Conditionally enable the llvmlite memory manager The default is to enable it on 64-bit ARM systems, since it solves the problem they encounter, and disable it elsewhere, to minimise the risk of an unintended side effect on platforms that don't need it. This can be overridden by manually specifying the value of `use_lmm` when creating the MCJIT compiler. --- docs/source/user-guide/binding/execution-engine.rst | 6 +++++- ffi/executionengine.cpp | 13 ++++++++----- llvmlite/binding/executionengine.py | 13 +++++++++++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/source/user-guide/binding/execution-engine.rst b/docs/source/user-guide/binding/execution-engine.rst index 8b3c63b12..f913783eb 100644 --- a/docs/source/user-guide/binding/execution-engine.rst +++ b/docs/source/user-guide/binding/execution-engine.rst @@ -12,11 +12,15 @@ present a single execution engine, ``MCJIT``, is exposed. Functions ========= -* .. function:: create_mcjit_compiler(module, target_machine) +* .. function:: create_mcjit_compiler(module, target_machine, use_lmm=None) Create a MCJIT-powered engine from the given *module* and *target_machine*. + *lmm* controls whether the llvmlite memory manager is used. If not + supplied, the default choice for the platform will be used (``True`` on + 64-bit ARM systems, ``False`` otherwise). + * *module* does not need to contain any code. * Returns a :class:`ExecutionEngine` instance. diff --git a/ffi/executionengine.cpp b/ffi/executionengine.cpp index 54a6f9bb2..f3fcd6467 100644 --- a/ffi/executionengine.cpp +++ b/ffi/executionengine.cpp @@ -66,6 +66,7 @@ LLVMPY_FinalizeObject(LLVMExecutionEngineRef EE) { static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, LLVMTargetMachineRef TM, + bool use_lmm, const char **OutError) { LLVMExecutionEngineRef ee = nullptr; @@ -74,9 +75,11 @@ static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, eb.setErrorStr(&err); eb.setEngineKind(llvm::EngineKind::JIT); - std::unique_ptr mm = - std::make_unique(); - eb.setMCJITMemoryManager(std::move(mm)); + if (use_lmm) { + std::unique_ptr mm = + std::make_unique(); + eb.setMCJITMemoryManager(std::move(mm)); + } /* EngineBuilder::create loads the current process symbols */ llvm::ExecutionEngine *engine = eb.create(llvm::unwrap(TM)); @@ -90,8 +93,8 @@ static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, API_EXPORT(LLVMExecutionEngineRef) LLVMPY_CreateMCJITCompiler(LLVMModuleRef M, LLVMTargetMachineRef TM, - const char **OutError) { - return create_execution_engine(M, TM, OutError); + bool use_lmm, const char **OutError) { + return create_execution_engine(M, TM, use_lmm, OutError); } API_EXPORT(uint64_t) diff --git a/llvmlite/binding/executionengine.py b/llvmlite/binding/executionengine.py index 07cb8dab0..7b7c37236 100644 --- a/llvmlite/binding/executionengine.py +++ b/llvmlite/binding/executionengine.py @@ -1,3 +1,4 @@ +import platform from ctypes import (POINTER, c_char_p, c_bool, c_void_p, c_int, c_uint64, c_size_t, CFUNCTYPE, string_at, cast, py_object, Structure) @@ -9,14 +10,21 @@ ffi.lib.LLVMPY_LinkInMCJIT -def create_mcjit_compiler(module, target_machine): +def create_mcjit_compiler(module, target_machine, use_lmm=None): """ Create a MCJIT ExecutionEngine from the given *module* and *target_machine*. + + *lmm* controls whether the llvmlite memory manager is used. If not supplied, + the default choice for the platform will be used (``True`` on 64-bit ARM + systems, ``False`` otherwise). """ + if use_lmm is None: + use_lmm = platform.machine() in ('arm64', 'aarch64') + with ffi.OutputString() as outerr: engine = ffi.lib.LLVMPY_CreateMCJITCompiler( - module, target_machine, outerr) + module, target_machine, use_lmm, outerr) if not engine: raise RuntimeError(str(outerr)) @@ -239,6 +247,7 @@ def _dispose(self): ffi.lib.LLVMPY_CreateMCJITCompiler.argtypes = [ ffi.LLVMModuleRef, ffi.LLVMTargetMachineRef, + c_bool, POINTER(c_char_p), ] ffi.lib.LLVMPY_CreateMCJITCompiler.restype = ffi.LLVMExecutionEngineRef From b673be6c4baacfafa802e382fbe5665e62196dd4 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Wed, 6 Dec 2023 16:50:47 +0000 Subject: [PATCH 7/8] Add LLVM license --- LICENSE.thirdparty | 225 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 LICENSE.thirdparty diff --git a/LICENSE.thirdparty b/LICENSE.thirdparty new file mode 100644 index 000000000..d1ea17c25 --- /dev/null +++ b/LICENSE.thirdparty @@ -0,0 +1,225 @@ +The llvmlite source tree includes code from LLVM that is governed by the +following license. + +============================================================================== +The Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. From 4ac8e2216a4ff19fadbd171da2a79635cd4a76d9 Mon Sep 17 00:00:00 2001 From: Graham Markall Date: Thu, 7 Dec 2023 13:51:40 +0000 Subject: [PATCH 8/8] Explicitly force default visibility for llvm::ErrorInfoBase --- ffi/memorymanager.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h index e5162490c..9ed028542 100644 --- a/ffi/memorymanager.h +++ b/ffi/memorymanager.h @@ -13,6 +13,26 @@ #pragma once +// Force default visibility of the llvm::ErrorInfoBase class. The conda +// compilers use the -fvisibility-inlines-hidden flag, which seems to +// erroneously result in ErrorInfoBase::isA() being hidden (and not exported) on +// PowerPC. The reason for this is not conclusively known, but the circumstances +// appear similar to those reported in GCC Bug 45065 +// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=45066) - ErrorInfoBase has +// both a template and non-template version, and the non-template version is +// overridden by the derived class ErrorInfo; the template vs. non-template +// versions may have different inlining decisions applied, and this could create +// a similar circumstance to that described in the bug. +// +// The workaround here adds the default visiblity attribute to ErrorInfoBase +// before its definition, which precludes it from being inferred to be hidden +// later on. +#if not defined(_MSC_VER) +namespace llvm { +class __attribute__((visibility("default"))) ErrorInfoBase; +} +#endif + #include "core.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h"