diff --git a/LICENSE.thirdparty b/LICENSE.thirdparty new file mode 100644 index 000000000..d1ea17c25 --- /dev/null +++ b/LICENSE.thirdparty @@ -0,0 +1,225 @@ +The llvmlite source tree includes code from LLVM that is governed by the +following license. + +============================================================================== +The Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. diff --git a/docs/source/user-guide/binding/execution-engine.rst b/docs/source/user-guide/binding/execution-engine.rst index 8b3c63b12..f913783eb 100644 --- a/docs/source/user-guide/binding/execution-engine.rst +++ b/docs/source/user-guide/binding/execution-engine.rst @@ -12,11 +12,15 @@ present a single execution engine, ``MCJIT``, is exposed. Functions ========= -* .. function:: create_mcjit_compiler(module, target_machine) +* .. function:: create_mcjit_compiler(module, target_machine, use_lmm=None) Create a MCJIT-powered engine from the given *module* and *target_machine*. + *lmm* controls whether the llvmlite memory manager is used. If not + supplied, the default choice for the platform will be used (``True`` on + 64-bit ARM systems, ``False`` otherwise). + * *module* does not need to contain any code. * Returns a :class:`ExecutionEngine` instance. diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index e4ce2c12c..3aab3927c 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -40,7 +40,7 @@ endif() add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp linker.cpp object_file.cpp - custom_passes.cpp orcjit.cpp) + custom_passes.cpp orcjit.cpp memorymanager.cpp) # Find the libraries that correspond to the LLVM components # that we wish to use. diff --git a/ffi/Makefile.freebsd b/ffi/Makefile.freebsd index a55349a34..150cbb285 100644 --- a/ffi/Makefile.freebsd +++ b/ffi/Makefile.freebsd @@ -11,7 +11,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp orcjit.cpp custom_passes.cpp + linker.cpp object_file.cpp orcjit.cpp custom_passes.cpp memorymanager.cpp OUTPUT = libllvmlite.so all: $(OUTPUT) diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index c1df42dbb..c101510f2 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -13,7 +13,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h OBJ = assembly.o bitcode.o core.o initfini.o module.o value.o \ executionengine.o transforms.o passmanagers.o targets.o dylib.o \ - linker.o object_file.o custom_passes.o orcjit.o + linker.o object_file.o custom_passes.o orcjit.o memorymanager.o OUTPUT = libllvmlite.so all: $(OUTPUT) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index a0d06db4f..9668dd356 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -8,7 +8,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp custom_passes.cpp orcjit.cpp + linker.cpp object_file.cpp custom_passes.cpp orcjit.cpp memorymanager.cpp OUTPUT = libllvmlite.dylib MACOSX_DEPLOYMENT_TARGET ?= 10.9 diff --git a/ffi/executionengine.cpp b/ffi/executionengine.cpp index 79f1ec9a3..f3fcd6467 100644 --- a/ffi/executionengine.cpp +++ b/ffi/executionengine.cpp @@ -1,4 +1,5 @@ #include "core.h" +#include "memorymanager.h" #include "llvm-c/ExecutionEngine.h" #include "llvm-c/Object.h" @@ -65,6 +66,7 @@ LLVMPY_FinalizeObject(LLVMExecutionEngineRef EE) { static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, LLVMTargetMachineRef TM, + bool use_lmm, const char **OutError) { LLVMExecutionEngineRef ee = nullptr; @@ -73,6 +75,12 @@ static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, eb.setErrorStr(&err); eb.setEngineKind(llvm::EngineKind::JIT); + if (use_lmm) { + std::unique_ptr mm = + std::make_unique(); + eb.setMCJITMemoryManager(std::move(mm)); + } + /* EngineBuilder::create loads the current process symbols */ llvm::ExecutionEngine *engine = eb.create(llvm::unwrap(TM)); @@ -85,8 +93,8 @@ static LLVMExecutionEngineRef create_execution_engine(LLVMModuleRef M, API_EXPORT(LLVMExecutionEngineRef) LLVMPY_CreateMCJITCompiler(LLVMModuleRef M, LLVMTargetMachineRef TM, - const char **OutError) { - return create_execution_engine(M, TM, OutError); + bool use_lmm, const char **OutError) { + return create_execution_engine(M, TM, use_lmm, OutError); } API_EXPORT(uint64_t) diff --git a/ffi/memorymanager.cpp b/ffi/memorymanager.cpp new file mode 100644 index 000000000..3163e2430 --- /dev/null +++ b/ffi/memorymanager.cpp @@ -0,0 +1,379 @@ +//===---- memorymanager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the section-based memory manager used by the MCJIT +// execution engine and RuntimeDyld +// +//===----------------------------------------------------------------------===// + +#include "memorymanager.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Process.h" + +#define DEBUG_TYPE "llvmlite-memory-manager" + +namespace llvm { + +uint8_t *LlvmliteMemoryManager::allocateDataSection(uintptr_t Size, + unsigned Alignment, + unsigned SectionID, + StringRef SectionName, + bool IsReadOnly) { + if (IsReadOnly) + return allocateSection(LlvmliteMemoryManager::AllocationPurpose::ROData, + Size, Alignment); + return allocateSection(LlvmliteMemoryManager::AllocationPurpose::RWData, + Size, Alignment); +} + +uint8_t *LlvmliteMemoryManager::allocateCodeSection(uintptr_t Size, + unsigned Alignment, + unsigned SectionID, + StringRef SectionName) { + return allocateSection(LlvmliteMemoryManager::AllocationPurpose::Code, Size, + Alignment); +} + +uint8_t *LlvmliteMemoryManager::allocateSection( + LlvmliteMemoryManager::AllocationPurpose Purpose, uintptr_t Size, + unsigned Alignment) { + LLVM_DEBUG( + dbgs() << "\nLlvmliteMemoryManager::allocateSection() request:\n"); + + LLVM_DEBUG(dbgs() << "Requested size / alignment: " + << format_hex(Size, 2, true) << " / " << Alignment + << "\n"); + + // Chosen to match the stub alignment value used in reserveAllocationSpace() + if (!Alignment) + Alignment = 8; + + assert(!(Alignment & (Alignment - 1)) && + "Alignment must be a power of two."); + + uintptr_t RequiredSize = + Alignment * ((Size + Alignment - 1) / Alignment + 1); + uintptr_t Addr = 0; + + LLVM_DEBUG(dbgs() << "Allocating " << format_hex(RequiredSize, 2, true) + << " bytes for "); + + MemoryGroup &MemGroup = [&]() -> MemoryGroup & { + switch (Purpose) { + case AllocationPurpose::Code: + LLVM_DEBUG(dbgs() << "CodeMem at "); + return CodeMem; + case AllocationPurpose::ROData: + LLVM_DEBUG(dbgs() << "RODataMem at "); + return RODataMem; + case AllocationPurpose::RWData: + LLVM_DEBUG(dbgs() << "RWDataMem at "); + return RWDataMem; + } + llvm_unreachable("Unknown LlvmliteMemoryManager::AllocationPurpose"); + }(); + + // Look in the list of free memory regions and use a block there if one + // is available. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.allocatedSize() >= RequiredSize) { + Addr = (uintptr_t)FreeMB.Free.base(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); + // Align the address. + Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + + if (FreeMB.PendingPrefixIndex == (unsigned)-1) { + // The part of the block we're giving out to the user is now + // pending + MemGroup.PendingMem.push_back( + sys::MemoryBlock((void *)Addr, Size)); + + // Remember this pending block, such that future allocations can + // just modify it rather than creating a new one + FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; + } else { + sys::MemoryBlock &PendingMB = + MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; + PendingMB = + sys::MemoryBlock(PendingMB.base(), + Addr + Size - (uintptr_t)PendingMB.base()); + } + + // Remember how much free space is now left in this block + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), + EndOfBlock - Addr - Size); + LLVM_DEBUG(dbgs() << format_hex(Addr, 18, true) << "\n"); + return (uint8_t *)Addr; + } + } + + assert(false && "All memory must be pre-allocated"); + + // If asserts are turned off, returning a null pointer in the event of a + // failure to find a preallocated block large enough should at least lead + // to a quick crash. + return nullptr; +} + +bool LlvmliteMemoryManager::hasSpace(const MemoryGroup &MemGroup, + uintptr_t Size) const { + for (const FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.allocatedSize() >= Size) + return true; + } + return false; +} + +void LlvmliteMemoryManager::reserveAllocationSpace( + uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize, + uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) { + LLVM_DEBUG( + dbgs() + << "\nLlvmliteMemoryManager::reserveAllocationSpace() request:\n\n"); + LLVM_DEBUG(dbgs() << "Code size / align: " << format_hex(CodeSize, 2, true) + << " / " << CodeAlign << "\n"); + LLVM_DEBUG(dbgs() << "ROData size / align: " + << format_hex(RODataSize, 2, true) << " / " << RODataAlign + << "\n"); + LLVM_DEBUG(dbgs() << "RWData size / align: " + << format_hex(RWDataSize, 2, true) << " / " << RWDataAlign + << "\n"); + + if (CodeSize == 0 && RODataSize == 0 && RWDataSize == 0) { + LLVM_DEBUG(dbgs() << "No memory requested - returning early.\n"); + return; + } + + // Code alignment needs to be at least the stub alignment - however, we + // don't have an easy way to get that here so as a workaround, we assume + // it's 8, which is the largest value I observed across all platforms. + constexpr uint32_t StubAlign = 8; + CodeAlign = std::max(CodeAlign, StubAlign); + + // ROData and RWData may not need to be aligned to the StubAlign, but the + // stub alignment seems like a reasonable (if slightly arbitrary) minimum + // alignment for them that should not cause any issues on all (i.e. 64-bit) + // platforms. + RODataAlign = std::max(RODataAlign, StubAlign); + RWDataAlign = std::max(RWDataAlign, StubAlign); + + // Get space required for each section. Use the same calculation as + // allocateSection because we need to be able to satisfy it. + uintptr_t RequiredCodeSize = alignTo(CodeSize, CodeAlign) + CodeAlign; + uintptr_t RequiredRODataSize = + alignTo(RODataSize, RODataAlign) + RODataAlign; + uintptr_t RequiredRWDataSize = + alignTo(RWDataSize, RWDataAlign) + RWDataAlign; + uint64_t TotalSize = + RequiredCodeSize + RequiredRODataSize + RequiredRWDataSize; + + if (hasSpace(CodeMem, RequiredCodeSize) && + hasSpace(RODataMem, RequiredRODataSize) && + hasSpace(RWDataMem, RequiredRWDataSize)) { + // Sufficient space in contiguous block already available. + LLVM_DEBUG( + dbgs() << "Previous preallocation sufficient; reusing it.\n"); + return; + } + + // MemoryManager does not have functions for releasing memory after it's + // allocated. Normally it tries to use any excess blocks that were + // allocated due to page alignment, but if we have insufficient free memory + // for the request this can lead to allocating disparate memory that can + // violate the ARM ABI. Clear free memory so only the new allocations are + // used, but do not release allocated memory as it may still be in-use. + CodeMem.FreeMem.clear(); + RODataMem.FreeMem.clear(); + RWDataMem.FreeMem.clear(); + + // Round up to the nearest page size. Blocks must be page-aligned. + static const size_t PageSize = sys::Process::getPageSizeEstimate(); + RequiredCodeSize = alignTo(RequiredCodeSize, PageSize); + RequiredRODataSize = alignTo(RequiredRODataSize, PageSize); + RequiredRWDataSize = alignTo(RequiredRWDataSize, PageSize); + uintptr_t RequiredSize = + RequiredCodeSize + RequiredRODataSize + RequiredRWDataSize; + + LLVM_DEBUG(dbgs() << "Reserving " << format_hex(TotalSize, 2, true) + << " bytes\n"); + + std::error_code ec; + const sys::MemoryBlock *near = nullptr; + sys::MemoryBlock MB = MMapper.allocateMappedMemory( + AllocationPurpose::RWData, RequiredSize, near, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); + if (ec) { + assert(false && "Failed to allocate mapped memory"); + } + + // CodeMem will arbitrarily own this MemoryBlock to handle cleanup. + CodeMem.AllocatedMem.push_back(MB); + + uintptr_t Addr = (uintptr_t)MB.base(); + FreeMemBlock FreeMB; + FreeMB.PendingPrefixIndex = (unsigned)-1; + + if (CodeSize > 0) { + LLVM_DEBUG(dbgs() << "Code mem starts at " << format_hex(Addr, 18, true) + << ", size " << format_hex(RequiredCodeSize, 2, true) + << "\n"); + assert(isAddrAligned(Align(CodeAlign), (void *)Addr)); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredCodeSize); + CodeMem.FreeMem.push_back(FreeMB); + Addr += RequiredCodeSize; + } + + if (RODataSize > 0) { + LLVM_DEBUG(dbgs() << "ROData mem starts at " + << format_hex(Addr, 18, true) << ", size " + << format_hex(RequiredRODataSize, 2, true) << "\n"); + assert(isAddrAligned(Align(RODataAlign), (void *)Addr)); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRODataSize); + RODataMem.FreeMem.push_back(FreeMB); + Addr += RequiredRODataSize; + } + + if (RWDataSize > 0) { + LLVM_DEBUG(dbgs() << "RWData mem starts at " + << format_hex(Addr, 18, true) << ", size " + << format_hex(RequiredRWDataSize, 2, true) << "\n"); + assert(isAddrAligned(Align(RWDataAlign), (void *)Addr)); + FreeMB.Free = sys::MemoryBlock((void *)Addr, RequiredRWDataSize); + RWDataMem.FreeMem.push_back(FreeMB); + } + + LLVM_DEBUG(dbgs() << "\n"); +} + +bool LlvmliteMemoryManager::finalizeMemory(std::string *ErrMsg) { + // FIXME: Should in-progress permissions be reverted if an error occurs? + std::error_code ec; + + // Make code memory executable. + ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + if (ec) { + if (ErrMsg) { + *ErrMsg = ec.message(); + } + return true; + } + + // Make read-only data memory read-only. + ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ); + if (ec) { + if (ErrMsg) { + *ErrMsg = ec.message(); + } + return true; + } + + // Read-write data memory already has the correct permissions + + // Some platforms with separate data cache and instruction cache require + // explicit cache flush, otherwise JIT code manipulations (like resolved + // relocations) will get to the data cache but not to the instruction cache. + invalidateInstructionCache(); + + return false; +} + +static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { + static const size_t PageSize = sys::Process::getPageSizeEstimate(); + + size_t StartOverlap = + (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; + + size_t TrimmedSize = M.allocatedSize(); + TrimmedSize -= StartOverlap; + TrimmedSize -= TrimmedSize % PageSize; + + sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), + TrimmedSize); + + assert(((uintptr_t)Trimmed.base() % PageSize) == 0); + assert((Trimmed.allocatedSize() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && + Trimmed.allocatedSize() <= M.allocatedSize()); + + return Trimmed; +} + +std::error_code +LlvmliteMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions) { + for (sys::MemoryBlock &MB : MemGroup.PendingMem) + if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions)) + return EC; + + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the + // entire page because one of the pending blocks may have overlapped it. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + FreeMB.Free = trimBlockToPageSize(FreeMB.Free); + // We cleared the PendingMem list, so all these pointers are now invalid + FreeMB.PendingPrefixIndex = (unsigned)-1; + } + + // Remove all blocks which are now empty + erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) { + return FreeMB.Free.allocatedSize() == 0; + }); + + return std::error_code(); +} + +void LlvmliteMemoryManager::invalidateInstructionCache() { + for (sys::MemoryBlock &Block : CodeMem.PendingMem) + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); +} + +LlvmliteMemoryManager::~LlvmliteMemoryManager() { + for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { + for (sys::MemoryBlock &Block : Group->AllocatedMem) + MMapper.releaseMappedMemory(Block); + } +} + +LlvmliteMemoryManager::MemoryMapper::~MemoryMapper() {} + +void LlvmliteMemoryManager::anchor() {} + +namespace { +// Trivial implementation of LlvmliteMemoryManager::MemoryMapper that just calls +// into sys::Memory. +class DefaultMMapper final : public LlvmliteMemoryManager::MemoryMapper { + public: + sys::MemoryBlock + allocateMappedMemory(LlvmliteMemoryManager::AllocationPurpose Purpose, + size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) override { + return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, + EC); + } + + std::error_code protectMappedMemory(const sys::MemoryBlock &Block, + unsigned Flags) override { + return sys::Memory::protectMappedMemory(Block, Flags); + } + + std::error_code releaseMappedMemory(sys::MemoryBlock &M) override { + return sys::Memory::releaseMappedMemory(M); + } +}; + +DefaultMMapper DefaultMMapperInstance; +} // namespace + +LlvmliteMemoryManager::LlvmliteMemoryManager(MemoryMapper *MM) + : MMapper(MM ? *MM : DefaultMMapperInstance) {} + +} // namespace llvm diff --git a/ffi/memorymanager.h b/ffi/memorymanager.h new file mode 100644 index 000000000..9ed028542 --- /dev/null +++ b/ffi/memorymanager.h @@ -0,0 +1,225 @@ +//===----- memorymanager.h - Memory manager for MCJIT/RtDyld -*- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the declaration of a section-based memory manager used by +// the MCJIT execution engine and RuntimeDyld. +// +//===----------------------------------------------------------------------===// + +#pragma once + +// Force default visibility of the llvm::ErrorInfoBase class. The conda +// compilers use the -fvisibility-inlines-hidden flag, which seems to +// erroneously result in ErrorInfoBase::isA() being hidden (and not exported) on +// PowerPC. The reason for this is not conclusively known, but the circumstances +// appear similar to those reported in GCC Bug 45065 +// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=45066) - ErrorInfoBase has +// both a template and non-template version, and the non-template version is +// overridden by the derived class ErrorInfo; the template vs. non-template +// versions may have different inlining decisions applied, and this could create +// a similar circumstance to that described in the bug. +// +// The workaround here adds the default visiblity attribute to ErrorInfoBase +// before its definition, which precludes it from being inferred to be hidden +// later on. +#if not defined(_MSC_VER) +namespace llvm { +class __attribute__((visibility("default"))) ErrorInfoBase; +} +#endif + +#include "core.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/Memory.h" +#include +#include +#include + +namespace llvm { + +/// This is a simple memory manager which implements the methods called by +/// the RuntimeDyld class to allocate memory for section-based loading of +/// objects, usually those generated by the MCJIT execution engine. +/// +/// This memory manager allocates all section memory as read-write. The +/// RuntimeDyld will copy JITed section memory into these allocated blocks +/// and perform any necessary linking and relocations. +/// +/// Any client using this memory manager MUST ensure that section-specific +/// page permissions have been applied before attempting to execute functions +/// in the JITed object. Permissions can be applied either by calling +/// MCJIT::finalizeObject or by calling LlvmliteMemoryManager::finalizeMemory +/// directly. Clients of MCJIT should call MCJIT::finalizeObject. +class API_EXPORT(LlvmliteMemoryManager : public RTDyldMemoryManager) { + public: + /// This enum describes the various reasons to allocate pages from + /// allocateMappedMemory. + enum class AllocationPurpose { + Code, + ROData, + RWData, + }; + + /// Implementations of this interface are used by LlvmliteMemoryManager to + /// request pages from the operating system. + class MemoryMapper { + public: + /// This method attempts to allocate \p NumBytes bytes of virtual memory + /// for \p Purpose. \p NearBlock may point to an existing allocation, + /// in which case an attempt is made to allocate more memory near the + /// existing block. The actual allocated address is not guaranteed to be + /// near the requested address. \p Flags is used to set the initial + /// protection flags for the block of the memory. \p EC [out] returns + /// an object describing any error that occurs. + /// + /// This method may allocate more than the number of bytes requested. + /// The actual number of bytes allocated is indicated in the returned + /// MemoryBlock. + /// + /// The start of the allocated block must be aligned with the system + /// allocation granularity (64K on Windows, page size on Linux). If the + /// address following \p NearBlock is not so aligned, it will be rounded + /// up to the next allocation granularity boundary. + /// + /// \r a non-null MemoryBlock if the function was successful, otherwise + /// a null MemoryBlock with \p EC describing the error. + virtual sys::MemoryBlock + allocateMappedMemory(AllocationPurpose Purpose, size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) = 0; + + /// This method sets the protection flags for a block of memory to the + /// state specified by \p Flags. The behavior is not specified if the + /// memory was not allocated using the allocateMappedMemory method. \p + /// Block describes the memory block to be protected. \p Flags specifies + /// the new protection state to be assigned to the block. + /// + /// If \p Flags is MF_WRITE, the actual behavior varies with the + /// operating system (i.e. MF_READ | MF_WRITE on Windows) and the target + /// architecture (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code + protectMappedMemory(const sys::MemoryBlock &Block, unsigned Flags) = 0; + + /// This method releases a block of memory that was allocated with the + /// allocateMappedMemory method. It should not be used to release any + /// memory block allocated any other way. \p Block describes the memory + /// to be released. + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code releaseMappedMemory(sys::MemoryBlock &M) = 0; + + virtual ~MemoryMapper(); + }; + + /// Creates a LlvmliteMemoryManager instance with \p MM as the associated + /// memory mapper. If \p MM is nullptr then a default memory mapper is used + /// that directly calls into the operating system. + LlvmliteMemoryManager(MemoryMapper *MM = nullptr); + LlvmliteMemoryManager(const LlvmliteMemoryManager &) = delete; + void operator=(const LlvmliteMemoryManager &) = delete; + ~LlvmliteMemoryManager() override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. + /// + /// The value of \p Alignment must be a power of two. If \p Alignment is + /// zero a default alignment of 16 will be used. + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. + /// + /// The value of \p Alignment must be a power of two. If \p Alignment is + /// zero a default alignment of 16 will be used. + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool isReadOnly) override; + + /// Update section-specific memory permissions and other attributes. + /// + /// This method is called when object loading is complete and section page + /// permissions can be applied. It is up to the memory manager + /// implementation to decide whether or not to act on this method. The + /// memory manager will typically allocate all sections as read-write and + /// then apply specific permissions when this method is called. Code + /// sections cannot be executed until this function has been called. In + /// addition, any cache coherency operations needed to reliably use the + /// memory are also performed. + /// + /// \returns true if an error occurred, false otherwise. + bool finalizeMemory(std::string *ErrMsg = nullptr) override; + + /// Invalidate instruction cache for code sections. + /// + /// Some platforms with separate data cache and instruction cache require + /// explicit cache flush, otherwise JIT code manipulations (like resolved + /// relocations) will get to the data cache but not to the instruction + /// cache. + /// + /// This method is called from finalizeMemory. + virtual void invalidateInstructionCache(); + + virtual bool needsToReserveAllocationSpace() override { return true; } + + virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, + uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override; + + private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right + // before this block, store it's index in PendingMem, to be able to + // update the pending region if part of this block is allocated, rather + // than having to create a new one + unsigned PendingPrefixIndex; + }; + + struct MemoryGroup { + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the + // user. + SmallVector PendingMem; + SmallVector FreeMem; + + // All memory blocks that have been requested from the system + SmallVector AllocatedMem; + + sys::MemoryBlock Near; + }; + + uint8_t *allocateSection(AllocationPurpose Purpose, uintptr_t Size, + unsigned Alignment); + + std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions); + + bool hasSpace(const MemoryGroup &MemGroup, uintptr_t Size) const; + + void anchor() override; + + MemoryGroup CodeMem; + MemoryGroup RWDataMem; + MemoryGroup RODataMem; + MemoryMapper &MMapper; +}; + +} // end namespace llvm diff --git a/llvmlite/binding/executionengine.py b/llvmlite/binding/executionengine.py index a381c4e5c..1ba0985ef 100644 --- a/llvmlite/binding/executionengine.py +++ b/llvmlite/binding/executionengine.py @@ -1,3 +1,4 @@ +import platform from ctypes import (POINTER, c_char_p, c_bool, c_void_p, c_int, c_uint64, c_size_t, CFUNCTYPE, string_at, cast, py_object, Structure) @@ -9,14 +10,21 @@ ffi.lib.LLVMPY_LinkInMCJIT -def create_mcjit_compiler(module, target_machine): +def create_mcjit_compiler(module, target_machine, use_lmm=None): """ Create a MCJIT ExecutionEngine from the given *module* and *target_machine*. + + *lmm* controls whether the llvmlite memory manager is used. If not supplied, + the default choice for the platform will be used (``True`` on 64-bit ARM + systems, ``False`` otherwise). """ + if use_lmm is None: + use_lmm = platform.machine() in ('arm64', 'aarch64') + with ffi.OutputString() as outerr: engine = ffi.lib.LLVMPY_CreateMCJITCompiler( - module, target_machine, outerr) + module, target_machine, use_lmm, outerr) if not engine: raise RuntimeError(str(outerr)) @@ -238,6 +246,7 @@ def _dispose(self): ffi.lib.LLVMPY_CreateMCJITCompiler.argtypes = [ ffi.LLVMModuleRef, ffi.LLVMTargetMachineRef, + c_bool, POINTER(c_char_p), ] ffi.lib.LLVMPY_CreateMCJITCompiler.restype = ffi.LLVMExecutionEngineRef