Addr11

enzyme/BCLoad/CMakeLists.txt

@@ -39,8 +39,8 @@ ExternalProject_Add(openblas
 set_target_properties(openblas PROPERTIES EXCLUDE_FROM_ALL TRUE)
 
 ExternalProject_Add(fblas
-    GIT_REPOSITORY https://github.com/UCSantaCruzComputationalGenomicsLab/clapack
-    GIT_TAG 8bac8d5cd7aa8506b11cdb2cfa2ce8a2e03048f3
+    GIT_REPOSITORY https://github.com/EnzymeAD/clapack
+    GIT_TAG 42603bf49ac2b27a7c00c53f92fa648748476510
     PREFIX fblas
     BUILD_IN_SOURCE 1
     INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/openblas/install

enzyme/Enzyme/CacheUtility.cpp

@@ -795,10 +795,10 @@ AllocaInst *CacheUtility::createCacheForScope(LimitContext ctx, Type *T,
       auto P = B.CreatePHI(i64, 1);
 
       CallInst *malloccall;
-      allocType = cast<PointerType>(CreateAllocation(B, types.back(), P,
+      Value *A = CreateAllocation(B, types.back(), P,
                                                      "tmpfortypecalc",
-                                                     &malloccall, nullptr)
-                                        ->getType());
+                                                     &malloccall, nullptr);
+      allocType = cast<PointerType>(A->getType());
       malloctypes.push_back(cast<PointerType>(malloccall->getType()));
       SmallVector<Instruction *, 2> toErase;
       for (auto &I : *BB)
@@ -908,6 +908,7 @@ AllocaInst *CacheUtility::createCacheForScope(LimitContext ctx, Type *T,
           }
           scopeInstructions[alloc].push_back(ZeroInst);
         }
+
         storealloc = allocationBuilder.CreateStore(firstallocation, storeInto);
 
         scopeAllocs[alloc].push_back(malloccall);
@@ -1016,11 +1017,21 @@ AllocaInst *CacheUtility::createCacheForScope(LimitContext ctx, Type *T,
 #else
       cast<LoadInst>(storeInto)->setAlignment(alignSize);
 #endif
-      storeInto = v.CreateGEP(storeInto->getType()->getPointerElementType(),
-                              storeInto, idx);
 #else
       storeInto = v.CreateLoad(storeInto);
       cast<LoadInst>(storeInto)->setAlignment(alignSize);
+#endif
+
+      Value *storeIntoN = GetAddressableFromAllocation(v, storeInto);
+      if (storeInto != storeIntoN) {
+        scopeInstructions[alloc].push_back(cast<Instruction>(storeIntoN));
+      }
+      storeInto = storeIntoN;
+
+#if LLVM_VERSION_MAJOR > 7
+      storeInto = v.CreateGEP(storeInto->getType()->getPointerElementType(),
+                              storeInto, idx);
+#else
       storeInto = v.CreateGEP(storeInto, idx);
 #endif
       cast<GetElementPtrInst>(storeInto)->setIsInBounds(true);
@@ -1557,6 +1568,13 @@ Value *CacheUtility::getCachePointer(bool inForwardPass, IRBuilder<> &BuilderM,
 #else
     cast<LoadInst>(next)->setAlignment(align);
 #endif
+
+    Value *nextN = GetAddressableFromAllocation(BuilderM, next);
+      if (next != nextN) {
+        scopeInstructions[cast<AllocaInst>(cache)].push_back(
+          cast<Instruction>(nextN));
+      }
+      next = nextN;
 
     const auto &containedloops = sublimits[i].second;
 

enzyme/Enzyme/EnzymeLogic.cpp

@@ -2525,6 +2525,7 @@ const AugmentedReturn &EnzymeLogic::CreateAugmentedPrimal(
         tapeMemory = CreateAllocation(
             ib, tapeType, ConstantInt::get(i64, 1), "tapemem", &malloccall,
             EnzymeZeroCache ? &zero : nullptr, /*isDefault*/ true);
+        tapeMemory = GetAddressableFromAllocation(ib, tapeMemory);
         memory = malloccall;
       } else {
         memory = ConstantPointerNull::get(

enzyme/Enzyme/FunctionUtils.cpp

@@ -472,6 +472,8 @@ UpgradeAllocasToMallocs(Function *NewF, DerivativeMode mode,
                                           MDNode::get(CI->getContext(), {}));
     }
 
+    rep = GetAddressableFromAllocation(B, rep);
+
     auto PT0 = cast<PointerType>(rep->getType());
     auto PT1 = cast<PointerType>(AI->getType());
     if (PT0->getAddressSpace() != PT1->getAddressSpace()) {

enzyme/Enzyme/Utils.cpp

@@ -72,6 +72,13 @@ llvm::SmallVector<llvm::Instruction *, 2> PostCacheStore(llvm::StoreInst *SI,
   return res;
 }
 
+Value *GetAddressableFromAllocation(IRBuilder <>&B, Value* gVal) {
+  auto PT = cast<PointerType>(gVal->getType());
+  if (PT->getAddressSpace() == 10)
+      return B.CreateAddrSpaceCast(gVal, PointerType::get(PT->getPointerElementType(), 11));
+  return gVal;
+}
+
 llvm::PointerType *getDefaultAnonymousTapeType(llvm::LLVMContext &C) {
   if (EnzymeDefaultTapeType)
     return cast<PointerType>(unwrap(EnzymeDefaultTapeType(wrap(&C))));
@@ -164,28 +171,46 @@ Function *getOrInsertExponentialAllocator(Module &M, Function *newFunc,
 
     Value *args[] = {B.CreatePointerCast(ptr, allocType), next};
     gVal = B.CreateCall(reallocF, args);
+    if (ZeroInit) {
+        Value *zeroSize = B.CreateSub(next, prevSize);
+
+        Value *margs[] = {
+#if LLVM_VERSION_MAJOR > 7
+          B.CreateInBoundsGEP(gVal->getType()->getPointerElementType(), gVal,
+                              prevSize),
+#else
+          B.CreateInBoundsGEP(gVal, prevSize),
+#endif
+          ConstantInt::get(Type::getInt8Ty(M.getContext()), 0),
+          zeroSize,
+          ConstantInt::getFalse(M.getContext())
+        };
+        Type *tys[] = {margs[0]->getType(), margs[2]->getType()};
+        auto memsetF = Intrinsic::getDeclaration(&M, Intrinsic::memset, tys);
+        B.CreateCall(memsetF, margs);
+      }
   } else {
     Value *tsize = ConstantInt::get(
         next->getType(),
         newFunc->getParent()->getDataLayout().getTypeAllocSizeInBits(RT) / 8);
     auto elSize = B.CreateUDiv(next, tsize, "", /*isExact*/ true);
     Instruction *SubZero = nullptr;
     gVal = CreateAllocation(B, RT, elSize, "", nullptr, &SubZero);
+    Value *tmp = GetAddressableFromAllocation(B, gVal);
 
-    gVal = B.CreatePointerCast(
-        gVal, PointerType::get(
-                  Type::getInt8Ty(gVal->getContext()),
-                  cast<PointerType>(gVal->getType())->getAddressSpace()));
-    auto pVal = B.CreatePointerCast(ptr, gVal->getType());
+    tmp = B.CreatePointerCast(
+        tmp, PointerType::get(
+                  Type::getInt8Ty(tmp->getContext()),
+                  cast<PointerType>(tmp->getType())->getAddressSpace()));
+    auto pVal = B.CreatePointerCast(ptr, tmp->getType());
 
-    Value *margs[] = {gVal, pVal, prevSize,
+    Value *margs[] = {tmp, pVal, prevSize,
                       ConstantInt::getFalse(M.getContext())};
     Type *tys[] = {margs[0]->getType(), margs[1]->getType(),
                    margs[2]->getType()};
     auto memsetF = Intrinsic::getDeclaration(&M, Intrinsic::memcpy, tys);
     B.CreateCall(memsetF, margs);
     if (SubZero) {
-      ZeroInit = false;
       IRBuilder<> BB(SubZero);
       Value *zeroSize = BB.CreateSub(next, prevSize);
       Value *tmp = SubZero->getOperand(0);
@@ -200,25 +225,6 @@ Function *getOrInsertExponentialAllocator(Module &M, Function *newFunc,
       SubZero->setOperand(2, zeroSize);
     }
   }
-
-  if (ZeroInit) {
-    Value *zeroSize = B.CreateSub(next, prevSize);
-
-    Value *margs[] = {
-#if LLVM_VERSION_MAJOR > 7
-      B.CreateInBoundsGEP(gVal->getType()->getPointerElementType(), gVal,
-                          prevSize),
-#else
-      B.CreateInBoundsGEP(gVal, prevSize),
-#endif
-      ConstantInt::get(Type::getInt8Ty(M.getContext()), 0),
-      zeroSize,
-      ConstantInt::getFalse(M.getContext())
-    };
-    Type *tys[] = {margs[0]->getType(), margs[2]->getType()};
-    auto memsetF = Intrinsic::getDeclaration(&M, Intrinsic::memset, tys);
-    B.CreateCall(memsetF, margs);
-  }
   gVal = B.CreatePointerCast(gVal, ptr->getType());
 
   B.CreateBr(ok);

enzyme/Enzyme/Utils.h

@@ -92,6 +92,8 @@ llvm::Value *CreateAllocation(llvm::IRBuilder<> &B, llvm::Type *T,
                               bool isDefault = false);
 llvm::CallInst *CreateDealloc(llvm::IRBuilder<> &B, llvm::Value *ToFree);
 
+llvm::Value *GetAddressableFromAllocation(llvm::IRBuilder <>&B, llvm::Value* gVal);
+
 llvm::Value *CreateReAllocation(llvm::IRBuilder<> &B, llvm::Value *prev,
                                 llvm::Type *T, llvm::Value *OuterCount,
                                 llvm::Value *InnerCount, llvm::Twine Name = "",