diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp
index 381380372db40..3bb11988b730e 100644
--- a/src/llvm-alloc-opt.cpp
+++ b/src/llvm-alloc-opt.cpp
@@ -350,7 +350,8 @@ void Optimizer::optimizeAll()
             if (field.hasobjref) {
                 has_ref = true;
                 // This can be relaxed a little based on hasload
-                if (field.hasaggr || field.multiloc) {
+                // TODO: add support for hasaggr load/store
+                if (field.hasaggr || field.multiloc || field.size != sizeof(void*)) {
                     has_refaggr = true;
                     break;
                 }
@@ -361,15 +362,12 @@ void Optimizer::optimizeAll()
             splitOnStack(orig);
             continue;
         }
-        if (has_ref) {
-            if (use_info.memops.size() != 1 || has_refaggr ||
-                use_info.memops.begin()->second.size != sz) {
-                if (use_info.hastypeof)
-                    optimizeTag(orig);
-                continue;
-            }
-            // The object only has a single field that's a reference with only one kind of access.
+        if (has_refaggr) {
+            if (use_info.hastypeof)
+                optimizeTag(orig);
+            continue;
         }
+        // The object has no fields with mix reference access
         moveToStack(orig, sz, has_ref);
     }
 }
@@ -444,6 +442,7 @@ Optimizer::AllocUseInfo::getField(uint32_t offset, uint32_t size, Type *elty)
         if (it->first + it->second.size >= offset + size) {
             if (it->second.elty != elty)
                 it->second.elty = nullptr;
+            assert(it->second.elty == nullptr || (it->first == offset && it->second.size == size));
             return *it;
         }
         if (it->first + it->second.size > offset) {
@@ -454,7 +453,7 @@ Optimizer::AllocUseInfo::getField(uint32_t offset, uint32_t size, Type *elty)
     else {
         it = memops.begin();
     }
-    // Now fine the last slot that overlaps with the current memory location.
+    // Now find the last slot that overlaps with the current memory location.
     // Also set `lb` if we didn't find any above.
     for (; it != end && it->first < offset + size; ++it) {
         if (lb == end)
@@ -493,8 +492,8 @@ bool Optimizer::AllocUseInfo::addMemOp(Instruction *inst, unsigned opno, uint32_
     memop.isaggr = isa<StructType>(elty) || isa<ArrayType>(elty) || isa<VectorType>(elty);
     memop.isobjref = hasObjref(elty);
     auto &field = getField(offset, size, elty);
-    if (field.first != offset || field.second.size != size)
-        field.second.multiloc = true;
+    if (field.second.hasobjref != memop.isobjref)
+        field.second.multiloc = true; // can't split this field, since it contains a mix of references and bits
     if (!isstore)
         field.second.hasload = true;
     if (memop.isobjref) {
diff --git a/test/llvmpasses/alloc-opt.jl b/test/llvmpasses/alloc-opt-gcframe.jl
similarity index 100%
rename from test/llvmpasses/alloc-opt.jl
rename to test/llvmpasses/alloc-opt-gcframe.jl
diff --git a/test/llvmpasses/alloc-opt2.jl b/test/llvmpasses/alloc-opt-pass.jl
similarity index 78%
rename from test/llvmpasses/alloc-opt2.jl
rename to test/llvmpasses/alloc-opt-pass.jl
index 8a3f5aa941feb..9bde40036ff73 100644
--- a/test/llvmpasses/alloc-opt2.jl
+++ b/test/llvmpasses/alloc-opt-pass.jl
@@ -107,3 +107,34 @@ declare void @llvm.memcpy.p11i8.p0i8.i64(i8 addrspace(11)* nocapture writeonly,
 declare token @llvm.julia.gc_preserve_begin(...)
 declare void @llvm.julia.gc_preserve_end(token)
 """)
+
+# CHECK-LABEL: @memref_collision
+# CHECK: call {}*** @julia.ptls_states()
+# CHECK-NOT: store {}
+# CHECK: store i
+# CHECK-NOT: store {}
+# CHECK: L1:
+# CHECK: load {}
+# CHECK: L2:
+# CHECK: load i
+println("""
+define void @memref_collision($isz %x) {
+  %ptls = call {}*** @julia.ptls_states()
+  %ptls_i8 = bitcast {}*** %ptls to i8*
+  %v = call noalias {} addrspace(10)* @julia.gc_alloc_obj(i8* %ptls_i8, $isz 8, {} addrspace(10)* @tag)
+  %v_p = bitcast {} addrspace(10)* %v to $isz addrspace(10)*
+  store $isz %x, $isz addrspace(10)* %v_p
+  br i1 0, label %L1, label %L2
+
+L1:
+  %v1 = bitcast {} addrspace(10)* %v to {} addrspace(10)* addrspace(10)*
+  %v1_x = load {} addrspace(10)*, {} addrspace(10)* addrspace(10)* %v1
+  ret void
+
+L2:
+  %v2 = bitcast {} addrspace(10)* %v to $isz addrspace(10)*
+  %v2_x = load i64, i64 addrspace(10)* %v2
+  ret void
+}
+""")
+# CHECK-LABEL: }{{$}}
diff --git a/test/llvmpasses/alloc-opt3.jl b/test/llvmpasses/alloc-opt-pipeline.jl
similarity index 100%
rename from test/llvmpasses/alloc-opt3.jl
rename to test/llvmpasses/alloc-opt-pipeline.jl