check the right return value for mmap (#77952)

dotnet · Nov 7, 2022 · 892ebf4 · 892ebf4
1 parent bfa4812
commit 892ebf4
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/src/coreclr/gc/unix/gcenv.unix.cpp b/src/coreclr/gc/unix/gcenv.unix.cpp
@@ -641,7 +641,7 @@ static void* VirtualReserveInner(size_t size, size_t alignment, uint32_t flags,
     size_t alignedSize = size + (alignment - OS_PAGE_SIZE);
     void * pRetVal = mmap(nullptr, alignedSize, PROT_NONE, MAP_ANON | MAP_PRIVATE | hugePagesFlag, -1, 0);
 
-    if (pRetVal != NULL)
+    if (pRetVal != MAP_FAILED)
     {
         void * pAlignedRetVal = (void *)(((size_t)pRetVal + (alignment - 1)) & ~(alignment - 1));
         size_t startPadding = (size_t)pAlignedRetVal - (size_t)pRetVal;
@@ -663,9 +663,10 @@ static void* VirtualReserveInner(size_t size, size_t alignment, uint32_t flags,
         // Do not include reserved memory in coredump.
         madvise(pRetVal, size, MADV_DONTDUMP);
 #endif
+        return pRetVal;
     }
 
-    return pRetVal;
+    return NULL; // return NULL if mmap failed
 }
 
 // Reserve virtual memory range.