LoadLibrary refactoring (#20841)

* Refactor LoadLibrary Methods

This change refactors the code in DllImport in preparation
for implementing the new NativeLibrary API here:
dotnet/corefx#32015

The two main changes are:

1) A change in the semantics of the internal LoadLibrary helper functions.

When a native library is loaded, there are two categories of callers
expecting different return values:

External callers like AssemblyNative::InternalLoadUnmanagedDllFromPath()
and the upcoming System.Runtime.Interop.Marshall.LoadLibrary()
need the raw system handle
Internal callers like LoadLibraryModule() need the PAL registered handle
This change modifies the internal LoadLibraryModule* methods to work
in terms of native system handles, so that external callers can obrain
them directly. Methods requiring PAL-handles can register them explicitly.

There is no change in external signature of DllImport class, or the
native Dll cache in AppDomain class.

2) Differentiate HMODULE and NATIVE_LIBRARY_HANDLE

This change defines NATIVE_LIBRARY_HANDLE type to represent
raw system handles to native libraries that are not registered
with the PAL (On Unix systems).

The types on PAL and DlImport methods are adjusted to make
this semantic distinction explicit.

* 
Fix loading LibC via PAL_LoadLibraryDirect()

src/inc/palclr_win.h

@@ -142,4 +142,10 @@
 #define WIN_PAL_ENDTRY_NAKED_DBG                                                          
 #endif // defined(ENABLE_CONTRACTS_IMPL) && !defined(JIT64_BUILD)
 
+#if !defined (FEATURE_PAL)
+// Native system libray handle.
+// In Windows, NATIVE_LIBRARY_HANDLE is the same as HMODULE.
+typedef HMODULE NATIVE_LIBRARY_HANDLE;
+#endif // !FEATURE_PAL
+
 #endif	// __PALCLR_WIN_H__

src/pal/inc/pal.h

@@ -68,6 +68,11 @@ extern "C" {
 #include <pal_error.h>
 #include <pal_mstypes.h>
 
+// Native system libray handle.
+// On Unix systems, NATIVE_LIBRARY_HANDLE type represents a library handle not registered with the PAL.
+// To get a HMODULE on Unix, call PAL_RegisterLibraryDirect() on a NATIVE_LIBRARY_HANDLE.
+typedef void * NATIVE_LIBRARY_HANDLE;
+
 /******************* Processor-specific glue  *****************************/
 
 #ifndef _MSC_VER
@@ -2590,7 +2595,7 @@ LoadLibraryExW(
         IN DWORD dwFlags);
 
 PALIMPORT
-void *
+NATIVE_LIBRARY_HANDLE
 PALAPI
 PAL_LoadLibraryDirect(
         IN LPCWSTR lpLibFileName);

src/pal/src/loader/module.cpp

@@ -103,10 +103,10 @@ static bool LOADConvertLibraryPathWideStringToMultibyteString(
     INT *multibyteLibraryPathLengthRef);
 static BOOL LOADValidateModule(MODSTRUCT *module);
 static LPWSTR LOADGetModuleFileName(MODSTRUCT *module);
-static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath);
-static void *LOADLoadLibraryDirect(LPCSTR libraryNameOrPath);
+static MODSTRUCT *LOADAddModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath);
+static NATIVE_LIBRARY_HANDLE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath);
 static BOOL LOADFreeLibrary(MODSTRUCT *module, BOOL fCallDllMain);
-static HMODULE LOADRegisterLibraryDirect(void *dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic);
+static HMODULE LOADRegisterLibraryDirect(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic);
 static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic);
 static BOOL LOADCallDllMainSafe(MODSTRUCT *module, DWORD dwReason, LPVOID lpReserved);
 
@@ -556,6 +556,33 @@ GetModuleFileNameW(
     return retval;
 }
 
+LPCSTR FixLibCName(LPCSTR shortAsciiName)
+{
+    // Check whether we have been requested to load 'libc'. If that's the case, then:
+    // * For Linux, use the full name of the library that is defined in <gnu/lib-names.h> by the
+    //   LIBC_SO constant. The problem is that calling dlopen("libc.so") will fail for libc even
+    //   though it works for other libraries. The reason is that libc.so is just linker script
+    //   (i.e. a test file).
+    //   As a result, we have to use the full name (i.e. lib.so.6) that is defined by LIBC_SO.
+    // * For macOS, use constant value absolute path "/usr/lib/libc.dylib".
+    // * For FreeBSD, use constant value "libc.so.7".
+    // * For rest of Unices, use constant value "libc.so".
+    if (strcmp(shortAsciiName, LIBC_NAME_WITHOUT_EXTENSION) == 0)
+    {
+#if defined(__APPLE__)
+        return "/usr/lib/libc.dylib";
+#elif defined(__FreeBSD__)
+        return "libc.so.7";
+#elif defined(LIBC_SO)
+        return LIBC_SO;
+#else
+        return "libc.so";
+#endif
+    }
+
+    return shortAsciiName;
+}
+
 /*
 Function:
   PAL_LoadLibraryDirect
@@ -564,15 +591,16 @@ GetModuleFileNameW(
 
   Returns the system handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-void *
+NATIVE_LIBRARY_HANDLE
 PALAPI
 PAL_LoadLibraryDirect(
     IN LPCWSTR lpLibFileName)
 {
     PathCharString pathstr;
     CHAR * lpstr = nullptr;
+    LPCSTR lpcstr = nullptr;
     INT name_length;
-    void *dl_handle = nullptr;
+    NATIVE_LIBRARY_HANDLE dl_handle = nullptr;
 
     PERF_ENTRY(LoadLibraryDirect);
     ENTRY("LoadLibraryDirect (lpLibFileName=%p (%S)) \n",
@@ -597,8 +625,9 @@ PAL_LoadLibraryDirect(
     /* do the Dos/Unix conversion on our own copy of the name */
     FILEDosToUnixPathA(lpstr);
     pathstr.CloseBuffer(name_length);
+    lpcstr = FixLibCName(lpstr);
 
-    dl_handle = LOADLoadLibraryDirect(lpstr);
+    dl_handle = LOADLoadLibraryDirect(lpcstr);
 
 done:
     LOGEXIT("LoadLibraryDirect returns HMODULE %p\n", dl_handle);
@@ -617,7 +646,7 @@ PAL_LoadLibraryDirect(
 HMODULE
 PALAPI
 PAL_RegisterLibraryDirect(
-    IN void *dl_handle,
+    IN NATIVE_LIBRARY_HANDLE dl_handle,
     IN LPCWSTR lpLibFileName)
 {
     PathCharString pathstr;
@@ -651,7 +680,7 @@ PAL_RegisterLibraryDirect(
 
     /* let LOADRegisterLibraryDirect call SetLastError in case of failure */
     LockModuleList();
-    hModule = LOADRegisterLibraryDirect((void *)dl_handle, lpstr, true /* fDynamic */);
+    hModule = LOADRegisterLibraryDirect(dl_handle, lpstr, true /* fDynamic */);
     UnlockModuleList();
 
 done:
@@ -684,7 +713,7 @@ PAL_RegisterModule(
 
         LockModuleList();
 
-        void *dl_handle = LOADLoadLibraryDirect(lpLibFileName);
+        NATIVE_LIBRARY_HANDLE dl_handle = LOADLoadLibraryDirect(lpLibFileName);
         if (dl_handle)
         {
             // This only creates/adds the module handle and doesn't call DllMain
@@ -1395,12 +1424,12 @@ static LPWSTR LOADGetModuleFileName(MODSTRUCT *module)
 Return value:
     System handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static void *LOADLoadLibraryDirect(LPCSTR libraryNameOrPath)
+static NATIVE_LIBRARY_HANDLE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath)
 {
     _ASSERTE(libraryNameOrPath != nullptr);
     _ASSERTE(libraryNameOrPath[0] != '\0');
 
-    void *dl_handle = dlopen(libraryNameOrPath, RTLD_LAZY);
+    NATIVE_LIBRARY_HANDLE dl_handle = dlopen(libraryNameOrPath, RTLD_LAZY);
     if (dl_handle == nullptr)
     {
         SetLastError(ERROR_MOD_NOT_FOUND);
@@ -1420,7 +1449,7 @@ Function :
     Allocate and initialize a new MODSTRUCT structure
 
 Parameters :
-    void *dl_handle :   handle returned by dl_open, goes in MODSTRUCT::dl_handle
+    NATIVE_LIBRARY_HANDLE dl_handle :   handle returned by dl_open, goes in MODSTRUCT::dl_handle
     
     char *name :        name of new module. after conversion to widechar, 
                         goes in MODSTRUCT::lib_name
@@ -1432,7 +1461,7 @@ Notes :
     'name' is used to initialize MODSTRUCT::lib_name. The other member is set to NULL
     In case of failure (in malloc or MBToWC), this function sets LastError.
 --*/
-static MODSTRUCT *LOADAllocModule(void *dl_handle, LPCSTR name)
+static MODSTRUCT *LOADAllocModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR name)
 {   
     MODSTRUCT *module;
     LPWSTR wide_name;
@@ -1485,13 +1514,13 @@ static MODSTRUCT *LOADAllocModule(void *dl_handle, LPCSTR name)
     Registers a system handle to a loaded library with the module list.
 
 Parameters:
-    void *dl_handle:                    System handle to the loaded library.
+    NATIVE_LIBRARY_HANDLE dl_handle:    System handle to the loaded library.
     LPCSTR libraryNameOrPath:           The library that was loaded.
 
 Return value:
     PAL handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath)
+static MODSTRUCT *LOADAddModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath)
 {
     _ASSERTE(dl_handle != nullptr);
     _ASSERTE(libraryNameOrPath != nullptr);
@@ -1555,14 +1584,14 @@ static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath)
     Registers a system handle to a loaded library with the module list.
 
 Parameters:
-    void *dl_handle:                    System handle to the loaded library.
+    NATIVE_LIBRARY_HANDLE dl_handle:    System handle to the loaded library.
     LPCSTR libraryNameOrPath:           The library that was loaded.
     BOOL fDynamic:                      TRUE if dynamic load through LoadLibrary, FALSE if static load through RegisterLibrary.
 
 Return value:
     PAL handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static HMODULE LOADRegisterLibraryDirect(void *dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic)
+static HMODULE LOADRegisterLibraryDirect(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic)
 {
     MODSTRUCT *module = LOADAddModule(dl_handle, libraryNameOrPath);
     if (module == nullptr)
@@ -1631,30 +1660,10 @@ Return value :
 static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic)
 {
     HMODULE module = nullptr;
-    void *dl_handle = nullptr;
-
-    // Check whether we have been requested to load 'libc'. If that's the case, then:
-    // * For Linux, use the full name of the library that is defined in <gnu/lib-names.h> by the
-    //   LIBC_SO constant. The problem is that calling dlopen("libc.so") will fail for libc even
-    //   though it works for other libraries. The reason is that libc.so is just linker script
-    //   (i.e. a test file).
-    //   As a result, we have to use the full name (i.e. lib.so.6) that is defined by LIBC_SO.
-    // * For macOS, use constant value absolute path "/usr/lib/libc.dylib".
-    // * For FreeBSD, use constant value "libc.so.7".
-    // * For rest of Unices, use constant value "libc.so".
-    if (strcmp(shortAsciiName, LIBC_NAME_WITHOUT_EXTENSION) == 0)
-    {
-#if defined(__APPLE__)
-        shortAsciiName = "/usr/lib/libc.dylib";
-#elif defined(__FreeBSD__)
-        shortAsciiName = "libc.so.7";
-#elif defined(LIBC_SO)
-        shortAsciiName = LIBC_SO;
-#else
-        shortAsciiName = "libc.so";
-#endif
-    }
+    NATIVE_LIBRARY_HANDLE dl_handle = nullptr;
 
+    shortAsciiName = FixLibCName(shortAsciiName);
+
     LockModuleList();
 
     dl_handle = LOADLoadLibraryDirect(shortAsciiName);

src/vm/assemblynative.cpp

@@ -310,7 +310,7 @@ INT_PTR QCALLTYPE AssemblyNative::InternalLoadUnmanagedDllFromPath(LPCWSTR unman
 {
     QCALL_CONTRACT;
 
-    HMODULE moduleHandle = nullptr;
+    NATIVE_LIBRARY_HANDLE moduleHandle = nullptr;
 
     BEGIN_QCALL;