JDK17+ JVM_LoadLibrary() opens shared library via J9PORT_SLOPEN_DECORATE

j9sl_open_shared_library() with the flag J9PORT_SLOPEN_DECORATE expects
the incoming library name to be platform independent, i.e., it must not
contain any prefix or file extension. A path to the library is
supported.

Signed-off-by: Jason Feng <fengj@ca.ibm.com>

runtime/j9vm/j9scar.tdf

@@ -387,3 +387,6 @@ TraceExit=Trc_SC_VirtualThreadStart_Exit Overhead=1 Level=3 Template="thread = %
 
 TraceEntry=Trc_SC_VirtualThreadEnd_Entry Overhead=1 Level=3 Template="thread = %p"
 TraceExit=Trc_SC_VirtualThreadEnd_Exit Overhead=1 Level=3 Template="thread = %p"
+
+TraceEvent=Trc_SC_libName_no_extension NoEnv Overhead=1 Level=3 Test Template="Skip JDK17+ libName (%s) without a file extension"
+TraceEvent=Trc_SC_libPath_not_sufficient NoEnv Overhead=1 Level=3 Test Template="libPath buffer size is not large enough"

runtime/j9vm/jvm.c

@@ -3989,15 +3989,57 @@ JVM_LoadLibrary(const char *libName, jboolean throwOnFailure)
 			UDATA slOpenResult = j9sl_open_shared_library((char *)libName, &handle, flags);
 			Trc_SC_LoadLibrary_OpenShared(libName);
 
-/* jdk17+ calls JVM_LoadLibrary with decorated library names. If the following is done
- * then it overwrites the real error with a failure to load "liblib<name>.so.so".
- */
-#if JAVA_SPEC_VERSION < 17
 			if (0 != slOpenResult) {
-				slOpenResult = j9sl_open_shared_library((char *)libName, &handle, flags | J9PORT_SLOPEN_DECORATE);
-				Trc_SC_LoadLibrary_OpenShared_Decorate(libName);
-			}
+				const char *libNameNotDecorated = libName;
+#if JAVA_SPEC_VERSION >= 17
+				/* JDK17+ calls JVM_LoadLibrary() with a decorated library name,
+				 * i.e., a path to Java_java_lang_System_mapLibraryName(nameNoPrefixNoExtension).
+				 * The library name passed to j9sl_open_shared_library() with the flag J9PORT_SLOPEN_DECORATE
+				 * has to be platform independent, i.e., it must not contain any prefix or file extension.
+				 * A path is supported.
+				 */
+				char libPath[EsMaxPath] = {0};
+				const char *fileExt = strrchr(libName, '.');
+				BOOLEAN doOpenLibrary = TRUE;
+				if (NULL != fileExt) {
+#if defined(J9OS_I5) || defined(WIN32)
+					/* no library prefix for J9OS_I5 and WIN32 */
+					const size_t libStrLength = 0;
+#else /* defined(J9OS_I5) || defined(WIN32) */
+					/*  strlen("lib") = 3 */
+					const size_t libStrLength = 3;
+#endif /* defined(J9OS_I5) || defined(WIN32) */
+					const size_t libSuffixLength = strlen(fileExt);
+					const char *fileName = strrchr(libName, DIR_SEPARATOR);
+					UDATA libPathLength = 0;
+					if (NULL != fileName) {
+						/* the libName specifies a path */
+						const char *fileNameNoPrefix = fileName + 1 + libStrLength;
+						libPathLength = j9str_printf(PORTLIB, libPath, sizeof(libPath), "%.*s%.*s", (uintptr_t)fileName - (uintptr_t)libName + 1, libName,
+								strlen(fileNameNoPrefix) - libSuffixLength, fileNameNoPrefix);
+					} else {
+						fileName = libName + libStrLength;
+						libPathLength = j9str_printf(PORTLIB, libPath, sizeof(libPath), "%.*s", strlen(fileName) - libSuffixLength, fileName);
+					}
+					/* j9sl_open_shared_library() expects the path not longer than EsMaxPath */
+					if (libPathLength < EsMaxPath) {
+						libNameNotDecorated = libPath;
+					} else {
+						doOpenLibrary = FALSE;
+						Trc_SC_libPath_not_sufficient();
+					}
+				} else {
+					/* A decorated library name is expected with a file extension,
+					 * pass to j9sl_open_shared_library w/o modification. */
+					Trc_SC_libName_no_extension(libName);
+				}
+				if (doOpenLibrary)
 #endif /* JAVA_SPEC_VERSION < 17 */
+				{
+					slOpenResult = j9sl_open_shared_library((char *)libNameNotDecorated, &handle, flags | J9PORT_SLOPEN_DECORATE);
+					Trc_SC_LoadLibrary_OpenShared_Decorate(libName);
+				}
+			}
 			if (0 == slOpenResult) {
 				result = (void *)handle;
 			}