Merge in 'release/6.0' changes

eng/pipelines/common/xplat-setup.yml

@@ -121,7 +121,7 @@ jobs:
 
         # OSX Build Pool (we don't have on-prem OSX BuildPool
         ${{ if in(parameters.osGroup, 'OSX', 'MacCatalyst', 'iOS', 'iOSSimulator', 'tvOS', 'tvOSSimulator') }}:
-          vmImage: 'macOS-10.15'
+          vmImage: 'macOS-11'
 
         # Official Build Windows Pool
         ${{ if and(eq(parameters.osGroup, 'windows'), ne(variables['System.TeamProject'], 'public')) }}:

src/libraries/Native/Unix/System.Globalization.Native/configure.cmake

@@ -19,6 +19,11 @@ else()
             ucol_setMaxVariable
             "unicode/ucol.h"
             HAVE_SET_MAX_VARIABLE)
+
+        check_symbol_exists(
+            ucol_clone
+            "unicode/ucol.h"
+            HAVE_UCOL_CLONE)
 
         unset(CMAKE_REQUIRED_LIBRARIES)
         unset(CMAKE_REQUIRED_INCLUDES)

src/libraries/Native/Unix/System.Globalization.Native/pal_collation.c

@@ -321,7 +321,24 @@ static UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t o
 
     if (customRuleLength == 0)
     {
+#if !defined(STATIC_ICU)
+        if (ucol_clone_ptr != NULL)
+        {
+            pClonedCollator = ucol_clone(pCollator, pErr);
+        }
+        else
+        {
+            pClonedCollator = ucol_safeClone_ptr(pCollator, NULL, NULL, pErr);
+        }
+#else // !defined(STATIC_ICU)
+
+#if HAVE_UCOL_CLONE
+        pClonedCollator = ucol_clone(pCollator, pErr);
+#else
         pClonedCollator = ucol_safeClone(pCollator, NULL, NULL, pErr);
+#endif // HAVE_UCOL_CLONE
+
+#endif // !defined(STATIC_ICU)
     }
     else
     {
@@ -365,6 +382,24 @@ static UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t o
     {
         ucol_setAttribute(pClonedCollator, UCOL_ALTERNATE_HANDLING, UCOL_SHIFTED, pErr);
 
+#if !defined(STATIC_ICU)
+    if (ucol_setMaxVariable_ptr != NULL)
+    {
+        // by default, ICU alternate shifted handling only ignores punctuation, but
+        // IgnoreSymbols needs symbols and currency as well, so change the "variable top"
+        // to include all symbols and currency
+        ucol_setMaxVariable(pClonedCollator, UCOL_REORDER_CODE_CURRENCY, pErr);
+    }
+    else
+    {
+        assert(ucol_setVariableTop_ptr != NULL);
+        // 0xfdfc is the last currency character before the first digit character
+        // in http://source.icu-project.org/repos/icu/icu/tags/release-52-1/source/data/unidata/FractionalUCA.txt
+        const UChar ignoreSymbolsVariableTop[] = { 0xfdfc };
+        ucol_setVariableTop_ptr(pClonedCollator, ignoreSymbolsVariableTop, 1, pErr);
+    }
+
+#else // !defined(STATIC_ICU)
         // by default, ICU alternate shifted handling only ignores punctuation, but
         // IgnoreSymbols needs symbols and currency as well, so change the "variable top"
         // to include all symbols and currency
@@ -376,6 +411,8 @@ static UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t o
         const UChar ignoreSymbolsVariableTop[] = { 0xfdfc };
         ucol_setVariableTop(pClonedCollator, ignoreSymbolsVariableTop, 1, pErr);
 #endif
+
+#endif //!defined(STATIC_ICU)
     }
 
     ucol_setAttribute(pClonedCollator, UCOL_STRENGTH, strength, pErr);

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim.c

@@ -29,7 +29,6 @@ FOR_ALL_ICU_FUNCTIONS
 #define SYMBOL_NAME_SIZE (128 + SYMBOL_CUSTOM_SUFFIX_SIZE)
 #define MaxICUVersionStringWithSuffixLength (MaxICUVersionStringLength + SYMBOL_CUSTOM_SUFFIX_SIZE)
 
-
 #if defined(TARGET_WINDOWS) || defined(TARGET_OSX) || defined(TARGET_ANDROID)
 
 #define MaxICUVersionStringLength 33
@@ -38,6 +37,8 @@ FOR_ALL_ICU_FUNCTIONS
 
 static void* libicuuc = NULL;
 static void* libicui18n = NULL;
+ucol_setVariableTop_func ucol_setVariableTop_ptr = NULL;
+ucol_safeClone_func ucol_safeClone_ptr = NULL;
 
 #if defined (TARGET_UNIX)
 
@@ -380,6 +381,58 @@ static void ValidateICUDataCanLoad()
     }
 }
 
+static void InitializeUColClonePointers(char* symbolVersion)
+{
+    if (ucol_clone_ptr != NULL)
+    {
+        return;
+    }
+
+#if defined(TARGET_WINDOWS)
+    char symbolName[SYMBOL_NAME_SIZE];
+    sprintf_s(symbolName, SYMBOL_NAME_SIZE, "ucol_safeClone%s", symbolVersion);
+    ucol_safeClone_ptr = (ucol_safeClone_func)GetProcAddress((HMODULE)libicui18n, symbolName);
+#else
+    char symbolName[SYMBOL_NAME_SIZE];
+    sprintf(symbolName, "ucol_safeClone%s", symbolVersion);
+    ucol_safeClone_ptr = (ucol_safeClone_func)dlsym(libicui18n, symbolName);
+#endif // defined(TARGET_WINDOWS)
+
+    if (ucol_safeClone_ptr == NULL)
+    {
+        fprintf(stderr, "Cannot get the symbols of ICU APIs ucol_safeClone or ucol_clone.\n");
+        abort();
+    }
+}
+
+static void InitializeVariableMaxAndTopPointers(char* symbolVersion)
+{
+    if (ucol_setMaxVariable_ptr != NULL)
+    {
+        return;
+    }
+
+#if defined(TARGET_OSX) || defined(TARGET_ANDROID)
+    // OSX and Android always run against ICU version which has ucol_setMaxVariable.
+    // We shouldn't come here.
+    assert(false);
+#elif defined(TARGET_WINDOWS)
+    char symbolName[SYMBOL_NAME_SIZE];
+    sprintf_s(symbolName, SYMBOL_NAME_SIZE, "ucol_setVariableTop%s", symbolVersion);
+    ucol_setVariableTop_ptr = (ucol_setVariableTop_func)GetProcAddress((HMODULE)libicui18n, symbolName);
+#else
+    char symbolName[SYMBOL_NAME_SIZE];
+    sprintf(symbolName, "ucol_setVariableTop%s", symbolVersion);
+    ucol_setVariableTop_ptr = (ucol_setVariableTop_func)dlsym(libicui18n, symbolName);
+#endif // defined(TARGET_OSX) || defined(TARGET_ANDROID)
+
+    if (ucol_setVariableTop_ptr == NULL)
+    {
+        fprintf(stderr, "Cannot get the symbols of ICU APIs ucol_setMaxVariable or ucol_setVariableTop.\n");
+        abort();
+    }
+}
+
 // GlobalizationNative_LoadICU
 // This method get called from the managed side during the globalization initialization.
 // This method shouldn't get called at all if we are running in globalization invariant mode
@@ -413,6 +466,10 @@ int32_t GlobalizationNative_LoadICU()
 
     FOR_ALL_ICU_FUNCTIONS
     ValidateICUDataCanLoad();
+
+    InitializeVariableMaxAndTopPointers(symbolVersion);
+    InitializeUColClonePointers(symbolVersion);
+
     return true;
 }
 
@@ -466,6 +523,8 @@ void GlobalizationNative_InitICUFunctions(void* icuuc, void* icuin, const char*
 
     FOR_ALL_ICU_FUNCTIONS
     ValidateICUDataCanLoad();
+
+    InitializeVariableMaxAndTopPointers(symbolVersion);
 }
 
 #undef PER_FUNCTION_BLOCK

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal.h

@@ -58,9 +58,32 @@
 // (U_ICU_VERSION_MAJOR_NUM < 52)
 // The following APIs are not supported in the ICU versions less than 52. We need to define them manually.
 // We have to do runtime check before using the pointers to these APIs. That is why these are listed in the FOR_ALL_OPTIONAL_ICU_FUNCTIONS list.
+U_CAPI void U_EXPORT2 ucol_setMaxVariable(UCollator* coll, UColReorderCode group, UErrorCode* pErrorCode);
 U_CAPI int32_t U_EXPORT2 ucal_getTimeZoneIDForWindowsID(const UChar* winid, int32_t len, const char* region, UChar* id, int32_t idCapacity, UErrorCode* status);
 U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len, UChar* winid, int32_t winidCapacity, UErrorCode* status);
-#endif
+
+// (U_ICU_VERSION_MAJOR_NUM < 71)
+// The following API is not supported in the ICU versions less than 71. We need to define it manually.
+// We have to do runtime check before using the pointers to this API. That is why these are listed in the FOR_ALL_OPTIONAL_ICU_FUNCTIONS list.
+U_CAPI UCollator* U_EXPORT2 ucol_clone(const UCollator* coll, UErrorCode* status);
+
+// ucol_setVariableTop is a deprecated function on the newer ICU versions and ucol_setMaxVariable should be used instead.
+// As can run against ICU versions which not supported ucol_setMaxVariable, we'll dynamically try to get the pointer to ucol_setVariableTop
+// when we couldn't get a pointer to ucol_setMaxVariable.
+typedef uint32_t (U_EXPORT2 *ucol_setVariableTop_func)(UCollator* coll, const UChar* varTop, int32_t len, UErrorCode* status);
+
+// ucol_safeClone is deprecated in ICU version 71. We have to handle it manually to avoid getting a build break when referencing it in the code.
+typedef UCollator* (U_EXPORT2 *ucol_safeClone_func)(const UCollator* coll, void* stackBuffer, int32_t* pBufferSize, UErrorCode* status);
+
+#else // !defined(TARGET_ANDROID)
+
+typedef uint32_t (*ucol_setVariableTop_func)(UCollator* coll, const UChar* varTop, int32_t len, UErrorCode* status);
+typedef UCollator* (*ucol_safeClone_func)(const UCollator* coll, void* stackBuffer, int32_t* pBufferSize, UErrorCode* status);
+
+#endif // !defined(TARGET_ANDROID)
+
+extern ucol_setVariableTop_func ucol_setVariableTop_ptr;
+extern ucol_safeClone_func ucol_safeClone_ptr;
 
 // List of all functions from the ICU libraries that are used in the System.Globalization.Native.so
 #define FOR_ALL_UNCONDITIONAL_ICU_FUNCTIONS \
@@ -99,7 +122,6 @@ U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len,
     PER_FUNCTION_BLOCK(ucol_open, libicui18n, true) \
     PER_FUNCTION_BLOCK(ucol_openElements, libicui18n, true) \
     PER_FUNCTION_BLOCK(ucol_openRules, libicui18n, true) \
-    PER_FUNCTION_BLOCK(ucol_safeClone, libicui18n, true) \
     PER_FUNCTION_BLOCK(ucol_setAttribute, libicui18n, true) \
     PER_FUNCTION_BLOCK(ucol_strcoll, libicui18n, true) \
     PER_FUNCTION_BLOCK(udat_close, libicui18n, true) \
@@ -164,15 +186,6 @@ U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len,
     PER_FUNCTION_BLOCK(usearch_setPattern, libicui18n, true) \
     PER_FUNCTION_BLOCK(usearch_setText, libicui18n, true)
 
-#if HAVE_SET_MAX_VARIABLE
-#define FOR_ALL_SET_VARIABLE_ICU_FUNCTIONS \
-    PER_FUNCTION_BLOCK(ucol_setMaxVariable, libicui18n, true)
-#else
-
-#define FOR_ALL_SET_VARIABLE_ICU_FUNCTIONS \
-    PER_FUNCTION_BLOCK(ucol_setVariableTop, libicui18n, true)
-#endif
-
 #if defined(TARGET_WINDOWS)
 #define FOR_ALL_OS_CONDITIONAL_ICU_FUNCTIONS \
     PER_FUNCTION_BLOCK(ucurr_forLocale, libicuuc, true) \
@@ -195,11 +208,12 @@ U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len,
 // Otherwise, we'll just not provide the functionality to users which needed these APIs.
 #define FOR_ALL_OPTIONAL_ICU_FUNCTIONS \
     PER_FUNCTION_BLOCK(ucal_getWindowsTimeZoneID, libicui18n, false) \
-    PER_FUNCTION_BLOCK(ucal_getTimeZoneIDForWindowsID, libicui18n, false)
+    PER_FUNCTION_BLOCK(ucal_getTimeZoneIDForWindowsID, libicui18n, false) \
+    PER_FUNCTION_BLOCK(ucol_setMaxVariable, libicui18n, false) \
+    PER_FUNCTION_BLOCK(ucol_clone, libicui18n, false)
 
 #define FOR_ALL_ICU_FUNCTIONS \
     FOR_ALL_UNCONDITIONAL_ICU_FUNCTIONS \
-    FOR_ALL_SET_VARIABLE_ICU_FUNCTIONS \
     FOR_ALL_OPTIONAL_ICU_FUNCTIONS \
     FOR_ALL_OS_CONDITIONAL_ICU_FUNCTIONS
 
@@ -235,6 +249,7 @@ FOR_ALL_ICU_FUNCTIONS
 #define ucal_openTimeZoneIDEnumeration(...) ucal_openTimeZoneIDEnumeration_ptr(__VA_ARGS__)
 #define ucal_set(...) ucal_set_ptr(__VA_ARGS__)
 #define ucal_setMillis(...) ucal_setMillis_ptr(__VA_ARGS__)
+#define ucol_clone(...) ucol_clone_ptr(__VA_ARGS__)
 #define ucol_close(...) ucol_close_ptr(__VA_ARGS__)
 #define ucol_closeElements(...) ucol_closeElements_ptr(__VA_ARGS__)
 #define ucol_getOffset(...) ucol_getOffset_ptr(__VA_ARGS__)
@@ -247,13 +262,8 @@ FOR_ALL_ICU_FUNCTIONS
 #define ucol_open(...) ucol_open_ptr(__VA_ARGS__)
 #define ucol_openElements(...) ucol_openElements_ptr(__VA_ARGS__)
 #define ucol_openRules(...) ucol_openRules_ptr(__VA_ARGS__)
-#define ucol_safeClone(...) ucol_safeClone_ptr(__VA_ARGS__)
 #define ucol_setAttribute(...) ucol_setAttribute_ptr(__VA_ARGS__)
-#if HAVE_SET_MAX_VARIABLE
 #define ucol_setMaxVariable(...) ucol_setMaxVariable_ptr(__VA_ARGS__)
-#else
-#define ucol_setVariableTop(...) ucol_setVariableTop_ptr(__VA_ARGS__)
-#endif
 #define ucol_strcoll(...) ucol_strcoll_ptr(__VA_ARGS__)
 #define ucurr_forLocale(...) ucurr_forLocale_ptr(__VA_ARGS__)
 #define ucurr_getName(...) ucurr_getName_ptr(__VA_ARGS__)

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal_android.h

@@ -470,7 +470,7 @@ int32_t ucol_previous(UCollationElements * elems, UErrorCode * status);
 UCollator * ucol_open(const char * loc, UErrorCode * status);
 UCollationElements * ucol_openElements(const UCollator * coll, const UChar * text, int32_t textLength, UErrorCode * status);
 UCollator * ucol_openRules(const UChar * rules, int32_t rulesLength, UColAttributeValue normalizationMode, UCollationStrength strength, UParseError * parseError, UErrorCode * status);
-UCollator * ucol_safeClone(const UCollator * coll, void * stackBuffer, int32_t * pBufferSize, UErrorCode * status);
+UCollator * ucol_clone(const UCollator * coll, UErrorCode * status);
 void ucol_setAttribute(UCollator * coll, UColAttribute attr, UColAttributeValue value, UErrorCode * status);
 UCollationResult ucol_strcoll(const UCollator * coll, const UChar * source, int32_t sourceLength, const UChar * target, int32_t targetLength);
 int32_t ucurr_forLocale(const char * locale, UChar * buff, int32_t buffCapacity, UErrorCode * ec);