diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index ddae1a639448c..6ca180f45bd2b 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -121,7 +121,7 @@ jobs: # OSX Build Pool (we don't have on-prem OSX BuildPool ${{ if in(parameters.osGroup, 'OSX', 'MacCatalyst', 'iOS', 'iOSSimulator', 'tvOS', 'tvOSSimulator') }}: - vmImage: 'macOS-10.15' + vmImage: 'macOS-11' # Official Build Windows Pool ${{ if and(eq(parameters.osGroup, 'windows'), ne(variables['System.TeamProject'], 'public')) }}: diff --git a/src/libraries/Native/Unix/System.Globalization.Native/configure.cmake b/src/libraries/Native/Unix/System.Globalization.Native/configure.cmake index fc00d106185b2..da5562b736d2b 100644 --- a/src/libraries/Native/Unix/System.Globalization.Native/configure.cmake +++ b/src/libraries/Native/Unix/System.Globalization.Native/configure.cmake @@ -19,6 +19,11 @@ else() ucol_setMaxVariable "unicode/ucol.h" HAVE_SET_MAX_VARIABLE) + + check_symbol_exists( + ucol_clone + "unicode/ucol.h" + HAVE_UCOL_CLONE) unset(CMAKE_REQUIRED_LIBRARIES) unset(CMAKE_REQUIRED_INCLUDES) diff --git a/src/libraries/Native/Unix/System.Globalization.Native/pal_collation.c b/src/libraries/Native/Unix/System.Globalization.Native/pal_collation.c index 4a4c969485cb1..8ae6b040bb9a4 100644 --- a/src/libraries/Native/Unix/System.Globalization.Native/pal_collation.c +++ b/src/libraries/Native/Unix/System.Globalization.Native/pal_collation.c @@ -321,7 +321,24 @@ static UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t o if (customRuleLength == 0) { +#if !defined(STATIC_ICU) + if (ucol_clone_ptr != NULL) + { + pClonedCollator = ucol_clone(pCollator, pErr); + } + else + { + pClonedCollator = ucol_safeClone_ptr(pCollator, NULL, NULL, pErr); + } +#else // !defined(STATIC_ICU) + +#if HAVE_UCOL_CLONE + pClonedCollator = ucol_clone(pCollator, pErr); +#else pClonedCollator = ucol_safeClone(pCollator, NULL, NULL, pErr); +#endif // HAVE_UCOL_CLONE + +#endif // !defined(STATIC_ICU) } else { @@ -365,6 +382,24 @@ static UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t o { ucol_setAttribute(pClonedCollator, UCOL_ALTERNATE_HANDLING, UCOL_SHIFTED, pErr); +#if !defined(STATIC_ICU) + if (ucol_setMaxVariable_ptr != NULL) + { + // by default, ICU alternate shifted handling only ignores punctuation, but + // IgnoreSymbols needs symbols and currency as well, so change the "variable top" + // to include all symbols and currency + ucol_setMaxVariable(pClonedCollator, UCOL_REORDER_CODE_CURRENCY, pErr); + } + else + { + assert(ucol_setVariableTop_ptr != NULL); + // 0xfdfc is the last currency character before the first digit character + // in http://source.icu-project.org/repos/icu/icu/tags/release-52-1/source/data/unidata/FractionalUCA.txt + const UChar ignoreSymbolsVariableTop[] = { 0xfdfc }; + ucol_setVariableTop_ptr(pClonedCollator, ignoreSymbolsVariableTop, 1, pErr); + } + +#else // !defined(STATIC_ICU) // by default, ICU alternate shifted handling only ignores punctuation, but // IgnoreSymbols needs symbols and currency as well, so change the "variable top" // to include all symbols and currency @@ -376,6 +411,8 @@ static UCollator* CloneCollatorWithOptions(const UCollator* pCollator, int32_t o const UChar ignoreSymbolsVariableTop[] = { 0xfdfc }; ucol_setVariableTop(pClonedCollator, ignoreSymbolsVariableTop, 1, pErr); #endif + +#endif //!defined(STATIC_ICU) } ucol_setAttribute(pClonedCollator, UCOL_STRENGTH, strength, pErr); diff --git a/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim.c b/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim.c index 4f71b66c2678f..234517d8c1f01 100644 --- a/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim.c +++ b/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim.c @@ -29,7 +29,6 @@ FOR_ALL_ICU_FUNCTIONS #define SYMBOL_NAME_SIZE (128 + SYMBOL_CUSTOM_SUFFIX_SIZE) #define MaxICUVersionStringWithSuffixLength (MaxICUVersionStringLength + SYMBOL_CUSTOM_SUFFIX_SIZE) - #if defined(TARGET_WINDOWS) || defined(TARGET_OSX) || defined(TARGET_ANDROID) #define MaxICUVersionStringLength 33 @@ -38,6 +37,8 @@ FOR_ALL_ICU_FUNCTIONS static void* libicuuc = NULL; static void* libicui18n = NULL; +ucol_setVariableTop_func ucol_setVariableTop_ptr = NULL; +ucol_safeClone_func ucol_safeClone_ptr = NULL; #if defined (TARGET_UNIX) @@ -380,6 +381,58 @@ static void ValidateICUDataCanLoad() } } +static void InitializeUColClonePointers(char* symbolVersion) +{ + if (ucol_clone_ptr != NULL) + { + return; + } + +#if defined(TARGET_WINDOWS) + char symbolName[SYMBOL_NAME_SIZE]; + sprintf_s(symbolName, SYMBOL_NAME_SIZE, "ucol_safeClone%s", symbolVersion); + ucol_safeClone_ptr = (ucol_safeClone_func)GetProcAddress((HMODULE)libicui18n, symbolName); +#else + char symbolName[SYMBOL_NAME_SIZE]; + sprintf(symbolName, "ucol_safeClone%s", symbolVersion); + ucol_safeClone_ptr = (ucol_safeClone_func)dlsym(libicui18n, symbolName); +#endif // defined(TARGET_WINDOWS) + + if (ucol_safeClone_ptr == NULL) + { + fprintf(stderr, "Cannot get the symbols of ICU APIs ucol_safeClone or ucol_clone.\n"); + abort(); + } +} + +static void InitializeVariableMaxAndTopPointers(char* symbolVersion) +{ + if (ucol_setMaxVariable_ptr != NULL) + { + return; + } + +#if defined(TARGET_OSX) || defined(TARGET_ANDROID) + // OSX and Android always run against ICU version which has ucol_setMaxVariable. + // We shouldn't come here. + assert(false); +#elif defined(TARGET_WINDOWS) + char symbolName[SYMBOL_NAME_SIZE]; + sprintf_s(symbolName, SYMBOL_NAME_SIZE, "ucol_setVariableTop%s", symbolVersion); + ucol_setVariableTop_ptr = (ucol_setVariableTop_func)GetProcAddress((HMODULE)libicui18n, symbolName); +#else + char symbolName[SYMBOL_NAME_SIZE]; + sprintf(symbolName, "ucol_setVariableTop%s", symbolVersion); + ucol_setVariableTop_ptr = (ucol_setVariableTop_func)dlsym(libicui18n, symbolName); +#endif // defined(TARGET_OSX) || defined(TARGET_ANDROID) + + if (ucol_setVariableTop_ptr == NULL) + { + fprintf(stderr, "Cannot get the symbols of ICU APIs ucol_setMaxVariable or ucol_setVariableTop.\n"); + abort(); + } +} + // GlobalizationNative_LoadICU // This method get called from the managed side during the globalization initialization. // This method shouldn't get called at all if we are running in globalization invariant mode @@ -413,6 +466,10 @@ int32_t GlobalizationNative_LoadICU() FOR_ALL_ICU_FUNCTIONS ValidateICUDataCanLoad(); + + InitializeVariableMaxAndTopPointers(symbolVersion); + InitializeUColClonePointers(symbolVersion); + return true; } @@ -466,6 +523,8 @@ void GlobalizationNative_InitICUFunctions(void* icuuc, void* icuin, const char* FOR_ALL_ICU_FUNCTIONS ValidateICUDataCanLoad(); + + InitializeVariableMaxAndTopPointers(symbolVersion); } #undef PER_FUNCTION_BLOCK diff --git a/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal.h b/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal.h index 1ddaff5594aad..47e30e50fda6f 100644 --- a/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal.h +++ b/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal.h @@ -58,9 +58,32 @@ // (U_ICU_VERSION_MAJOR_NUM < 52) // The following APIs are not supported in the ICU versions less than 52. We need to define them manually. // We have to do runtime check before using the pointers to these APIs. That is why these are listed in the FOR_ALL_OPTIONAL_ICU_FUNCTIONS list. +U_CAPI void U_EXPORT2 ucol_setMaxVariable(UCollator* coll, UColReorderCode group, UErrorCode* pErrorCode); U_CAPI int32_t U_EXPORT2 ucal_getTimeZoneIDForWindowsID(const UChar* winid, int32_t len, const char* region, UChar* id, int32_t idCapacity, UErrorCode* status); U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len, UChar* winid, int32_t winidCapacity, UErrorCode* status); -#endif + +// (U_ICU_VERSION_MAJOR_NUM < 71) +// The following API is not supported in the ICU versions less than 71. We need to define it manually. +// We have to do runtime check before using the pointers to this API. That is why these are listed in the FOR_ALL_OPTIONAL_ICU_FUNCTIONS list. +U_CAPI UCollator* U_EXPORT2 ucol_clone(const UCollator* coll, UErrorCode* status); + +// ucol_setVariableTop is a deprecated function on the newer ICU versions and ucol_setMaxVariable should be used instead. +// As can run against ICU versions which not supported ucol_setMaxVariable, we'll dynamically try to get the pointer to ucol_setVariableTop +// when we couldn't get a pointer to ucol_setMaxVariable. +typedef uint32_t (U_EXPORT2 *ucol_setVariableTop_func)(UCollator* coll, const UChar* varTop, int32_t len, UErrorCode* status); + +// ucol_safeClone is deprecated in ICU version 71. We have to handle it manually to avoid getting a build break when referencing it in the code. +typedef UCollator* (U_EXPORT2 *ucol_safeClone_func)(const UCollator* coll, void* stackBuffer, int32_t* pBufferSize, UErrorCode* status); + +#else // !defined(TARGET_ANDROID) + +typedef uint32_t (*ucol_setVariableTop_func)(UCollator* coll, const UChar* varTop, int32_t len, UErrorCode* status); +typedef UCollator* (*ucol_safeClone_func)(const UCollator* coll, void* stackBuffer, int32_t* pBufferSize, UErrorCode* status); + +#endif // !defined(TARGET_ANDROID) + +extern ucol_setVariableTop_func ucol_setVariableTop_ptr; +extern ucol_safeClone_func ucol_safeClone_ptr; // List of all functions from the ICU libraries that are used in the System.Globalization.Native.so #define FOR_ALL_UNCONDITIONAL_ICU_FUNCTIONS \ @@ -99,7 +122,6 @@ U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len, PER_FUNCTION_BLOCK(ucol_open, libicui18n, true) \ PER_FUNCTION_BLOCK(ucol_openElements, libicui18n, true) \ PER_FUNCTION_BLOCK(ucol_openRules, libicui18n, true) \ - PER_FUNCTION_BLOCK(ucol_safeClone, libicui18n, true) \ PER_FUNCTION_BLOCK(ucol_setAttribute, libicui18n, true) \ PER_FUNCTION_BLOCK(ucol_strcoll, libicui18n, true) \ PER_FUNCTION_BLOCK(udat_close, libicui18n, true) \ @@ -164,15 +186,6 @@ U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len, PER_FUNCTION_BLOCK(usearch_setPattern, libicui18n, true) \ PER_FUNCTION_BLOCK(usearch_setText, libicui18n, true) -#if HAVE_SET_MAX_VARIABLE -#define FOR_ALL_SET_VARIABLE_ICU_FUNCTIONS \ - PER_FUNCTION_BLOCK(ucol_setMaxVariable, libicui18n, true) -#else - -#define FOR_ALL_SET_VARIABLE_ICU_FUNCTIONS \ - PER_FUNCTION_BLOCK(ucol_setVariableTop, libicui18n, true) -#endif - #if defined(TARGET_WINDOWS) #define FOR_ALL_OS_CONDITIONAL_ICU_FUNCTIONS \ PER_FUNCTION_BLOCK(ucurr_forLocale, libicuuc, true) \ @@ -195,11 +208,12 @@ U_CAPI int32_t U_EXPORT2 ucal_getWindowsTimeZoneID(const UChar* id, int32_t len, // Otherwise, we'll just not provide the functionality to users which needed these APIs. #define FOR_ALL_OPTIONAL_ICU_FUNCTIONS \ PER_FUNCTION_BLOCK(ucal_getWindowsTimeZoneID, libicui18n, false) \ - PER_FUNCTION_BLOCK(ucal_getTimeZoneIDForWindowsID, libicui18n, false) + PER_FUNCTION_BLOCK(ucal_getTimeZoneIDForWindowsID, libicui18n, false) \ + PER_FUNCTION_BLOCK(ucol_setMaxVariable, libicui18n, false) \ + PER_FUNCTION_BLOCK(ucol_clone, libicui18n, false) #define FOR_ALL_ICU_FUNCTIONS \ FOR_ALL_UNCONDITIONAL_ICU_FUNCTIONS \ - FOR_ALL_SET_VARIABLE_ICU_FUNCTIONS \ FOR_ALL_OPTIONAL_ICU_FUNCTIONS \ FOR_ALL_OS_CONDITIONAL_ICU_FUNCTIONS @@ -235,6 +249,7 @@ FOR_ALL_ICU_FUNCTIONS #define ucal_openTimeZoneIDEnumeration(...) ucal_openTimeZoneIDEnumeration_ptr(__VA_ARGS__) #define ucal_set(...) ucal_set_ptr(__VA_ARGS__) #define ucal_setMillis(...) ucal_setMillis_ptr(__VA_ARGS__) +#define ucol_clone(...) ucol_clone_ptr(__VA_ARGS__) #define ucol_close(...) ucol_close_ptr(__VA_ARGS__) #define ucol_closeElements(...) ucol_closeElements_ptr(__VA_ARGS__) #define ucol_getOffset(...) ucol_getOffset_ptr(__VA_ARGS__) @@ -247,13 +262,8 @@ FOR_ALL_ICU_FUNCTIONS #define ucol_open(...) ucol_open_ptr(__VA_ARGS__) #define ucol_openElements(...) ucol_openElements_ptr(__VA_ARGS__) #define ucol_openRules(...) ucol_openRules_ptr(__VA_ARGS__) -#define ucol_safeClone(...) ucol_safeClone_ptr(__VA_ARGS__) #define ucol_setAttribute(...) ucol_setAttribute_ptr(__VA_ARGS__) -#if HAVE_SET_MAX_VARIABLE #define ucol_setMaxVariable(...) ucol_setMaxVariable_ptr(__VA_ARGS__) -#else -#define ucol_setVariableTop(...) ucol_setVariableTop_ptr(__VA_ARGS__) -#endif #define ucol_strcoll(...) ucol_strcoll_ptr(__VA_ARGS__) #define ucurr_forLocale(...) ucurr_forLocale_ptr(__VA_ARGS__) #define ucurr_getName(...) ucurr_getName_ptr(__VA_ARGS__) diff --git a/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal_android.h b/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal_android.h index 1125ce91c3f80..424f72597dc7d 100644 --- a/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal_android.h +++ b/src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal_android.h @@ -470,7 +470,7 @@ int32_t ucol_previous(UCollationElements * elems, UErrorCode * status); UCollator * ucol_open(const char * loc, UErrorCode * status); UCollationElements * ucol_openElements(const UCollator * coll, const UChar * text, int32_t textLength, UErrorCode * status); UCollator * ucol_openRules(const UChar * rules, int32_t rulesLength, UColAttributeValue normalizationMode, UCollationStrength strength, UParseError * parseError, UErrorCode * status); -UCollator * ucol_safeClone(const UCollator * coll, void * stackBuffer, int32_t * pBufferSize, UErrorCode * status); +UCollator * ucol_clone(const UCollator * coll, UErrorCode * status); void ucol_setAttribute(UCollator * coll, UColAttribute attr, UColAttributeValue value, UErrorCode * status); UCollationResult ucol_strcoll(const UCollator * coll, const UChar * source, int32_t sourceLength, const UChar * target, int32_t targetLength); int32_t ucurr_forLocale(const char * locale, UChar * buff, int32_t buffCapacity, UErrorCode * ec);