From 2d23e4d2d910b21e35acad3765e216bf96374e0b Mon Sep 17 00:00:00 2001 From: ScottPJones Date: Sun, 7 Jun 2015 11:11:23 +0200 Subject: [PATCH] Fix #10959 UTF-32 conversion errors Added new `convert` methods that use the `check_string` function to validate input Added tests for many sorts of valid/invalid data Depends on PR #11551 and #11575 --- base/utf32.jl | 260 ++++++++++++++++++++++++++++++++++++++++++++++-- test/strings.jl | 33 ++++-- 2 files changed, 277 insertions(+), 16 deletions(-) diff --git a/base/utf32.jl b/base/utf32.jl index 90962e2538ea81..4844a11c886122 100644 --- a/base/utf32.jl +++ b/base/utf32.jl @@ -5,20 +5,264 @@ next(s::UTF32String, i::Int) = (s.data[i], i+1) endof(s::UTF32String) = length(s.data) - 1 length(s::UTF32String) = length(s.data) - 1 +const empty_utf32 = UTF32String(UInt32[0]) + utf32(x) = convert(UTF32String, x) convert(::Type{UTF32String}, c::Char) = UTF32String(Char[c, Char(0)]) convert(::Type{UTF32String}, s::UTF32String) = s -function convert(::Type{UTF32String}, s::AbstractString) - a = Array(Char, length(s) + 1) - i = 0 - for c in s - a[i += 1] = c +" +Converts an AbstractString to a UTF16String + +### Input Arguments: +* ::Type{UTF32String} +* str::AbstractString + +### Returns: +* ::UTF32String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF32String}, str::AbstractString) + len, flags = check_string(str) + buf = Vector{Char}(len+1) + out = 0 + @inbounds for ch in str ; buf[out += 1] = ch ; end + @inbounds buf[out + 1] = 0 # NULL termination + UTF32String(buf) +end + +" +Converts a UTF-32 encoded vector of UInt32 to a UTF8String + +### Input Arguments: +* ::Type{UTF8String} +* dat::Vector{UInt32} + +### Returns: +* ::UTF8String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF8String}, dat::Vector{UInt32}) + len = sizeof(dat) + # handle zero length string quickly + len == 0 && return empty_utf8 + # get number of bytes to allocate + len, flags, num4byte, num3byte, num2byte = check_string(dat, len>>>2) + flags == 0 && @inbounds return UTF8String(copy!(Vector{UInt8}(len), 1, dat, 1, len)) + return encode_to_utf8(UInt32, dat, len + num2byte + num3byte*2 + num4byte*3) +end + +" +Converts a UTF32String to a UTF8String + +### Input Arguments: +* ::Type{UTF8String} +* str::UTF32String + +### Returns: +* ::UTF8String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF8String}, str::UTF32String) + dat = reinterpret(UInt32, str.data) + len = sizeof(dat) >>> 2 + # handle zero length string quickly + len <= 1 && return empty_utf8 + # get number of bytes to allocate + len, flags, num4byte, num3byte, num2byte = check_string(dat, len-1) + flags == 0 && @inbounds return UTF8String(copy!(Vector{UInt8}(len), 1, dat, 1, len)) + return encode_to_utf8(UInt32, dat, len + num2byte + num3byte*2 + num4byte*3) +end + +" +Converts a UTF8String to a UTF32String + +### Input Arguments: +* ::Type{UTF32String} +* str::UTF8String + +### Returns: +* ::UTF32String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF32String}, str::UTF8String) + dat = str.data + # handle zero length string quickly + sizeof(dat) == 0 && return empty_utf32 + # Validate UTF-8 encoding, and get number of words to create + len, flags = check_string(dat) + # Optimize case where no characters > 0x7f + totlen = len+1 + flags == 0 && return fast_utf_copy(UTF32String, Char, totlen, dat) + # has multi-byte UTF-8 sequences + buf = Vector{Char}(totlen) + @inbounds buf[totlen] = 0 # NULL termination + local ch::UInt32, surr::UInt32 + out = 0 + pos = 0 + @inbounds while out < len + ch = dat[pos += 1] + # Handle ASCII characters + if ch <= 0x7f + buf[out += 1] = ch + # Handle range 0x80-0x7ff + elseif ch < 0xe0 + buf[out += 1] = ((ch & 0x1f) << 6) | (dat[pos += 1] & 0x3f) + # Handle range 0x800-0xffff + elseif ch < 0xf0 + pos += 2 + ch = get_utf8_3byte(dat, pos, ch) + # Handle surrogate pairs (should have been encoded in 4 bytes) + if is_surrogate_lead(ch) + # Build up 32-bit character from ch and trailing surrogate in next 3 bytes + pos += 3 + surr = ((UInt32(dat[pos-2] & 0xf) << 12) + | (UInt32(dat[pos-1] & 0x3f) << 6) + | (dat[pos] & 0x3f)) + ch = get_supplementary(ch, surr) + end + buf[out += 1] = ch + # Handle range 0x10000-0x10ffff + else + pos += 3 + buf[out += 1] = get_utf8_4byte(dat, pos, ch) + end end - a[end] = Char(0) # NULL terminate - UTF32String(a) + UTF32String(buf) end +" +Converts a UTF16String to UTF32String + +### Input Arguments: +* ::Type{UTF32String} +* str::UTF16String + +### Returns: +* ::UTF32String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF32String}, str::UTF16String) + dat = str.data + len = sizeof(dat) + # handle zero length string quickly (account for trailing \0) + len <= 2 && return empty_utf32 + # get number of words to create + len, flags, num4byte = check_string(dat, len>>>1) + # No surrogate pairs, do optimized copy + (flags & UTF_UNICODE4) == 0 && @inbounds return UTF32String(copy!(Vector{Char}(len), dat)) + local ch::UInt32 + buf = Vector{Char}(len) + out = 0 + pos = 0 + @inbounds while out < len + ch = dat[pos += 1] + # check for surrogate pair + if is_surrogate_lead(ch) ; ch = get_supplementary(ch, dat[pos += 1]) ; end + buf[out += 1] = ch + end + UTF32String(buf) +end + +" +Converts a UTF-32 encoded vector of UInt32 to a UTF16String + +### Input Arguments: +* ::Type{UTF16String} +* dat::Vector{UInt32} + +### Returns: +* ::UTF16String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF16String}, dat::Vector{UInt32}) + len = sizeof(dat) + # handle zero length string quickly + len <= 4 && return empty_utf16 + # get number of words to allocate + len, flags, num4byte = check_string(dat, len>>>2) + len += num4byte + 1 + # optimized path, no surrogates + num4byte == 0 && return fast_utf_copy(UTF16String, UInt16, len, dat) + return encode_to_utf16(dat, len) +end + +" +Converts a UTF32String to UTF16String + +### Input Arguments: +* ::Type{UTF16String} +* str::UTF32String + +### Returns: +* ::UTF16String + +### Throws: +* UnicodeError +" +function convert(::Type{UTF16String}, str::UTF32String) + dat = reinterpret(UInt32, str.data) + len = sizeof(dat) + # handle zero length string quickly + len <= 4 && return empty_utf16 + # get number of words to allocate + len, flags, num4byte = check_string(dat, len>>>2) + # optimized path, no surrogates + num4byte == 0 && @inbounds return UTF16String(copy!(Vector{UInt16}(len), dat)) + return encode_to_utf16(dat, len + num4byte) +end + +" +Converts an already validated UTF-32 encoded vector of UInt32 to a UTF16String + +### Input Arguments: +* dat::Vector{UInt32} UTF-32 encoded data +* len length of output in 16-bit words + +### Returns: +* ::UTF16String +" +function encode_to_utf16(dat, len) + buf = Vector{UInt16}(len) + @inbounds buf[len] = 0 # NULL termination + out = 0 + pos = 0 + @inbounds while out < len + ch = UInt32(dat[pos += 1]) + if ch > 0xffff + # Output surrogate pair for 0x10000-0x10ffff + buf[out += 1] = 0xd7c0 + (ch >>> 10) + ch = 0xdc00 + (ch & 0x3ff) + end + buf[out += 1] = ch + end + UTF16String(buf) +end + +convert(::Type{UTF8String}, dat::Vector{Char}) = convert(UTF8String, reinterpret(UInt32, dat)) + +convert(::Type{UTF16String}, dat::Vector{Char}) = convert(UTF16String, reinterpret(UInt32, dat)) +convert(::Type{UTF32String}, c::Char) = UTF32String(Char[c, Char(0)]) + +function convert(::Type{UTF32String}, str::ASCIIString) + dat = str.data + fast_utf_copy(UTF32String, Char, length(dat)+1, dat) +end + +convert(::Type{UTF32String}, dat::AbstractVector{Char}) = fast_utf_copy(UTF32String, Char, length(dat), dat, true) + function convert(::Type{UTF32String}, data::AbstractVector{Char}) len = length(data) d = Array(Char, len + 1) @@ -51,7 +295,7 @@ unsafe_convert{T<:Union(Int32,UInt32,Char)}(::Type{Ptr{T}}, s::UTF32String) = convert(Ptr{T}, pointer(s)) function convert(T::Type{UTF32String}, bytes::AbstractArray{UInt8}) - isempty(bytes) && return UTF32String(Char[0]) + isempty(bytes) && return empty_utf32 length(bytes) & 3 != 0 && throw(UnicodeError(UTF_ERR_ODD_BYTES_32,0,0)) data = reinterpret(Char, bytes) # check for byte-order mark (BOM): diff --git a/test/strings.jl b/test/strings.jl index 31c4c928713b6b..caf271e0b922b3 100644 --- a/test/strings.jl +++ b/test/strings.jl @@ -1820,12 +1820,16 @@ byt = 0x0 @test_throws UnicodeError Base.check_string(UInt32[0x110000]) # issue #11551 (#11004,#10959) -function tstcvt(strUTF8::UTF8String, strUTF16::UTF16String) +function tstcvt(strUTF8::UTF8String, strUTF16::UTF16String, strUTF32::UTF32String) @test utf16(strUTF8) == strUTF16 + @test utf32(strUTF8) == strUTF32 @test utf8(strUTF16) == strUTF8 + @test utf32(strUTF16) == strUTF32 + @test utf8(strUTF32) == strUTF8 + @test utf16(strUTF32) == strUTF16 end -# Create some ASCII, UTF8 and UTF16 +# Create some ASCII, UTF8, UTF16, and UTF32 strings strAscii = "abcdefgh" strA_UTF8 = ("abcdefgh\uff")[1:8] strL_UTF8 = "abcdef\uff\uff" @@ -1844,27 +1848,40 @@ str3_UTF16 = utf16(str3_UTF8) str4_UTF16 = utf16(str4_UTF8) strS_UTF16 = utf16(strS_UTF8) +strA_UTF32 = utf32(strA_UTF8) +strL_UTF32 = utf32(strL_UTF8) +str2_UTF32 = utf32(str2_UTF8) +str3_UTF32 = utf32(str3_UTF8) +str4_UTF32 = utf32(str4_UTF8) +strS_UTF32 = utf32(strS_UTF8) + @test utf8(strAscii) == strAscii @test utf16(strAscii) == strAscii +@test utf32(strAscii) == strAscii -tstcvt(strA_UTF8,strA_UTF16) -tstcvt(strL_UTF8,strL_UTF16) -tstcvt(str2_UTF8,str2_UTF16) -tstcvt(str3_UTF8,str3_UTF16) -tstcvt(str4_UTF8,str4_UTF16) +tstcvt(strA_UTF8,strA_UTF16,strA_UTF32) +tstcvt(strL_UTF8,strL_UTF16,strL_UTF32) +tstcvt(str2_UTF8,str2_UTF16,str2_UTF32) +tstcvt(str3_UTF8,str3_UTF16,str3_UTF32) +tstcvt(str4_UTF8,str4_UTF16,str4_UTF32) # Test converting surrogate pairs @test utf16(strS_UTF8) == strC_UTF8 +@test utf32(strS_UTF8) == strC_UTF8 @test utf8(strS_UTF16) == strC_UTF8 +@test utf32(strS_UTF16) == strC_UTF8 +@test utf8(strS_UTF32) == strC_UTF8 +@test utf16(strS_UTF32) == strC_UTF8 # Test converting overlong \0 # @test utf8(strZ_UTF8) == strz_UTF8 # currently broken! (in utf8.jl) @test utf16(strZ_UTF8) == strz_UTF8 +@test utf32(strZ_UTF8) == strz_UTF8 # Test invalid sequences byt = 0x0 -for T in (UTF16String,) # UTF32String +for T in (UTF16String, UTF32String) try # Continuation byte not after lead for byt in 0x80:0xbf