Merge pull request JuliaLang#12360 from ScottPJones/spj/cesu8fix

Fix a bug handling CESU-8 strings in convert(UTF8String, Vector{UInt8}

base/unicode/utf8.jl

@@ -267,9 +267,9 @@ function convert(::Type{UTF8String}, dat::Vector{UInt8})
                 buf[out += 1] = dat[pos += 1]
             else
                 # Pick up surrogate pairs (CESU-8 format)
-                ch = (((((ch & 0x3f) << 6) | (dat[pos + 1] & 0x3f)) << 10)
-                        + (((dat[pos + 3] & 0x3f) << 6) | (dat[pos + 4] & 0x3f))
-                        - 0xc00)
+                ch = ((((((ch & 0x3f) << 6) | (dat[pos + 1] & 0x3f)) << 10)
+                       + (((dat[pos + 3] & 0x3f)%UInt32 << 6) | (dat[pos + 4] & 0x3f)))
+                      - 0x01f0c00)
                 pos += 4
                 output_utf8_4byte!(buf, out, ch)
                 out += 4

test/unicode.jl

@@ -1,6 +1,7 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
 include("unicode/checkstring.jl")
+include("unicode/utf8.jl")
 include("unicode/utf16.jl")
 include("unicode/utf32.jl")
 include("unicode/utf8proc.jl")

test/unicode/utf8.jl

@@ -0,0 +1,12 @@
+# This file is a part of Julia. License is MIT: http://julialang.org/license
+
+## Test for CESU-8 sequences
+
+let ch = 0x10000
+    for hichar = 0xd800:0xdbff
+        for lochar = 0xdc00:0xdfff
+            @test convert(UTF8String, utf8(Char[hichar, lochar]).data) == string(Char(ch))
+            ch += 1
+        end
+    end
+end