diff --git a/Projects/Server/Client/BwtDecompress.cs b/Projects/Server/Client/BwtDecompress.cs new file mode 100644 index 0000000000..414b278fa6 --- /dev/null +++ b/Projects/Server/Client/BwtDecompress.cs @@ -0,0 +1,175 @@ +using System; +using System.IO; +using System.Numerics; +using System.Runtime.InteropServices; + +namespace Server; + +public static class BwtDecompress +{ + public static byte[] Decompress(Stream stream, int length) + { + var firstChar = (byte)stream.ReadByte(); + + Span table = GC.AllocateUninitializedArray(256 * 256); + Span output = GC.AllocateUninitializedArray(length); + BuildTable(table, firstChar); + + var i = 0; + while (stream.Position < stream.Length) + { + var currentValue = firstChar; + var value = table[currentValue]; + if (currentValue > 0) + { + do + { + table[currentValue] = table[currentValue - 1]; + } while (--currentValue > 0); + } + + table[0] = value; + + output[i++] = (byte)value; + firstChar = (byte)stream.ReadByte(); + } + + return InternalDecompress(output); + } + + private static void BuildTable(Span table, byte startValue) + { + var index = 0; + var firstByte = startValue; + byte secondByte = 0; + for (var i = 0; i < 256 * 256; i++) + { + var val = (ushort)(firstByte + (secondByte << 8)); + table[index++] = val; + + firstByte++; + if (firstByte == 0) + { + secondByte++; + } + } + + table.Sort(); + } + + private static byte[] InternalDecompress(Span input) + { + Span symbolTable = stackalloc byte[256]; + Span frequency = stackalloc byte[256]; + Span partialInput = stackalloc int[256 * 3]; + + for (var i = 0; i < 256; i++) + { + symbolTable[i] = (byte)i; + } + + MemoryMarshal.Cast(input)[..256].CopyTo(partialInput); + + var sum = 0; + for (var i = 0; i < 256; i++) + { + sum += partialInput[i]; + } + + var nonZeroCount = 256 - partialInput[..256].Count(0); + + Frequency(partialInput, frequency); + + for (int i = 0, m = 0; i < nonZeroCount; ++i) + { + var freq = frequency[i]; + symbolTable[input[m + 1024]] = freq; + partialInput[freq + 256] = m + 1; + m += partialInput[freq]; + partialInput[freq + 512] = m; + } + + var val = symbolTable[0]; + var output = GC.AllocateUninitializedArray(sum); + + var count = 0; + do + { + ref var firstValRef = ref partialInput[val + 256]; + output[count] = val; + + if (firstValRef >= partialInput[val + 512]) + { + if (nonZeroCount-- > 0) + { + ShiftLeftSimd(symbolTable, nonZeroCount); + val = symbolTable[0]; + } + } + else + { + var idx = input[firstValRef + 1024]; + firstValRef++; + + if (idx != 0) + { + ShiftLeftSimd(symbolTable, idx); + symbolTable[idx] = val; + val = symbolTable[0]; + } + } + + count++; + } while (count < sum); + + return output; + } + + private static void Frequency(Span input, Span output) + { + Span tmp = stackalloc int[256]; + input[..256].CopyTo(tmp); + + for (var i = 0; i < 256; i++) + { + uint value = 0; + byte index = 0; + + for (var j = 0; j < 256; j++) + { + if (tmp[j] > value) + { + index = (byte)j; + value = (uint)tmp[j]; + } + } + + if (value == 0) + { + break; + } + + output[i] = index; + tmp[index] = 0; + } + } + + private static void ShiftLeftSimd(Span input, int max) + { + var i = 0; + var vectorSize = Vector.Count; + + while (i + vectorSize <= max) + { + var vector = new Vector(input[(i + 1)..]); + vector.CopyTo(input[i..]); + i += vectorSize; + } + + while (i < max) + { + input[i] = input[i + 1]; + i++; + } + } +} diff --git a/Projects/Server/Localization/Localization.cs b/Projects/Server/Localization/Localization.cs index cae4e93b51..a92d69f249 100644 --- a/Projects/Server/Localization/Localization.cs +++ b/Projects/Server/Localization/Localization.cs @@ -14,6 +14,7 @@ *************************************************************************/ using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; @@ -96,28 +97,46 @@ private static Dictionary LoadClilocs(string lang, strin if (File.Exists(file)) { using var fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read); - using var bin = new BinaryReader(fs); + Span header = stackalloc byte[6]; + fs.Read(header); - bin.ReadInt32(); - bin.ReadInt16(); + byte[] data; + BufferReader br; + if (BinaryPrimitives.ReadInt32LittleEndian(header) != 2 || BinaryPrimitives.ReadInt16LittleEndian(header[4..]) != 1) + { + // Skip header + fs.Position = 4; + data = BwtDecompress.Decompress(fs, (int)fs.Length - 4); + br = new BufferReader(data); + + var header2 = br.ReadInt(); // Header 2 + var header1 = br.ReadShort(); // Header 1 + + if (header2 != 2 || header1 != 1) + { + throw new Exception($"Invalid cliloc header in {file}"); + } + } + else + { + data = GC.AllocateUninitializedArray((int)fs.Length - 6); + fs.Read(data); + br = new BufferReader(data); + } byte[] buffer = null; - while (bin.BaseStream.Length != bin.BaseStream.Position) + while (br.Position < data.Length) { - var number = bin.ReadInt32(); - var flag = bin.ReadByte(); // Original, Custom, Modified - var length = bin.ReadInt16(); + var number = br.ReadInt(); + var flag = br.ReadByte(); // Original, Custom, Modified + var length = br.ReadShort(); if (buffer == null || buffer.Length < length) { buffer = GC.AllocateUninitializedArray(length); } - var bytesRead = bin.Read(buffer, 0, length); - if (bytesRead != length) - { - throw new Exception($"Could not read enough bytes from {file}"); - } + br.Read(buffer.AsSpan(0, length)); var text = Encoding.UTF8.GetString(buffer.AsSpan(0, length)); entries[number] = new LocalizationEntry(lang, number, text);