From 637d4f941614f4738ec45146d889f2eeea5b0554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Sun, 18 Mar 2018 19:19:50 +0100 Subject: [PATCH 1/8] Commits from https://github.com/aspnet/Common/pull/334 --- Common.sln | 7 + ...soft.Extensions.Internal.Benchmarks.csproj | 29 + .../Properties/AssemblyInfo.cs | 4 + .../WebEncodersBenchmarks.cs | 80 ++ .../Properties/EncoderResources.cs | 10 + .../WebEncoders.cs | 1187 ++++++++++++++--- .../Microsoft.Extensions.Internal.Test.csproj | 1 + .../WebEncodersTests.cs | 270 ++++ 8 files changed, 1410 insertions(+), 178 deletions(-) create mode 100644 benchmarks/Microsoft.Extensions.Internal.Benchmarks/Microsoft.Extensions.Internal.Benchmarks.csproj create mode 100644 benchmarks/Microsoft.Extensions.Internal.Benchmarks/Properties/AssemblyInfo.cs create mode 100644 benchmarks/Microsoft.Extensions.Internal.Benchmarks/WebEncodersBenchmarks.cs diff --git a/Common.sln b/Common.sln index 9e34e08031a..7219dcc2edc 100644 --- a/Common.sln +++ b/Common.sln @@ -71,6 +71,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Internal.AspNetCore.Analyze EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Internal.AspNetCore.Analyzers.Tests", "test\Internal.AspNetCore.Analyzers.Tests\Internal.AspNetCore.Analyzers.Tests.csproj", "{1A579BD1-A4C4-4B1B-B092-D1670DF7F239}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Extensions.Internal.Benchmarks", "benchmarks\Microsoft.Extensions.Internal.Benchmarks\Microsoft.Extensions.Internal.Benchmarks.csproj", "{7EAB9B74-5E5E-4D93-BA1E-865906C50676}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -137,6 +139,10 @@ Global {1A579BD1-A4C4-4B1B-B092-D1670DF7F239}.Debug|Any CPU.Build.0 = Debug|Any CPU {1A579BD1-A4C4-4B1B-B092-D1670DF7F239}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A579BD1-A4C4-4B1B-B092-D1670DF7F239}.Release|Any CPU.Build.0 = Release|Any CPU + {7EAB9B74-5E5E-4D93-BA1E-865906C50676}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7EAB9B74-5E5E-4D93-BA1E-865906C50676}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7EAB9B74-5E5E-4D93-BA1E-865906C50676}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7EAB9B74-5E5E-4D93-BA1E-865906C50676}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -157,6 +163,7 @@ Global {B439E0C8-F892-4AC5-BBF7-63BCDAACA7A9} = {6878D8F1-6DCE-4677-AA1A-4D14BA6D2D60} {FACBCBCB-D043-4AE8-A22D-A683040999DD} = {FEAA3936-5906-4383-B750-F07FE1B156C5} {1A579BD1-A4C4-4B1B-B092-D1670DF7F239} = {6878D8F1-6DCE-4677-AA1A-4D14BA6D2D60} + {7EAB9B74-5E5E-4D93-BA1E-865906C50676} = {A9A93AF9-2113-4321-AD20-51F60FF8B2BD} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {371030CF-B541-4BA9-9F54-3C7563415CF1} diff --git a/benchmarks/Microsoft.Extensions.Internal.Benchmarks/Microsoft.Extensions.Internal.Benchmarks.csproj b/benchmarks/Microsoft.Extensions.Internal.Benchmarks/Microsoft.Extensions.Internal.Benchmarks.csproj new file mode 100644 index 00000000000..e98bdd99e03 --- /dev/null +++ b/benchmarks/Microsoft.Extensions.Internal.Benchmarks/Microsoft.Extensions.Internal.Benchmarks.csproj @@ -0,0 +1,29 @@ + + + + netcoreapp2.1 + Exe + true + true + false + latest + + + + + Shared\%(FileName)%(Extension) + + + + + + + + + + + + + + + diff --git a/benchmarks/Microsoft.Extensions.Internal.Benchmarks/Properties/AssemblyInfo.cs b/benchmarks/Microsoft.Extensions.Internal.Benchmarks/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..409fcf814af --- /dev/null +++ b/benchmarks/Microsoft.Extensions.Internal.Benchmarks/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +[assembly: BenchmarkDotNet.Attributes.AspNetCoreBenchmark] diff --git a/benchmarks/Microsoft.Extensions.Internal.Benchmarks/WebEncodersBenchmarks.cs b/benchmarks/Microsoft.Extensions.Internal.Benchmarks/WebEncodersBenchmarks.cs new file mode 100644 index 00000000000..51151ab8439 --- /dev/null +++ b/benchmarks/Microsoft.Extensions.Internal.Benchmarks/WebEncodersBenchmarks.cs @@ -0,0 +1,80 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; + +namespace Microsoft.Extensions.Internal.Benchmarks +{ + public class WebEncodersBenchmarks + { + private const int ByteArraySize = 500; + private readonly byte[] _data; + private readonly string _dataEncoded; + private readonly byte[] _dataWithOffset; + private readonly string _dataWithOffsetEncoded; + private readonly byte[] _guid; + private readonly string _guidEncoded; + + public WebEncodersBenchmarks() + { + var random = new Random(); + _data = new byte[ByteArraySize]; + random.NextBytes(_data); + _dataEncoded = WebEncoders.Base64UrlEncode(_data); + + _dataWithOffset = new byte[3].Concat(_data).Concat(new byte[2]).ToArray(); + _dataWithOffsetEncoded = "xx" + _dataEncoded + "yyy"; + + _guid = Guid.NewGuid().ToByteArray(); + _guidEncoded = WebEncoders.Base64UrlEncode(_guid); + } + + [Benchmark] + public byte[] Base64UrlDecode_Data() + { + return WebEncoders.Base64UrlDecode(_dataEncoded); + } + + [Benchmark] + public byte[] Base64UrlDecode_DataWithOffset() + { + return WebEncoders.Base64UrlDecode(_dataWithOffsetEncoded, 2, _dataEncoded.Length); + } + + [Benchmark] + public byte[] Base64UrlDecode_Guid() + { + return WebEncoders.Base64UrlDecode(_guidEncoded); + } + + [Benchmark] + public string Base64UrlEncode_Data() + { + return WebEncoders.Base64UrlEncode(_data); + } + + [Benchmark] + public string Base64UrlEncode_DataWithOffset() + { + return WebEncoders.Base64UrlEncode(_dataWithOffset, 3, _data.Length); + } + + [Benchmark] + public string Base64UrlEncode_Guid() + { + return WebEncoders.Base64UrlEncode(_guid); + } + + [Benchmark] + public int GetArraySizeRequiredToDecode() + { + return WebEncoders.GetArraySizeRequiredToDecode(ByteArraySize); + } + + [Benchmark] + public int GetArraySizeRequiredToEncode() + { + return WebEncoders.GetArraySizeRequiredToEncode(ByteArraySize); + } + } +} diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/Properties/EncoderResources.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/Properties/EncoderResources.cs index 3474ae82c5b..ca56e35599d 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/Properties/EncoderResources.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/Properties/EncoderResources.cs @@ -19,6 +19,16 @@ internal static class EncoderResources /// internal static readonly string WebEncoders_MalformedInput = "Malformed input: {0} is an invalid input length."; + /// + /// Invalid input, that doesn't conform a base64 string. + /// + internal static readonly string WebEncoders_InvalidInput = "The input is not a valid Base-64 string as it contains a non-base 64 character, more than two padding characters, or an illegal character among the padding characters."; + + /// + /// Destination buffer is too small. + /// + internal static readonly string WebEncoders_DestinationTooSmall = "The destination buffer is too small."; + /// /// Invalid {0}, {1} or {2} length. /// diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index 17068ae67a5..412ed5ae0c1 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -2,10 +2,17 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Buffers; +using System.Buffers.Text; using System.Diagnostics; -using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using Microsoft.Extensions.WebEncoders.Sources; +#if !NET461 +using System.Numerics; +#endif + #if WebEncoders_In_WebUtilities namespace Microsoft.AspNetCore.WebUtilities #else @@ -22,6 +29,8 @@ namespace Microsoft.Extensions.Internal #endif static class WebEncoders { + private const int MaxStackallocBytes = 256; + private const int MaxEncodedLength = (int.MaxValue / 4) * 3; // encode inflates the data by 4/3 private static readonly byte[] EmptyBytes = new byte[0]; /// @@ -37,10 +46,10 @@ public static byte[] Base64UrlDecode(string input) { if (input == null) { - throw new ArgumentNullException(nameof(input)); + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.input); } - return Base64UrlDecode(input, offset: 0, count: input.Length); + return Base64UrlDecode(input.AsSpan()); } /// @@ -56,23 +65,102 @@ public static byte[] Base64UrlDecode(string input) /// public static byte[] Base64UrlDecode(string input, int offset, int count) { - if (input == null) + if (input == null + || (uint)offset > (uint)input.Length + || (uint)count > (uint)(input.Length - offset)) { - throw new ArgumentNullException(nameof(input)); + ThrowInvalidArguments(input, offset, count); } - ValidateParameters(input.Length, nameof(input), offset, count); + return Base64UrlDecode(input.AsSpan(offset, count)); + } + /// + /// Decodes a base64url-encoded span of chars. + /// + /// The base64url-encoded input to decode. + /// The base64url-decoded form of the input. + /// + /// The input must not contain any whitespace or padding characters. + /// Throws if the input is malformed. + /// + public static byte[] Base64UrlDecode(ReadOnlySpan base64Url) + { // Special-case empty input - if (count == 0) + if (base64Url.IsEmpty) { return EmptyBytes; } - // Create array large enough for the Base64 characters, not just shorter Base64-URL-encoded form. - var buffer = new char[GetArraySizeRequiredToDecode(count)]; + var base64Len = GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength); + var data = new byte[dataLength]; + var written = Base64UrlDecodeCore(base64Url, data, base64Len); + Debug.Assert(data.Length == written); + + return data; + } + + /// + /// Decodes a base64url-encoded span of chars into a span of bytes. + /// + /// A span containing the base64url-encoded input to decode. + /// The base64url-decoded form of . + /// The number of the bytes written to . + /// + /// The input must not contain any whitespace or padding characters. + /// Throws if the input is malformed. + /// + public static int Base64UrlDecode(ReadOnlySpan base64Url, Span data) + { + // Special-case empty input + if (base64Url.IsEmpty) + { + return 0; + } + + var base64Len = GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength); + var written = Base64UrlDecodeCore(base64Url, data, base64Len); + Debug.Assert(dataLength == written); - return Base64UrlDecode(input, offset, buffer, bufferOffset: 0, count: count); + return written; + } + + /// + /// Decode the span of UTF-8 base64url-encoded text into binary data. + /// + /// The input span which contains UTF-8 base64url-encoded text that needs to be decoded. + /// The output span which contains the result of the operation, i.e. the decoded binary data. + /// The number of input bytes consumed during the operation. This can be used to slice the input for subsequent calls, if necessary. + /// The number of bytes written into the output span. This can be used to slice the output for subsequent calls, if necessary. + /// True (default) when the input span contains the entire data to decode. + /// Set to false only if it is known that the input span contains partial data with more data to follow. + /// It returns the OperationStatus enum values: + /// - Done - on successful processing of the entire input span + /// - DestinationTooSmall - if there is not enough space in the output span to fit the decoded input + /// - NeedMoreData - only if isFinalBlock is false and the input is not a multiple of 4, otherwise the partial input would be considered as InvalidData + /// - InvalidData - if the input contains bytes outside of the expected base 64 range, or if it contains invalid/more than two padding characters, + /// or if the input is incomplete (i.e. not a multiple of 4) and isFinalBlock is true. + public static OperationStatus Base64UrlDecode(ReadOnlySpan base64Url, Span data, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) + { + // Special-case empty input + if (base64Url.IsEmpty) + { + bytesConsumed = 0; + bytesWritten = 0; + return OperationStatus.Done; + } + + var base64Len = isFinalBlock + ? GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength) + : base64Url.Length; + + if (base64Len > MaxStackallocBytes / sizeof(byte)) + { + return Base64UrlDecodeCoreSlow(base64Url, data, base64Len, out bytesConsumed, out bytesWritten, isFinalBlock); + } + + Span base64 = stackalloc byte[base64Len]; + return Base64UrlDecodeCore(base64Url, base64, data, out bytesConsumed, out bytesWritten, isFinalBlock); } /// @@ -96,144 +184,193 @@ public static byte[] Base64UrlDecode(string input, int offset, int count) /// public static byte[] Base64UrlDecode(string input, int offset, char[] buffer, int bufferOffset, int count) { - if (input == null) + if (input == null + || (uint)offset > (uint)input.Length + || (uint)count > (uint)(input.Length - offset) + || buffer == null + || (uint)bufferOffset > (uint)buffer.Length) { - throw new ArgumentNullException(nameof(input)); - } - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - ValidateParameters(input.Length, nameof(input), offset, count); - if (bufferOffset < 0) - { - throw new ArgumentOutOfRangeException(nameof(bufferOffset)); + ThrowInvalidArguments(input, offset, count, buffer, bufferOffset, validateBuffer: true); } + // Special-case empty input if (count == 0) { return EmptyBytes; } - // Assumption: input is base64url encoded without padding and contains no whitespace. - - var paddingCharsToAdd = GetNumBase64PaddingCharsToAddForDecode(count); - var arraySizeRequired = checked(count + paddingCharsToAdd); - Debug.Assert(arraySizeRequired % 4 == 0, "Invariant: Array length must be a multiple of 4."); + var base64Len = GetBufferSizeRequiredToUrlDecode(count, out int dataLength); - if (buffer.Length - bufferOffset < arraySizeRequired) + if ((uint)buffer.Length < (uint)(bufferOffset + base64Len)) { - throw new ArgumentException( - string.Format( - CultureInfo.CurrentCulture, - EncoderResources.WebEncoders_InvalidCountOffsetOrLength, - nameof(count), - nameof(bufferOffset), - nameof(input)), - nameof(count)); + ThrowHelper.ThrowInvalidCountOffsetOrLengthException(ExceptionArgument.count, ExceptionArgument.bufferOffset, ExceptionArgument.input); } - // Copy input into buffer, fixing up '-' -> '+' and '_' -> '/'. - var i = bufferOffset; - for (var j = offset; i - bufferOffset < count; i++, j++) +#if NETCOREAPP2_1 + var data = new byte[dataLength]; + var base64 = buffer.AsSpan(bufferOffset, base64Len); + UrlCharsHelper.SubstituteUrlCharsForDecoding(input.AsSpan(offset, count), base64); + Convert.TryFromBase64Chars(base64, data, out int written); + Debug.Assert(written == dataLength); + + return data; +#else + UrlCharsHelper.SubstituteUrlCharsForDecoding(input.AsSpan(offset, count), buffer.AsSpan(bufferOffset, base64Len)); + return Convert.FromBase64CharArray(buffer, bufferOffset, base64Len); +#endif + } + + /// + /// Encodes using base64url-encoding. + /// + /// The binary input to encode. + /// The base64url-encoded form of . + public static string Base64UrlEncode(byte[] input) + { + if (input == null) { - var ch = input[j]; - if (ch == '-') - { - buffer[i] = '+'; - } - else if (ch == '_') - { - buffer[i] = '/'; - } - else - { - buffer[i] = ch; - } + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.input); } - // Add the padding characters back. - for (; paddingCharsToAdd > 0; i++, paddingCharsToAdd--) + return Base64UrlEncode(input.AsSpan()); + } + + /// + /// Encodes using base64url-encoding. + /// + /// The binary input to encode. + /// The offset into at which to begin encoding. + /// The number of bytes from to encode. + /// The base64url-encoded form of . + public static string Base64UrlEncode(byte[] input, int offset, int count) + { + if (input == null + || (uint)offset > (uint)input.Length + || (uint)count > (uint)(input.Length - offset)) { - buffer[i] = '='; + ThrowInvalidArguments(input, offset, count); } - // Decode. - // If the caller provided invalid base64 chars, they'll be caught here. - return Convert.FromBase64CharArray(buffer, bufferOffset, arraySizeRequired); + return Base64UrlEncode(input.AsSpan(offset, count)); } /// - /// Gets the minimum char[] size required for decoding of characters - /// with the method. + /// Encodes using base64url-encoding. /// - /// The number of characters to decode. - /// - /// The minimum char[] size required for decoding of characters. - /// - public static int GetArraySizeRequiredToDecode(int count) + /// The binary input to encode. + /// The base64url-encoded form of . + public static unsafe string Base64UrlEncode(ReadOnlySpan data) { - if (count < 0) + // Special-case empty input + if (data.IsEmpty) { - throw new ArgumentOutOfRangeException(nameof(count)); + return string.Empty; } - if (count == 0) + var base64Len = GetBufferSizeRequiredToBase64Encode(data.Length, out int numPaddingChars); +#if NETCOREAPP2_1 + fixed (byte* ptr = &MemoryMarshal.GetReference(data)) { - return 0; - } + return string.Create(base64Len - numPaddingChars, (Ptr: (IntPtr)ptr, data.Length, base64Len), (base64Url, state) => + { + var bytes = new ReadOnlySpan(state.Ptr.ToPointer(), state.Length); - var numPaddingCharsToAdd = GetNumBase64PaddingCharsToAddForDecode(count); + Base64UrlEncodeCore(bytes, base64Url, state.base64Len); + }); + } +#else +#if !NET461 + char[] arrayToReturnToPool = null; + try + { +#endif + var base64UrlLen = base64Len - numPaddingChars; + var base64Url = base64UrlLen <= MaxStackallocBytes / sizeof(char) + ? stackalloc char[base64UrlLen] +#if NET461 + : new char[base64UrlLen]; +#else + : arrayToReturnToPool = ArrayPool.Shared.Rent(base64UrlLen); +#endif + var urlEncodedLen = Base64UrlEncodeCore(data, base64Url, base64Len); + Debug.Assert(base64UrlLen == urlEncodedLen); - return checked(count + numPaddingCharsToAdd); + fixed (char* ptr = &MemoryMarshal.GetReference(base64Url)) + { + return new string(ptr, 0, urlEncodedLen); + } +#if !NET461 + } + finally + { + if (arrayToReturnToPool != null) + { + ArrayPool.Shared.Return(arrayToReturnToPool); + } + } +#endif +#endif } /// - /// Encodes using base64url encoding. + /// Encodes using base64url-encoding into . /// - /// The binary input to encode. - /// The base64url-encoded form of . - public static string Base64UrlEncode(byte[] input) + /// The binary input to encode. + /// The base64url-encoded form of . + /// The number of chars written to . + public static int Base64UrlEncode(ReadOnlySpan data, Span base64Url) { - if (input == null) + // Use base64url encoding with no padding characters. See RFC 4648, Sec. 5. + + // Special-case empty input + if (data.IsEmpty) { - throw new ArgumentNullException(nameof(input)); + return 0; } - return Base64UrlEncode(input, offset: 0, count: input.Length); + var base64Len = GetArraySizeRequiredToEncode(data.Length); + return Base64UrlEncodeCore(data, base64Url, base64Len); } /// - /// Encodes using base64url encoding. + /// Encode the span of binary data into UTF-8 base64url-encoded representation. /// - /// The binary input to encode. - /// The offset into at which to begin encoding. - /// The number of bytes from to encode. - /// The base64url-encoded form of . - public static string Base64UrlEncode(byte[] input, int offset, int count) + /// The input span which contains binary data that needs to be encoded. + /// + /// The output span which contains the result of the operation, i.e. the UTF-8 base64url-encoded text. + /// The span must be large enough to hold the full base64-encoded form of , included padding characters. + /// + /// The number of input bytes consumed during the operation. This can be used to slice the input for subsequent calls, if necessary. + /// The number of bytes written into the output span. This can be used to slice the output for subsequent calls, if necessary. + /// True (default) when the input span contains the entire data to decode. + /// Set to false only if it is known that the input span contains partial data with more data to follow. + /// It returns the OperationStatus enum values: + /// - Done - on successful processing of the entire input span + /// - DestinationTooSmall - if there is not enough space in the output span to fit the encoded input + /// - NeedMoreData - only if isFinalBlock is false, otherwise the output is padded if the input is not a multiple of 3 + /// It does not return InvalidData since that is not possible for base 64 encoding. + public static OperationStatus Base64UrlEncode(ReadOnlySpan data, Span base64Url, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) { - if (input == null) + // Special-case empty input + if (data.IsEmpty) { - throw new ArgumentNullException(nameof(input)); + bytesConsumed = 0; + bytesWritten = 0; + return OperationStatus.Done; } - ValidateParameters(input.Length, nameof(input), offset, count); + var status = Base64.EncodeToUtf8(data, base64Url, out bytesConsumed, out bytesWritten, isFinalBlock); - // Special-case empty input - if (count == 0) + if (status == OperationStatus.Done || status == OperationStatus.NeedMoreData) { - return string.Empty; + bytesWritten = UrlCharsHelper.SubstituteUrlCharsForEncoding(base64Url, bytesWritten); } - var buffer = new char[GetArraySizeRequiredToEncode(count)]; - var numBase64Chars = Base64UrlEncode(input, offset, buffer, outputOffset: 0, count: count); - - return new String(buffer, startIndex: 0, length: numBase64Chars); + return status; } /// - /// Encodes using base64url encoding. + /// Encodes using base64url-encoding. /// /// The binary input to encode. /// The offset into at which to begin encoding. @@ -252,137 +389,831 @@ public static string Base64UrlEncode(byte[] input, int offset, int count) /// public static int Base64UrlEncode(byte[] input, int offset, char[] output, int outputOffset, int count) { - if (input == null) + if (input == null + || (uint)offset > (uint)input.Length + || (uint)count > (uint)(input.Length - offset) + || output == null + || (uint)outputOffset > (uint)output.Length) + { + ThrowInvalidArguments(input, offset, count, output, outputOffset, ExceptionArgument.output, validateBuffer: true); + } + + var base64Len = GetArraySizeRequiredToEncode(count); + if ((uint)output.Length < (uint)(outputOffset + base64Len)) { - throw new ArgumentNullException(nameof(input)); + ThrowHelper.ThrowInvalidCountOffsetOrLengthException(ExceptionArgument.count, ExceptionArgument.outputOffset, ExceptionArgument.output); } - if (output == null) + + // Special-case empty input. + if (count == 0) { - throw new ArgumentNullException(nameof(output)); + return 0; } - ValidateParameters(input.Length, nameof(input), offset, count); - if (outputOffset < 0) + return Base64UrlEncodeCore(input.AsSpan(offset, count), output.AsSpan(outputOffset), base64Len); + } + + /// + /// Gets the minimum buffer size required for decoding of characters. + /// + /// The number of characters to decode. + /// + /// The minimum buffer size required for decoding of characters. + /// + /// + /// The returned buffer size is large enough to hold characters as well + /// as base64 padding characters. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetArraySizeRequiredToDecode(int count) + { + if (count < 0) { - throw new ArgumentOutOfRangeException(nameof(outputOffset)); + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); } - var arraySizeRequired = GetArraySizeRequiredToEncode(count); - if (output.Length - outputOffset < arraySizeRequired) + return count == 0 ? 0 : GetBufferSizeRequiredToUrlDecode(count, out int dataLength); + } + + /// + /// Gets the minimum output buffer size required for encoding bytes. + /// + /// The number of characters to encode. + /// + /// The minimum output buffer size required for encoding s. + /// + /// + /// The returned buffer size is large enough to hold bytes as well + /// as base64 padding characters. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetArraySizeRequiredToEncode(int count) + { + if ((uint)count > MaxEncodedLength) { - throw new ArgumentException( - string.Format( - CultureInfo.CurrentCulture, - EncoderResources.WebEncoders_InvalidCountOffsetOrLength, - nameof(count), - nameof(outputOffset), - nameof(output)), - nameof(count)); + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); } - // Special-case empty input. - if (count == 0) + return count == 0 ? 0 : GetBufferSizeRequiredToBase64Encode(count); + } + + private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data, int base64Len) + { + // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 + Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); + + if (base64Len > MaxStackallocBytes / sizeof(char)) { - return 0; + return Base64UrlDecodeCoreSlow(base64Url, data, base64Len); } - // Use base64url encoding with no padding characters. See RFC 4648, Sec. 5. + Span base64Bytes = stackalloc byte[base64Len]; + return Base64UrlDecodeCore(base64Url, base64Bytes, data); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int Base64UrlDecodeCoreSlow(ReadOnlySpan base64Url, Span data, int base64Len) + { + // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 + Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); +#if !NET461 + byte[] arrayToReturnToPool = null; + try + { + var base64Bytes = new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); + return Base64UrlDecodeCore(base64Url, base64Bytes, data); + } + finally + { + if (arrayToReturnToPool != null) + { + ArrayPool.Shared.Return(arrayToReturnToPool); + } + } +#else + var base64Bytes = new byte[base64Len]; + return Base64UrlDecodeCore(base64Url, base64Bytes, data); +#endif + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span base64Bytes, Span data) + { + UrlCharsHelper.SubstituteUrlCharsForDecoding(base64Url, base64Bytes); + var status = Base64.DecodeFromUtf8(base64Bytes, data, out int consumed, out int written); + + if (status != OperationStatus.Done) + { + ThrowHelper.ThrowOperationNotDone(status); + } + + return written; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static OperationStatus Base64UrlDecodeCoreSlow(ReadOnlySpan base64Url, Span data, int base64Len, out int bytesConsumed, out int bytesWritten, bool isFinalBlock) + { + // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 + Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); +#if NET461 + var base64 = new byte[base64Len]; + return Base64UrlDecodeCore(base64Url, base64, data, out bytesConsumed, out bytesWritten, isFinalBlock); +#else + byte[] arrayToReturnToPool = null; + try + { + var base64 = new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); + return Base64UrlDecodeCore(base64Url, base64, data, out bytesConsumed, out bytesWritten, isFinalBlock); + } + finally + { + if (arrayToReturnToPool != null) + { + ArrayPool.Shared.Return(arrayToReturnToPool); + } + } +#endif + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static OperationStatus Base64UrlDecodeCore(ReadOnlySpan base64Url, Span base64, Span data, out int consumed, out int written, bool isFinalBlock) + { + UrlCharsHelper.SubstituteUrlCharsForDecoding(base64Url, base64, isFinalBlock); + var status = Base64.DecodeFromUtf8(base64, data, out consumed, out written, isFinalBlock); + + if (status != OperationStatus.Done && status != OperationStatus.NeedMoreData) + { + ThrowHelper.ThrowOperationNotDone(status); + } + + // Fix bytesConsumed to match the input 'base64Url' (and not the 'base64') + consumed = base64Url.Length - (base64.Length - consumed); + + return status; + } + +#if NETCOREAPP2_1 + private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url, int base64Len) + { + // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 + Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); - // Start with default Base64 encoding. - var numBase64Chars = Convert.ToBase64CharArray(input, offset, count, output, outputOffset); + if (base64Len > MaxStackallocBytes / sizeof(char)) + { + return Base64UrlEncodeCoreSlow(data, base64Url, base64Len); + } + + Span base64 = stackalloc char[base64Len]; + Convert.TryToBase64Chars(data, base64, out int written); + return UrlCharsHelper.SubstituteUrlCharsForEncoding(base64, base64Url); + } - // Fix up '+' -> '-' and '/' -> '_'. Drop padding characters. - for (var i = outputOffset; i - outputOffset < numBase64Chars; i++) + [MethodImpl(MethodImplOptions.NoInlining)] + private static int Base64UrlEncodeCoreSlow(ReadOnlySpan data, Span base64Url, int base64Len) + { + // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 + Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); + + char[] arrayToReturnToPool = null; + try + { + var base64 = new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); + Convert.TryToBase64Chars(data, base64, out int written); + return UrlCharsHelper.SubstituteUrlCharsForEncoding(base64, base64Url); + } + finally { - var ch = output[i]; - if (ch == '+') + if (arrayToReturnToPool != null) { - output[i] = '-'; + ArrayPool.Shared.Return(arrayToReturnToPool); } - else if (ch == '/') + } + } +#else + private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url, int base64Len) + { + // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 + Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); +#if !NET461 + byte[] arrayToReturnToPool = null; + try + { +#endif + var base64Bytes = base64Len <= MaxStackallocBytes / sizeof(byte) + ? stackalloc byte[base64Len] +#if NET461 + : new byte[base64Len]; +#else + : new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); +#endif + var status = Base64.EncodeToUtf8(data, base64Bytes, out int consumed, out int written); + + if (status != OperationStatus.Done) { - output[i] = '_'; + ThrowHelper.ThrowOperationNotDone(status); } - else if (ch == '=') + + return UrlCharsHelper.SubstituteUrlCharsForEncoding(base64Bytes, base64Url); +#if !NET461 + } + finally + { + if (arrayToReturnToPool != null) { - // We've reached a padding character; truncate the remainder. - return i - outputOffset; + ArrayPool.Shared.Return(arrayToReturnToPool); } } +#endif + } +#endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetBufferSizeRequiredToUrlDecode(int urlEncodedLen, out int dataLength) + { + // Shortcut for Guid and other 16 byte data + if (urlEncodedLen == 22) + { + dataLength = 16; + return 24; + } + + var numPaddingChars = GetNumBase64PaddingCharsToAddForDecode(urlEncodedLen); + var base64Len = urlEncodedLen + numPaddingChars; + Debug.Assert(base64Len % 4 == 0, "Invariant: Array length must be a multiple of 4."); + dataLength = (base64Len >> 2) * 3 - numPaddingChars; - return numBase64Chars; + return base64Len; } - /// - /// Get the minimum output char[] size required for encoding - /// s with the method. - /// - /// The number of characters to encode. - /// - /// The minimum output char[] size required for encoding s. - /// - public static int GetArraySizeRequiredToEncode(int count) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetNumBase64PaddingCharsToAddForDecode(int urlEncodedLen) + { + // Calculation is: + // switch (inputLength % 4) + // 0 -> 0 + // 2 -> 2 + // 3 -> 1 + // default -> format exception + + var result = (4 - urlEncodedLen) & 3; + + if (result == 3) + { + ThrowHelper.ThrowMalformedInputException(urlEncodedLen); + } + + return result; + } + + private static int GetBufferSizeRequiredToBase64Encode(int dataLength) + { + // overflow conditions are already eliminated, so 'checked' is not necessary + Debug.Assert(dataLength >= 0 && dataLength <= MaxEncodedLength); + + var numWholeOrPartialInputBlocks = (dataLength + 2) / 3; + return numWholeOrPartialInputBlocks * 4; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetBufferSizeRequiredToBase64Encode(int dataLength, out int numPaddingChars) + { + // Shortcut for Guid and other 16 byte data + if (dataLength == 16) + { + numPaddingChars = 2; + return 24; + } + + numPaddingChars = GetNumBase64PaddingCharsAddedByEncode(dataLength); + return GetBufferSizeRequiredToBase64Encode(dataLength); + } + + private static int GetNumBase64PaddingCharsAddedByEncode(int dataLength) { - var numWholeOrPartialInputBlocks = checked(count + 2) / 3; - return checked(numWholeOrPartialInputBlocks * 4); + // Calculation is: + // switch (dataLength % 3) + // 0 -> 0 + // 1 -> 2 + // 2 -> 1 + + return dataLength % 3 == 0 ? 0 : 3 - dataLength % 3; } - private static int GetNumBase64PaddingCharsInString(string str) + private static void ThrowInvalidArguments(object input, int offset, int count, char[] buffer = null, int bufferOffset = 0, ExceptionArgument bufferName = ExceptionArgument.buffer, bool validateBuffer = false) { - // Assumption: input contains a well-formed base64 string with no whitespace. + throw GetInvalidArgumentsException(); - // base64 guaranteed have 0 - 2 padding characters. - if (str[str.Length - 1] == '=') + Exception GetInvalidArgumentsException() { - if (str[str.Length - 2] == '=') + if (input == null) + { + return ThrowHelper.GetArgumentNullException(ExceptionArgument.input); + } + + if (validateBuffer && buffer == null) + { + return ThrowHelper.GetArgumentNullException(bufferName); + } + + if (offset < 0) + { + return ThrowHelper.GetArgumentOutOfRangeException(ExceptionArgument.offset); + } + + if (count < 0) + { + return ThrowHelper.GetArgumentOutOfRangeException(ExceptionArgument.count); + } + + if (bufferOffset < 0) { - return 2; + return ThrowHelper.GetArgumentOutOfRangeException(ExceptionArgument.bufferOffset); } - return 1; + + return ThrowHelper.GetInvalidCountOffsetOrLengthException(ExceptionArgument.count, ExceptionArgument.offset, ExceptionArgument.input); } - return 0; } - private static int GetNumBase64PaddingCharsToAddForDecode(int inputLength) + // TODO: replace IntPtr and (int*) with nuint once available + internal static unsafe class UrlCharsHelper { - switch (inputLength % 4) - { - case 0: - return 0; - case 2: - return 2; - case 3: - return 1; - default: - throw new FormatException( - string.Format( - CultureInfo.CurrentCulture, - EncoderResources.WebEncoders_MalformedInput, - inputLength)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SubstituteUrlCharsForDecoding(ReadOnlySpan urlEncoded, Span base64, bool isFinalBlock = true) + { + ref var input = ref MemoryMarshal.GetReference(urlEncoded); + ref var output = ref MemoryMarshal.GetReference(base64); + + SubstituteUrlCharsForDecoding(ref input, ref output, urlEncoded.Length, base64.Length, isFinalBlock); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SubstituteUrlCharsForDecoding(ReadOnlySpan urlEncoded, Span base64) + { + ref var input = ref Unsafe.As(ref MemoryMarshal.GetReference(urlEncoded)); + ref var output = ref Unsafe.As(ref MemoryMarshal.GetReference(base64)); + + SubstituteUrlCharsForDecoding(ref input, ref output, urlEncoded.Length, base64.Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SubstituteUrlCharsForDecoding(ref T urlEncoded, ref T base64, int urlEncodedLen, int base64Len, bool isFinalBlock = true) where T : struct + { + // Copy input into base64, fixing up '-' -> '+' and '_' -> '/' and add padding. + + var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations + var n = (IntPtr)urlEncodedLen; + var m = i; +#if !NET461 + if (Vector.IsHardwareAccelerated && (int*)n >= (int*)Vector.Count) + { + m = (IntPtr)((int)(int*)n & ~(Vector.Count - 1)); + for (; (int*)i < (int*)m; i += Vector.Count) + { + var vec = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref urlEncoded, i))); + + if (typeof(T) == typeof(byte)) + { + vec = Substitute(vec, (T)(object)(byte)'-', (T)(object)(byte)'+'); + vec = Substitute(vec, (T)(object)(byte)'_', (T)(object)(byte)'/'); + } + else if (typeof(T) == typeof(ushort)) + { + vec = Substitute(vec, (T)(object)(ushort)'-', (T)(object)(ushort)'+'); + vec = Substitute(vec, (T)(object)(ushort)'_', (T)(object)(ushort)'/'); + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref base64, i)), vec); + } + } +#endif + m = (IntPtr)((int)(int*)n & ~3); + for (; (int*)i < (int*)m; i += 4) + { + SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 0); + SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 1); + SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 2); + SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 3); + } + + for (; (int*)i < (int*)n; i += 1) + { + SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i); + } + + if (isFinalBlock) + { + n = (IntPtr)base64Len; + + // There will be a maximum of 2 padding chars. + if ((int*)i < (int*)n) + { + Pad(ref base64, i); + + i += 1; + if ((int*)i < (int*)n) + { + Pad(ref base64, i); + } + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SubstituteUrlCharsForDecoding(ReadOnlySpan urlEncoded, Span base64) + { + // Copy input into base64, fixing up '-' -> '+' and '_' -> '/' and add padding. + + ref var input = ref MemoryMarshal.GetReference(urlEncoded); + ref var output = ref MemoryMarshal.GetReference(base64); + + var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations + var n = (IntPtr)urlEncoded.Length; + var m = i; +#if !NET461 + if (Vector.IsHardwareAccelerated && (int*)n >= (int*)(2 * Vector.Count)) + { + m = (IntPtr)((int)(int*)n & ~(2 * Vector.Count - 1)); + for (; (int*)i < (int*)m; i += 2 * Vector.Count) + { + ref var tmp = ref Unsafe.Add(ref input, i); + var charsVec1 = Unsafe.As>(ref tmp); + var charsVec2 = Unsafe.As>(ref Unsafe.Add(ref tmp, Vector.Count)); + var bytesVec = Vector.Narrow(charsVec1, charsVec2); + + bytesVec = Substitute(bytesVec, (byte)'-', (byte)'+'); + bytesVec = Substitute(bytesVec, (byte)'_', (byte)'/'); + + Unsafe.WriteUnaligned(ref Unsafe.Add(ref output, i), bytesVec); + } + } +#endif + m = (IntPtr)((int)(int*)n & ~3); + for (; (int*)i < (int*)m; i += 4) + { + SubstituteUrlCharsForDecoding(ref input, ref output, i + 0); + SubstituteUrlCharsForDecoding(ref input, ref output, i + 1); + SubstituteUrlCharsForDecoding(ref input, ref output, i + 2); + SubstituteUrlCharsForDecoding(ref input, ref output, i + 3); + } + + for (; (int*)i < (int*)n; i += 1) + { + SubstituteUrlCharsForDecoding(ref input, ref output, i); + } + + n = (IntPtr)base64.Length; + + // There will be a maximum of 2 padding chars. + if ((int*)i < (int*)n) + { + Pad(ref output, i); + + i += 1; + if ((int*)i < (int*)n) + { + Pad(ref output, i); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SubstituteUrlCharsForEncoding(Span base64, int count) + { + ref var r = ref MemoryMarshal.GetReference(base64); + + return SubstituteUrlCharsForEncoding(ref r, ref r, count); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SubstituteUrlCharsForEncoding(ReadOnlySpan base64, Span urlEncoded) + { + ref var input = ref Unsafe.As(ref MemoryMarshal.GetReference(base64)); + ref var output = ref Unsafe.As(ref MemoryMarshal.GetReference(urlEncoded)); + + return SubstituteUrlCharsForEncoding(ref input, ref output, base64.Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int SubstituteUrlCharsForEncoding(ref T base64, ref T urlEncoded, int base64Length) where T : struct + { + // Use base64url encoding with no padding characters. See RFC 4648, Sec. 5. + + var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations + var n = (IntPtr)base64Length; + var m = i; +#if !NET461 + if (Vector.IsHardwareAccelerated && (int*)n >= (int*)Vector.Count) + { + m = (IntPtr)((int)(int*)n & ~(Vector.Count - 1)); + for (; (int*)i < (int*)m; i += Vector.Count) + { + var vec = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref base64, i))); + + if (typeof(T) == typeof(byte)) + { + if (Vector.EqualsAny(vec, new Vector((T)(object)(byte)'='))) break; + + vec = Substitute(vec, (T)(object)(byte)'+', (T)(object)(byte)'-'); + vec = Substitute(vec, (T)(object)(byte)'/', (T)(object)(byte)'_'); + } + else if (typeof(T) == typeof(ushort)) + { + if (Vector.EqualsAny(vec, new Vector((T)(object)(ushort)'='))) break; + + vec = Substitute(vec, (T)(object)(ushort)'+', (T)(object)(ushort)'-'); + vec = Substitute(vec, (T)(object)(ushort)'/', (T)(object)(ushort)'_'); + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref urlEncoded, i)), vec); + } + } +#endif + // n is always a multiple of 4 + Debug.Assert((int)n % 4 == 0); + for (; (int*)i < (int*)n; i += 4) + { + if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 0)) goto Exit0; + if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 1)) goto Exit1; + if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 2)) goto Exit2; + if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 3)) goto Exit3; + } + goto Exit0; + + Exit3: i += 1; + Exit2: i += 1; + Exit1: i += 1; + Exit0: + return (int)(int*)i; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SubstituteUrlCharsForEncoding(ReadOnlySpan base64, Span urlEncoded) + { + // A subset of the ASCII-range can be assumed, so no need + // to call the encoders necessary for byte -> char + + ref var input = ref MemoryMarshal.GetReference(base64); + ref var output = ref MemoryMarshal.GetReference(MemoryMarshal.Cast(urlEncoded)); + + var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations + var n = (IntPtr)base64.Length; + var m = i; +#if !NET461 + if (Vector.IsHardwareAccelerated && (int*)n >= (int*)Vector.Count) + { + m = (IntPtr)((int)(int*)n & ~(Vector.Count - 1)); + for (; (int*)i < (int*)m; i += Vector.Count) + { + var bytesVec = Unsafe.As>(ref Unsafe.Add(ref input, i)); + + if (Vector.EqualsAny(bytesVec, new Vector((byte)'='))) + { + break; + } + + bytesVec = Substitute(bytesVec, (byte)'+', (byte)'-'); + bytesVec = Substitute(bytesVec, (byte)'/', (byte)'_'); + + Vector.Widen(bytesVec, out Vector charsVec1, out Vector charsVec2); + ref var tmp = ref Unsafe.Add(ref output, i); + Unsafe.WriteUnaligned(ref Unsafe.As(ref tmp), charsVec1); + Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref tmp, Vector.Count)), charsVec2); + } + } +#endif + // n is always a multiple of 4 + Debug.Assert((int)n == ((int)n & ~3)); + for (; (int*)i < (int*)n; i += 4) + { + if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 0)) goto Exit0; + if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 1)) goto Exit1; + if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 2)) goto Exit2; + if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 3)) goto Exit3; + } + goto Exit0; + + Exit3: i += 1; + Exit2: i += 1; + Exit1: i += 1; + Exit0: + return (int)(int*)i; + } +#if !NET461 + private static Vector Substitute(Vector vector, T match, T substitution) where T : struct + => Vector.ConditionalSelect( + Vector.Equals(vector, new Vector(match)), + new Vector(substitution), + vector); +#endif + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SubstituteUrlCharsForDecoding(ref TIn urlEncoded, ref TOut base64, IntPtr idx) + where TIn : struct + where TOut : struct + { + TIn tmp = Unsafe.Add(ref urlEncoded, idx); + int value = default; + + if (typeof(TIn) == typeof(byte)) + { + value = (byte)(object)tmp; + } + else if (typeof(TIn) == typeof(ushort)) + { + value = (ushort)(object)tmp; + } + else if (typeof(TIn) == typeof(char)) + { + value = (char)(object)tmp; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + var subst = value; + + if (value == '-') + { + subst = '+'; + } + else if (value == '_') + { + subst = '/'; + } + + if (typeof(TOut) == typeof(byte)) + { + Unsafe.Add(ref base64, idx) = (TOut)(object)(byte)subst; + } + else if (typeof(TOut) == typeof(ushort)) + { + Unsafe.Add(ref base64, idx) = (TOut)(object)(ushort)subst; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Pad(ref T base64, IntPtr idx) where T : struct + { + if (typeof(T) == typeof(byte)) + { + Unsafe.Add(ref base64, idx) = (T)(object)(byte)'='; + } + else if (typeof(T) == typeof(ushort)) + { + Unsafe.Add(ref base64, idx) = (T)(object)(ushort)'='; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool SubstituteUrlCharsForEncoding(ref TIn base64, ref TOut urlEncoded, IntPtr idx) + where TIn : struct + where TOut : struct + { + TIn tmp = Unsafe.Add(ref base64, idx); + int value = default; + + if (typeof(TIn) == typeof(byte)) + { + value = (byte)(object)tmp; + } + else if (typeof(TIn) == typeof(ushort)) + { + value = (ushort)(object)tmp; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + var subst = value; + + if (value == '+') + { + subst = '-'; + } + else if (value == '/') + { + subst = '_'; + } + else if (value == '=') + { + return true; + } + + if (typeof(TOut) == typeof(byte)) + { + Unsafe.Add(ref urlEncoded, idx) = (TOut)(object)(byte)subst; + } + else if (typeof(TOut) == typeof(ushort)) + { + Unsafe.Add(ref urlEncoded, idx) = (TOut)(object)(ushort)subst; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + return false; } } - private static void ValidateParameters(int bufferLength, string inputName, int offset, int count) + private static class ThrowHelper { - if (offset < 0) + public static void ThrowArgumentNullException(ExceptionArgument argument) { - throw new ArgumentOutOfRangeException(nameof(offset)); + throw GetArgumentNullException(argument); } - if (count < 0) + + public static void ThrowArgumentOutOfRangeException(ExceptionArgument argument) + { + throw GetArgumentOutOfRangeException(argument); + } + + public static void ThrowInvalidCountOffsetOrLengthException(ExceptionArgument arg1, ExceptionArgument arg2, ExceptionArgument arg3) { - throw new ArgumentOutOfRangeException(nameof(count)); + throw GetInvalidCountOffsetOrLengthException(arg1, arg2, arg3); } - if (bufferLength - offset < count) + + public static void ThrowMalformedInputException(int inputLength) + { + throw GetMalformdedInputException(inputLength); + } + + public static void ThrowOperationNotDone(OperationStatus status) + { + throw GetOperationNotDoneException(status); + } + + public static ArgumentNullException GetArgumentNullException(ExceptionArgument argument) + { + return new ArgumentNullException(GetArgumentName(argument)); + } + + public static ArgumentOutOfRangeException GetArgumentOutOfRangeException(ExceptionArgument argument) + { + return new ArgumentOutOfRangeException(GetArgumentName(argument)); + } + + public static ArgumentException GetInvalidCountOffsetOrLengthException(ExceptionArgument arg1, ExceptionArgument arg2, ExceptionArgument arg3) + { + return new ArgumentException(EncoderResources.FormatWebEncoders_InvalidCountOffsetOrLength( + GetArgumentName(arg1), + GetArgumentName(arg2), + GetArgumentName(arg3))); + } + + private static Exception GetOperationNotDoneException(OperationStatus status) + { + switch (status) + { + case OperationStatus.DestinationTooSmall: + return new InvalidOperationException(EncoderResources.WebEncoders_DestinationTooSmall); + case OperationStatus.InvalidData: + return new FormatException(EncoderResources.WebEncoders_InvalidInput); + default: // This case won't happen. + throw new NotSupportedException(); // Just in case new states are introduced + } + } + + private static string GetArgumentName(ExceptionArgument argument) { - throw new ArgumentException( - string.Format( - CultureInfo.CurrentCulture, - EncoderResources.WebEncoders_InvalidCountOffsetOrLength, - nameof(count), - nameof(offset), - inputName), - nameof(count)); + Debug.Assert(Enum.IsDefined(typeof(ExceptionArgument), argument), + "The enum value is not defined, please check the ExceptionArgument Enum."); + + return argument.ToString(); + } + + private static FormatException GetMalformdedInputException(int inputLength) + { + return new FormatException(EncoderResources.FormatWebEncoders_MalformedInput(inputLength)); } } + + private enum ExceptionArgument + { + input, + buffer, + output, + count, + offset, + bufferOffset, + outputOffset + } } } diff --git a/test/Microsoft.Extensions.Internal.Test/Microsoft.Extensions.Internal.Test.csproj b/test/Microsoft.Extensions.Internal.Test/Microsoft.Extensions.Internal.Test.csproj index 376775ec074..d098b7bb105 100755 --- a/test/Microsoft.Extensions.Internal.Test/Microsoft.Extensions.Internal.Test.csproj +++ b/test/Microsoft.Extensions.Internal.Test/Microsoft.Extensions.Internal.Test.csproj @@ -4,6 +4,7 @@ $(StandardTestTfms) portable true + latest diff --git a/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs b/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs index 5c71403fd65..5f23bb83bfd 100644 --- a/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs +++ b/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs @@ -3,12 +3,68 @@ using System; using System.Linq; +using System.Buffers; using Xunit; +using System.Text; namespace Microsoft.Extensions.Internal { public class WebEncodersTests { + // Taken from https://github.com/aspnet/HttpAbstractions/pull/926 + [Fact] + public void DataOfVariousLength_RoundTripCorrectly() + { + for (var length = 0; length < 256; length++) + { + var data = new byte[length]; + for (var i = 0; i < length; i++) + { + data[i] = (byte)(5 + length + (i * 23)); + } + + string text = WebEncoders.Base64UrlEncode(data); + byte[] result = WebEncoders.Base64UrlDecode(text); + + for (var i = 0; i < length; i++) + { + Assert.Equal(data[i], result[i]); + } + } + } + + [Fact] + public void DataOfVariousLengthAsSpan_RoundTripCorrectly() + { + for (var length = 0; length < 256; length++) + { + var data = new byte[length]; + for (var i = 0; i < length; i++) + { + data[i] = (byte)(5 + length + (i * 23)); + } + + var num = WebEncoders.GetArraySizeRequiredToEncode(data.Length); + var utf8Buffer = new byte[num].AsSpan(); + var status = WebEncoders.Base64UrlEncode(data, utf8Buffer, out int bytesConsumed, out int bytesWritten); + Assert.Equal(OperationStatus.Done, status); + Assert.Equal(data.Length, bytesConsumed); + + utf8Buffer = utf8Buffer.Slice(0, bytesWritten); + num = WebEncoders.GetArraySizeRequiredToDecode(utf8Buffer.Length); + var byteBuffer = new byte[num].AsSpan(); + status = WebEncoders.Base64UrlDecode(utf8Buffer, byteBuffer, out bytesConsumed, out bytesWritten); + Assert.Equal(OperationStatus.Done, status); + Assert.Equal(utf8Buffer.Length, bytesConsumed); + var result = byteBuffer.Slice(0, bytesWritten).ToArray(); + + for (var i = 0; i < length; i++) + { + Assert.Equal(data[i], result[i]); + } + } + } + [Theory] [InlineData("", 1, 0)] [InlineData("", 0, 1)] @@ -36,6 +92,17 @@ public void Base64UrlDecode_MalformedInput(string input) }); } + [Fact] + public void Base64UrlDecodeAsSpan_InputIsEmptyReturns0() + { + var input = string.Empty.AsSpan(); + var output = new byte[100].AsSpan(); + + var result = WebEncoders.Base64UrlDecode(input, output); + + Assert.Equal(0, result); + } + [Theory] [InlineData("", "")] [InlineData("123456qwerty++//X+/x", "123456qwerty--__X-_x")] @@ -109,5 +176,208 @@ public void Base64UrlEncode_BadOffsets(int inputLength, int offset, int count) var retVal = WebEncoders.Base64UrlEncode(input, offset, count); }); } + + [Theory] + [InlineData(0, 0)] + [InlineData(2, 4)] + [InlineData(3, 4)] + [InlineData(4, 4)] + [InlineData(6, 8)] + [InlineData(7, 8)] + public void GetArraySizeRequiredToDecode(int inputLength, int expectedLength) + { + var result = WebEncoders.GetArraySizeRequiredToDecode(inputLength); + + Assert.Equal(expectedLength, result); + } + + [Fact] + public void GetArraySizeRequiredToDecode_NegativeInputLength_Throws() + { + var exception = Assert.Throws(() => WebEncoders.GetArraySizeRequiredToDecode(-1)); + Assert.Equal("count", exception.ParamName); + } + + [Theory] + [InlineData(1)] + [InlineData(5)] + public void GetArraySizeRequiredToDecode_MalformedInputLength(int inputLength) + { + Assert.Throws(() => + { + var retVal = WebEncoders.GetArraySizeRequiredToDecode(inputLength); + }); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(2, 4)] + [InlineData(3, 4)] + [InlineData(4, 8)] + [InlineData(6, 8)] + [InlineData(7, 12)] + [InlineData(16, 24)] + public void GetArraySizeRequiredToEncode(int inputLength, int expectedLength) + { + var result = WebEncoders.GetArraySizeRequiredToEncode(inputLength); + + Assert.Equal(expectedLength, result); + } + + [Fact] + public void GetArraySizeRequiredToEncode_NegativeInputLength_Throws() + { + var exception = Assert.Throws(() => WebEncoders.GetArraySizeRequiredToEncode(-1)); + Assert.Equal("count", exception.ParamName); + } + + [Fact] + public void GetArraySizeRequiredToEncode_InputLengthTooBig_Throws() + { + var exception = Assert.Throws(() => WebEncoders.GetArraySizeRequiredToEncode((int.MaxValue / 4) * 3 + 1)); + Assert.Equal("count", exception.ParamName); + } + + // Taken from https://github.com/aspnet/HttpAbstractions/pull/926 + [Theory] + [InlineData("_", "/")] + [InlineData("-", "+")] + [InlineData("a-b-c", "a+b+c=")] + [InlineData("a_b_c_d", "a/b/c/d=")] + [InlineData("a-b_c", "a+b/c==")] + [InlineData("a-b_c-d", "a+b/c+d=")] + [InlineData("abcd", "abcd")] + public void SubstituteUrlCharsForDecoding_ReturnsValid_Base64String(string text, string expectedValue) + { +// To test the alternate code-path +#if NETCOREAPP2_1 + // Arrange + Span bytes = stackalloc byte[text.Length]; + Encoding.ASCII.GetBytes(text.AsSpan(), bytes); + Span expected = stackalloc byte[expectedValue.Length]; + Encoding.ASCII.GetBytes(expectedValue.AsSpan(), expected); + Span result = stackalloc byte[expectedValue.Length]; + + // Act + WebEncoders.UrlCharsHelper.SubstituteUrlCharsForDecoding(bytes, result); + + // Assert + Assert.True(expected.SequenceEqual(result)); +#else + // Arrange + var buffer = new char[expectedValue.Length]; + + // Act + WebEncoders.UrlCharsHelper.SubstituteUrlCharsForDecoding(text.AsSpan(), buffer); + + // Assert + Assert.Equal(expectedValue, new string(buffer)); +#endif + } + + // Taken from https://github.com/aspnet/HttpAbstractions/pull/926 + // Input length must be a multiple of 4 + [Theory] + [InlineData(" ", " ")] + [InlineData("+ ", "- ")] + [InlineData("/ ", "_ ")] + [InlineData("= ", "")] + [InlineData("== ", "")] + [InlineData("a+b+c+==", "a-b-c-")] + [InlineData("a/b/c== ", "a_b_c")] + [InlineData("a+b/c== ", "a-b_c")] + [InlineData("a+b/c ", "a-b_c ")] + [InlineData("abcd", "abcd")] + public void SubstituteUrlCharsForEncoding_Replaces_UrlEncodableCharacters(string base64EncodedValue, string expectedValue) + { +// To test the alternate code-path +#if NETCOREAPP2_1 + // Arrange + Span bytes = stackalloc byte[base64EncodedValue.Length]; + Encoding.ASCII.GetBytes(base64EncodedValue.AsSpan(), bytes); + Span expected = stackalloc byte[expectedValue.Length]; + Encoding.ASCII.GetBytes(expectedValue.AsSpan(), expected); + + // Act + var result = WebEncoders.UrlCharsHelper.SubstituteUrlCharsForEncoding(bytes, bytes.Length); + + // Assert + Assert.True(expected.SequenceEqual(bytes.Slice(0, result))); +#else + // Arrange + var buffer = base64EncodedValue.ToCharArray(); + + // Act + var result = WebEncoders.UrlCharsHelper.SubstituteUrlCharsForEncoding(buffer, buffer); + + // Assert + Assert.Equal(expectedValue.Length, result); + for (var i = 0; i < result; i++) + { + Assert.Equal(expectedValue[i], buffer[i]); + } +#endif + } + + [Fact] + public void Base64UrlDecode_BufferChain() + { + // Arrange + var data = new byte[20]; + var rnd = new Random(0); + rnd.NextBytes(data); + var base64UrlString = WebEncoders.Base64UrlEncode(data); + var base64Url = new byte[base64UrlString.Length]; + Encoding.ASCII.GetBytes(base64UrlString, 0, base64UrlString.Length, base64Url, 0); + + var size = WebEncoders.GetArraySizeRequiredToDecode(base64Url.Length); + var bytes = new byte[size]; + + // Act + var status = WebEncoders.Base64UrlDecode(base64Url.AsSpan(0, base64Url.Length / 2), bytes.AsSpan(), out int consumed, out int written1, isFinalBlock: false); + Assert.Equal(OperationStatus.NeedMoreData, status); + status = WebEncoders.Base64UrlDecode(base64Url.AsSpan(consumed), bytes.AsSpan(written1), out consumed, out int written2, isFinalBlock: true); + Assert.Equal(OperationStatus.Done, status); + + // Assert + var expected = data; + var actual = bytes.AsSpan(0, written1 + written2); + Assert.Equal(expected.Length, actual.Length); +#if !NET461 + Assert.True(expected.AsSpan().SequenceEqual(actual)); +#else + Assert.Equal(string.Join(",", expected), string.Join(",", actual.ToArray())); +#endif + } + + [Fact] + public void Base64UrlEncode_BufferChain() + { + // Arrange + var data = new byte[200]; + var rnd = new Random(0); + rnd.NextBytes(data); + + var size = WebEncoders.GetArraySizeRequiredToEncode(data.Length); + var base64Url = new byte[size]; + + // Act + var status = WebEncoders.Base64UrlEncode(data.AsSpan(0, data.Length / 2), base64Url.AsSpan(), out int consumed, out int written1, isFinalBlock: false); + Assert.Equal(OperationStatus.NeedMoreData, status); + status = WebEncoders.Base64UrlEncode(data.AsSpan(consumed), base64Url.AsSpan(written1), out consumed, out int written2, isFinalBlock: true); + Assert.Equal(OperationStatus.Done, status); + + // Assert + var expected = WebEncoders.Base64UrlEncode(data); + Assert.Equal(expected.Length, written1 + written2); + var chars = new char[expected.Length]; + Encoding.ASCII.GetChars(base64Url, 0, written1 + written2, chars, 0); +#if NETCOREAPP2_1 + var actual = new String(chars); +#else + var actual = new String(chars.ToArray()); +#endif + Assert.Equal(expected, actual); + } } } From 355fc8bde89b473a713908d4c30d54c4a3e9fcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Tue, 10 Apr 2018 21:27:14 +0200 Subject: [PATCH 2/8] UrlEncoder.Encode and Decode added --- .../WebEncoders.cs | 480 +++++++++++++++++- 1 file changed, 461 insertions(+), 19 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index 412ed5ae0c1..b8c26975d55 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -628,21 +628,29 @@ private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base6 #endif [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetBufferSizeRequiredToUrlDecode(int urlEncodedLen, out int dataLength) + private static int GetBufferSizeRequiredToUrlDecode(int urlEncodedLen, out int dataLength, bool isFinalBlock = true) { - // Shortcut for Guid and other 16 byte data - if (urlEncodedLen == 22) + if (isFinalBlock) { - dataLength = 16; - return 24; - } + // Shortcut for Guid and other 16 byte data + if (urlEncodedLen == 22) + { + dataLength = 16; + return 24; + } - var numPaddingChars = GetNumBase64PaddingCharsToAddForDecode(urlEncodedLen); - var base64Len = urlEncodedLen + numPaddingChars; - Debug.Assert(base64Len % 4 == 0, "Invariant: Array length must be a multiple of 4."); - dataLength = (base64Len >> 2) * 3 - numPaddingChars; + var numPaddingChars = GetNumBase64PaddingCharsToAddForDecode(urlEncodedLen); + var base64Len = urlEncodedLen + numPaddingChars; + Debug.Assert(base64Len % 4 == 0, "Invariant: Array length must be a multiple of 4."); + dataLength = (base64Len >> 2) * 3 - numPaddingChars; - return base64Len; + return base64Len; + } + else + { + dataLength = (urlEncodedLen >> 2) * 3; + return urlEncodedLen; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -734,6 +742,440 @@ Exception GetInvalidArgumentsException() } } + internal static class UrlEncoder + { + public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data, out int consumed, out int written, bool isFinalBlock = true) + { + ref T source = ref MemoryMarshal.GetReference(urlEncoded); + ref byte destBytes = ref MemoryMarshal.GetReference(data); + + int base64Len = GetBufferSizeRequiredToUrlDecode(urlEncoded.Length, out int dataLength, isFinalBlock); + int srcLength = base64Len & ~0x3; // only decode input up to closest multiple of 4. + int destLength = data.Length; + + int sourceIndex = 0; + int destIndex = 0; + + if (urlEncoded.Length == 0) + { + goto DoneExit; + } + + ref sbyte decodingMap = ref s_decodingMap[0]; + + // Last bytes could have padding characters, so process them separately and treat them as valid only if isFinalBlock is true. + // If isFinalBlock is false, padding characters are considered invalid. + int skipLastChunk = isFinalBlock ? 4 : 0; + + int maxSrcLength = 0; + if (destLength >= dataLength) + { + maxSrcLength = srcLength - skipLastChunk; + } + else + { + // This should never overflow since destLength here is less than int.MaxValue / 4 * 3. + // Therefore, (destLength / 3) * 4 will always be less than int.MaxValue. + maxSrcLength = (destLength / 3) * 4; + } + + while (sourceIndex < maxSrcLength) + { + int result = DecodeFour(ref Unsafe.Add(ref source, sourceIndex), ref decodingMap); + + if (result < 0) goto InvalidExit; + + WriteThreeLowOrderBytes(ref destBytes, destIndex, result); + destIndex += 3; + sourceIndex += 4; + } + + if (maxSrcLength != srcLength - skipLastChunk) + { + goto DestinationSmallExit; + } + + // If input is less than 4 bytes, srcLength == sourceIndex == 0 + // If input is not a multiple of 4, sourceIndex == srcLength != 0 + if (sourceIndex == srcLength) + { + if (isFinalBlock) + { + goto InvalidExit; + } + + goto NeedMoreExit; + } + + // If isFinalBlock is false, we will never reach this point. + + // Handle last four bytes. There are 0, 1, 2 padding chars. + int numPaddingChars = base64Len - urlEncoded.Length; + ref T lastFourStart = ref Unsafe.Add(ref source, srcLength - 4); + + if (numPaddingChars == 0) + { + int result = DecodeFour(ref lastFourStart, ref decodingMap); + + if (result < 0) goto InvalidExit; + if (destIndex > destLength - 3) goto DestinationSmallExit; + + WriteThreeLowOrderBytes(ref destBytes, destIndex, result); + destIndex += 3; + sourceIndex += 4; + } + else if (numPaddingChars == 1) + { + int result = DecodeThree(ref lastFourStart, ref decodingMap); + + if (result < 0) goto InvalidExit; + if (destIndex > destLength - 2) goto DestinationSmallExit; + + WriteTwoLowOrderBytes(ref destBytes, destIndex, result); + destIndex += 2; + sourceIndex += 3; + } + else + { + int result = DecodeTwo(ref lastFourStart, ref decodingMap); + + if (result < 0) goto InvalidExit; + if (destIndex > destLength - 1) goto DestinationSmallExit; + + WriteOneLowOrderByte(ref destBytes, destIndex, result); + destIndex += 1; + sourceIndex += 2; + } + + if (srcLength != base64Len) + { + goto InvalidExit; + } + + DoneExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.Done; + + DestinationSmallExit: + if (srcLength != urlEncoded.Length && isFinalBlock) + { + goto InvalidExit; // if input is not a multiple of 4, and there is no more data, return invalid data instead + } + consumed = sourceIndex; + written = destIndex; + return OperationStatus.DestinationTooSmall; + + NeedMoreExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.NeedMoreData; + + InvalidExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.InvalidData; + } + + public static OperationStatus Encode(ReadOnlySpan data, Span urlEncoded, out int consumed, out int written, bool isFinalBlock = true) + { + ref byte srcBytes = ref MemoryMarshal.GetReference(data); + ref T destination = ref MemoryMarshal.GetReference(urlEncoded); + + int srcLength = data.Length; + int destLength = urlEncoded.Length; + + int maxSrcLength = -2; + if (srcLength <= MaxEncodedLength && destLength >= GetBufferSizeRequiredToBase64Encode(srcLength, out int numPaddingChars) - numPaddingChars) + { + maxSrcLength += srcLength; + } + else + { + maxSrcLength += (destLength >> 2) * 3; + } + + int sourceIndex = 0; + int destIndex = 0; + + ref byte encodingMap = ref s_encodingMap[0]; + + while (sourceIndex < maxSrcLength) + { + EncodeThreeBytes(ref Unsafe.Add(ref srcBytes, sourceIndex), ref Unsafe.Add(ref destination, destIndex), ref encodingMap); + destIndex += 4; + sourceIndex += 3; + } + + if (maxSrcLength != srcLength - 2) + goto DestinationSmallExit; + + if (!isFinalBlock) + goto NeedMoreDataExit; + + if (sourceIndex == srcLength - 1) + { + EncodeOneByte(ref Unsafe.Add(ref srcBytes, sourceIndex), ref Unsafe.Add(ref destination, destIndex), ref encodingMap); + destIndex += 2; + sourceIndex += 1; + } + else if (sourceIndex == srcLength - 2) + { + EncodeTwoBytes(ref Unsafe.Add(ref srcBytes, sourceIndex), ref Unsafe.Add(ref destination, destIndex), ref encodingMap); + destIndex += 3; + sourceIndex += 2; + } + + consumed = sourceIndex; + written = destIndex; + return OperationStatus.Done; + + NeedMoreDataExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.NeedMoreData; + + DestinationSmallExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.DestinationTooSmall; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int DecodeFour(ref T encoded, ref sbyte decodingMap) + { + int i0, i1, i2, i3; + + if (typeof(T) == typeof(byte)) + { + ref byte tmp = ref Unsafe.As(ref encoded); + i0 = Unsafe.Add(ref tmp, 0); + i1 = Unsafe.Add(ref tmp, 1); + i2 = Unsafe.Add(ref tmp, 2); + i3 = Unsafe.Add(ref tmp, 3); + } + else if (typeof(T) == typeof(char)) + { + ref char tmp = ref Unsafe.As(ref encoded); + i0 = Unsafe.Add(ref tmp, 0); + i1 = Unsafe.Add(ref tmp, 1); + i2 = Unsafe.Add(ref tmp, 2); + i3 = Unsafe.Add(ref tmp, 3); + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + i0 = Unsafe.Add(ref decodingMap, i0); + i1 = Unsafe.Add(ref decodingMap, i1); + i2 = Unsafe.Add(ref decodingMap, i2); + i3 = Unsafe.Add(ref decodingMap, i3); + + return i0 << 18 + | i1 << 12 + | i2 << 6 + | i3; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int DecodeThree(ref T encoded, ref sbyte decodingMap) + { + int i0, i1, i2; + + if (typeof(T) == typeof(byte)) + { + ref byte tmp = ref Unsafe.As(ref encoded); + i0 = Unsafe.Add(ref tmp, 0); + i1 = Unsafe.Add(ref tmp, 1); + i2 = Unsafe.Add(ref tmp, 2); + } + else if (typeof(T) == typeof(char)) + { + ref char tmp = ref Unsafe.As(ref encoded); + i0 = Unsafe.Add(ref tmp, 0); + i1 = Unsafe.Add(ref tmp, 1); + i2 = Unsafe.Add(ref tmp, 2); + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + i0 = Unsafe.Add(ref decodingMap, i0); + i1 = Unsafe.Add(ref decodingMap, i1); + i2 = Unsafe.Add(ref decodingMap, i2); + + return i0 << 18 + | i1 << 12 + | i2 << 6; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int DecodeTwo(ref T encoded, ref sbyte decodingMap) + { + int i0, i1; + + if (typeof(T) == typeof(byte)) + { + ref byte tmp = ref Unsafe.As(ref encoded); + i0 = Unsafe.Add(ref tmp, 0); + i1 = Unsafe.Add(ref tmp, 1); + } + else if (typeof(T) == typeof(char)) + { + ref char tmp = ref Unsafe.As(ref encoded); + i0 = Unsafe.Add(ref tmp, 0); + i1 = Unsafe.Add(ref tmp, 1); + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + + i0 = Unsafe.Add(ref decodingMap, i0); + i1 = Unsafe.Add(ref decodingMap, i1); + + return i0 << 18 + | i1 << 12; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void WriteThreeLowOrderBytes(ref byte destination, int destIndex, int value) + { + Unsafe.Add(ref destination, destIndex + 0) = (byte)(value >> 16); + Unsafe.Add(ref destination, destIndex + 1) = (byte)(value >> 8); + Unsafe.Add(ref destination, destIndex + 2) = (byte)value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void WriteTwoLowOrderBytes(ref byte destination, int destIndex, int value) + { + Unsafe.Add(ref destination, destIndex + 0) = (byte)(value >> 16); + Unsafe.Add(ref destination, destIndex + 1) = (byte)(value >> 8); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void WriteOneLowOrderByte(ref byte destination, int destIndex, int value) + { + Unsafe.Add(ref destination, destIndex) = (byte)(value >> 16); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EncodeThreeBytes(ref byte threeBytes, ref T encoded, ref byte encodingMap) + { + int i = (threeBytes << 16) | (Unsafe.Add(ref threeBytes, 1) << 8) | Unsafe.Add(ref threeBytes, 2); + + int i0 = Unsafe.Add(ref encodingMap, i >> 18); + int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + int i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); + int i3 = Unsafe.Add(ref encodingMap, i & 0x3F); + + if (typeof(T) == typeof(byte)) + { + i = i0 | (i1 << 8) | (i2 << 16) | (i3 << 24); + Unsafe.WriteUnaligned(ref Unsafe.As(ref encoded), i); + } + else if (typeof(T) == typeof(char)) + { + ref char enc = ref Unsafe.As(ref encoded); + Unsafe.Add(ref enc, 0) = (char)i0; + Unsafe.Add(ref enc, 1) = (char)i1; + Unsafe.Add(ref enc, 2) = (char)i2; + Unsafe.Add(ref enc, 3) = (char)i3; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EncodeTwoBytes(ref byte twoBytes, ref T encoded, ref byte encodingMap) + { + int i = (twoBytes << 16) | (Unsafe.Add(ref twoBytes, 1) << 8); + + int i0 = Unsafe.Add(ref encodingMap, i >> 18); + int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + int i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); + + if (typeof(T) == typeof(byte)) + { + ref byte enc = ref Unsafe.As(ref encoded); + Unsafe.Add(ref enc, 0) = (byte)i0; + Unsafe.Add(ref enc, 1) = (byte)i1; + Unsafe.Add(ref enc, 2) = (byte)i2; + } + else if (typeof(T) == typeof(char)) + { + ref char enc = ref Unsafe.As(ref encoded); + Unsafe.Add(ref enc, 0) = (char)i0; + Unsafe.Add(ref enc, 1) = (char)i1; + Unsafe.Add(ref enc, 2) = (char)i2; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EncodeOneByte(ref byte oneByte, ref T encoded, ref byte encodingMap) + { + int i = (oneByte << 16); + + int i0 = Unsafe.Add(ref encodingMap, i >> 18); + int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + + if (typeof(T) == typeof(byte)) + { + ref byte enc = ref Unsafe.As(ref encoded); + Unsafe.Add(ref enc, 0) = (byte)i0; + Unsafe.Add(ref enc, 1) = (byte)i1; + } + else if (typeof(T) == typeof(char)) + { + ref char enc = ref Unsafe.As(ref encoded); + Unsafe.Add(ref enc, 0) = (char)i0; + Unsafe.Add(ref enc, 1) = (char)i1; + } + else + { + throw new NotSupportedException(); // just in case new types are introduced in the future + } + } + + private static readonly sbyte[] s_decodingMap = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 + }; + + private static readonly byte[] s_encodingMap = { + 65/*A*/, 66/*B*/, 67/*C*/, 68/*D*/, 69/*E*/, 70/*F*/, 71/*G*/, 72/*H*/, + 73/*I*/, 74/*J*/, 75/*K*/, 76/*L*/, 77/*M*/, 78/*N*/, 79/*O*/, 80/*P*/, + 81/*Q*/, 82/*R*/, 83/*S*/, 84/*T*/, 85/*U*/, 86/*V*/, 87/*W*/, 88/*X*/, + 89/*Y*/, 90/*Z*/, 97/*a*/, 98/*b*/, 99/*c*/, 100/*d*/, 101/*e*/, 102/*f*/, + 103/*g*/, 104/*h*/, 105/*i*/, 106/*j*/, 107/*k*/, 108/*l*/, 109/*m*/, 110/*n*/, + 111/*o*/, 112/*p*/, 113/*q*/, 114/*r*/, 115/*s*/, 116/*t*/, 117/*u*/, 118/*v*/, + 119/*w*/, 120/*x*/, 121/*y*/, 122/*z*/, 48/*0*/, 49/*1*/, 50/*2*/, 51/*3*/, + 52/*4*/, 53/*5*/, 54/*6*/, 55/*7*/, 56/*8*/, 57/*9*/, 45/*-*/, 95/*_*/ + }; + } + // TODO: replace IntPtr and (int*) with nuint once available internal static unsafe class UrlCharsHelper { @@ -947,10 +1389,10 @@ private static int SubstituteUrlCharsForEncoding(ref T base64, ref T urlEncod } goto Exit0; - Exit3: i += 1; - Exit2: i += 1; - Exit1: i += 1; - Exit0: + Exit3: i += 1; + Exit2: i += 1; + Exit1: i += 1; + Exit0: return (int)(int*)i; } @@ -1000,10 +1442,10 @@ public static int SubstituteUrlCharsForEncoding(ReadOnlySpan base64, Span< } goto Exit0; - Exit3: i += 1; - Exit2: i += 1; - Exit1: i += 1; - Exit0: + Exit3: i += 1; + Exit2: i += 1; + Exit1: i += 1; + Exit0: return (int)(int*)i; } #if !NET461 From 0fbee68d85ee636757788b0921d1ec55cc05e297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Thu, 12 Apr 2018 17:52:56 +0200 Subject: [PATCH 3/8] Encode and Decode uses UrlEncoder --- .../WebEncoders.cs | 222 +++--------------- 1 file changed, 31 insertions(+), 191 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index b8c26975d55..a992ad0fdcf 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -94,7 +94,7 @@ public static byte[] Base64UrlDecode(ReadOnlySpan base64Url) var base64Len = GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength); var data = new byte[dataLength]; - var written = Base64UrlDecodeCore(base64Url, data, base64Len); + var written = Base64UrlDecodeCore(base64Url, data); Debug.Assert(data.Length == written); return data; @@ -118,11 +118,7 @@ public static int Base64UrlDecode(ReadOnlySpan base64Url, Span data) return 0; } - var base64Len = GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength); - var written = Base64UrlDecodeCore(base64Url, data, base64Len); - Debug.Assert(dataLength == written); - - return written; + return Base64UrlDecodeCore(base64Url, data); } /// @@ -150,17 +146,7 @@ public static OperationStatus Base64UrlDecode(ReadOnlySpan base64Url, Span return OperationStatus.Done; } - var base64Len = isFinalBlock - ? GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength) - : base64Url.Length; - - if (base64Len > MaxStackallocBytes / sizeof(byte)) - { - return Base64UrlDecodeCoreSlow(base64Url, data, base64Len, out bytesConsumed, out bytesWritten, isFinalBlock); - } - - Span base64 = stackalloc byte[base64Len]; - return Base64UrlDecodeCore(base64Url, base64, data, out bytesConsumed, out bytesWritten, isFinalBlock); + return Base64UrlDecodeCore(base64Url, data, out bytesConsumed, out bytesWritten, isFinalBlock); } /// @@ -206,18 +192,11 @@ public static byte[] Base64UrlDecode(string input, int offset, char[] buffer, in ThrowHelper.ThrowInvalidCountOffsetOrLengthException(ExceptionArgument.count, ExceptionArgument.bufferOffset, ExceptionArgument.input); } -#if NETCOREAPP2_1 var data = new byte[dataLength]; - var base64 = buffer.AsSpan(bufferOffset, base64Len); - UrlCharsHelper.SubstituteUrlCharsForDecoding(input.AsSpan(offset, count), base64); - Convert.TryFromBase64Chars(base64, data, out int written); - Debug.Assert(written == dataLength); + var written = Base64UrlDecodeCore(input.AsSpan(offset, count), data); + Debug.Assert(dataLength == written); return data; -#else - UrlCharsHelper.SubstituteUrlCharsForDecoding(input.AsSpan(offset, count), buffer.AsSpan(bufferOffset, base64Len)); - return Convert.FromBase64CharArray(buffer, bufferOffset, base64Len); -#endif } /// @@ -268,14 +247,15 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) } var base64Len = GetBufferSizeRequiredToBase64Encode(data.Length, out int numPaddingChars); + var base64UrlLen = base64Len - numPaddingChars; #if NETCOREAPP2_1 fixed (byte* ptr = &MemoryMarshal.GetReference(data)) { - return string.Create(base64Len - numPaddingChars, (Ptr: (IntPtr)ptr, data.Length, base64Len), (base64Url, state) => + return string.Create(base64UrlLen, (Ptr: (IntPtr)ptr, data.Length), (base64Url, state) => { var bytes = new ReadOnlySpan(state.Ptr.ToPointer(), state.Length); - - Base64UrlEncodeCore(bytes, base64Url, state.base64Len); + var urlEncodedLen = Base64UrlEncodeCore(bytes, base64Url); + Debug.Assert(base64Url.Length == urlEncodedLen); }); } #else @@ -284,7 +264,7 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) try { #endif - var base64UrlLen = base64Len - numPaddingChars; + var base64Url = base64UrlLen <= MaxStackallocBytes / sizeof(char) ? stackalloc char[base64UrlLen] #if NET461 @@ -292,7 +272,7 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) #else : arrayToReturnToPool = ArrayPool.Shared.Rent(base64UrlLen); #endif - var urlEncodedLen = Base64UrlEncodeCore(data, base64Url, base64Len); + var urlEncodedLen = Base64UrlEncodeCore(data, base64Url); Debug.Assert(base64UrlLen == urlEncodedLen); fixed (char* ptr = &MemoryMarshal.GetReference(base64Url)) @@ -328,8 +308,7 @@ public static int Base64UrlEncode(ReadOnlySpan data, Span base64Url) return 0; } - var base64Len = GetArraySizeRequiredToEncode(data.Length); - return Base64UrlEncodeCore(data, base64Url, base64Len); + return Base64UrlEncodeCore(data, base64Url); } /// @@ -359,14 +338,9 @@ public static OperationStatus Base64UrlEncode(ReadOnlySpan data, Span.Encode(data, base64Url, out bytesConsumed, out bytesWritten, isFinalBlock); } /// @@ -410,7 +384,7 @@ public static int Base64UrlEncode(byte[] input, int offset, char[] output, int o return 0; } - return Base64UrlEncodeCore(input.AsSpan(offset, count), output.AsSpan(outputOffset), base64Len); + return Base64UrlEncodeCore(input.AsSpan(offset, count), output.AsSpan(outputOffset)); } /// @@ -449,58 +423,13 @@ public static int GetArraySizeRequiredToDecode(int count) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int GetArraySizeRequiredToEncode(int count) { - if ((uint)count > MaxEncodedLength) - { - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); - } - return count == 0 ? 0 : GetBufferSizeRequiredToBase64Encode(count); } - private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data, int base64Len) - { - // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 - Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); - - if (base64Len > MaxStackallocBytes / sizeof(char)) - { - return Base64UrlDecodeCoreSlow(base64Url, data, base64Len); - } - - Span base64Bytes = stackalloc byte[base64Len]; - return Base64UrlDecodeCore(base64Url, base64Bytes, data); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static int Base64UrlDecodeCoreSlow(ReadOnlySpan base64Url, Span data, int base64Len) - { - // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 - Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); -#if !NET461 - byte[] arrayToReturnToPool = null; - try - { - var base64Bytes = new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); - return Base64UrlDecodeCore(base64Url, base64Bytes, data); - } - finally - { - if (arrayToReturnToPool != null) - { - ArrayPool.Shared.Return(arrayToReturnToPool); - } - } -#else - var base64Bytes = new byte[base64Len]; - return Base64UrlDecodeCore(base64Url, base64Bytes, data); -#endif - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span base64Bytes, Span data) + private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data) { - UrlCharsHelper.SubstituteUrlCharsForDecoding(base64Url, base64Bytes); - var status = Base64.DecodeFromUtf8(base64Bytes, data, out int consumed, out int written); + var status = UrlEncoder.Decode(base64Url, data, out int consumed, out int written); if (status != OperationStatus.Done) { @@ -510,122 +439,31 @@ private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span return written; } - [MethodImpl(MethodImplOptions.NoInlining)] - private static OperationStatus Base64UrlDecodeCoreSlow(ReadOnlySpan base64Url, Span data, int base64Len, out int bytesConsumed, out int bytesWritten, bool isFinalBlock) - { - // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 - Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); -#if NET461 - var base64 = new byte[base64Len]; - return Base64UrlDecodeCore(base64Url, base64, data, out bytesConsumed, out bytesWritten, isFinalBlock); -#else - byte[] arrayToReturnToPool = null; - try - { - var base64 = new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); - return Base64UrlDecodeCore(base64Url, base64, data, out bytesConsumed, out bytesWritten, isFinalBlock); - } - finally - { - if (arrayToReturnToPool != null) - { - ArrayPool.Shared.Return(arrayToReturnToPool); - } - } -#endif - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static OperationStatus Base64UrlDecodeCore(ReadOnlySpan base64Url, Span base64, Span data, out int consumed, out int written, bool isFinalBlock) + private static OperationStatus Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data, out int consumed, out int written, bool isFinalBlock) { - UrlCharsHelper.SubstituteUrlCharsForDecoding(base64Url, base64, isFinalBlock); - var status = Base64.DecodeFromUtf8(base64, data, out consumed, out written, isFinalBlock); + var status = UrlEncoder.Decode(base64Url, data, out consumed, out written, isFinalBlock); if (status != OperationStatus.Done && status != OperationStatus.NeedMoreData) { ThrowHelper.ThrowOperationNotDone(status); } - // Fix bytesConsumed to match the input 'base64Url' (and not the 'base64') - consumed = base64Url.Length - (base64.Length - consumed); - return status; } -#if NETCOREAPP2_1 - private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url, int base64Len) - { - // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 - Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); - - if (base64Len > MaxStackallocBytes / sizeof(char)) - { - return Base64UrlEncodeCoreSlow(data, base64Url, base64Len); - } - - Span base64 = stackalloc char[base64Len]; - Convert.TryToBase64Chars(data, base64, out int written); - return UrlCharsHelper.SubstituteUrlCharsForEncoding(base64, base64Url); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static int Base64UrlEncodeCoreSlow(ReadOnlySpan data, Span base64Url, int base64Len) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url) { - // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 - Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); + var status = UrlEncoder.Encode(data, base64Url, out int consumed, out int written); - char[] arrayToReturnToPool = null; - try - { - var base64 = new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); - Convert.TryToBase64Chars(data, base64, out int written); - return UrlCharsHelper.SubstituteUrlCharsForEncoding(base64, base64Url); - } - finally + if (status != OperationStatus.Done) { - if (arrayToReturnToPool != null) - { - ArrayPool.Shared.Return(arrayToReturnToPool); - } + ThrowHelper.ThrowOperationNotDone(status); } - } -#else - private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url, int base64Len) - { - // Internal usage, so it can be assumed that base64Len is exactely a multiple of 4 - Debug.Assert(base64Len % 4 == 0, "base64Len must be multiple of 4"); -#if !NET461 - byte[] arrayToReturnToPool = null; - try - { -#endif - var base64Bytes = base64Len <= MaxStackallocBytes / sizeof(byte) - ? stackalloc byte[base64Len] -#if NET461 - : new byte[base64Len]; -#else - : new Span(arrayToReturnToPool = ArrayPool.Shared.Rent(base64Len), 0, base64Len); -#endif - var status = Base64.EncodeToUtf8(data, base64Bytes, out int consumed, out int written); - if (status != OperationStatus.Done) - { - ThrowHelper.ThrowOperationNotDone(status); - } - - return UrlCharsHelper.SubstituteUrlCharsForEncoding(base64Bytes, base64Url); -#if !NET461 - } - finally - { - if (arrayToReturnToPool != null) - { - ArrayPool.Shared.Return(arrayToReturnToPool); - } - } -#endif + return written; } -#endif [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int GetBufferSizeRequiredToUrlDecode(int urlEncodedLen, out int dataLength, bool isFinalBlock = true) @@ -673,12 +511,14 @@ private static int GetNumBase64PaddingCharsToAddForDecode(int urlEncodedLen) return result; } - private static int GetBufferSizeRequiredToBase64Encode(int dataLength) + private static int GetBufferSizeRequiredToBase64Encode(int count) { - // overflow conditions are already eliminated, so 'checked' is not necessary - Debug.Assert(dataLength >= 0 && dataLength <= MaxEncodedLength); + if ((uint)count > MaxEncodedLength) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); + } - var numWholeOrPartialInputBlocks = (dataLength + 2) / 3; + var numWholeOrPartialInputBlocks = (count + 2) / 3; return numWholeOrPartialInputBlocks * 4; } From 5b4d531ee6f051c7234fa39bdd86110da6ee60d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Mon, 16 Apr 2018 14:54:09 +0200 Subject: [PATCH 4/8] UrlCharHelper removed, is not needed anymore --- .../WebEncoders.cs | 399 ------------------ .../WebEncodersTests.cs | 81 ---- 2 files changed, 480 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index a992ad0fdcf..17c46904548 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -1016,405 +1016,6 @@ private static void EncodeOneByte(ref byte oneByte, ref T encoded, ref byte enco }; } - // TODO: replace IntPtr and (int*) with nuint once available - internal static unsafe class UrlCharsHelper - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SubstituteUrlCharsForDecoding(ReadOnlySpan urlEncoded, Span base64, bool isFinalBlock = true) - { - ref var input = ref MemoryMarshal.GetReference(urlEncoded); - ref var output = ref MemoryMarshal.GetReference(base64); - - SubstituteUrlCharsForDecoding(ref input, ref output, urlEncoded.Length, base64.Length, isFinalBlock); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SubstituteUrlCharsForDecoding(ReadOnlySpan urlEncoded, Span base64) - { - ref var input = ref Unsafe.As(ref MemoryMarshal.GetReference(urlEncoded)); - ref var output = ref Unsafe.As(ref MemoryMarshal.GetReference(base64)); - - SubstituteUrlCharsForDecoding(ref input, ref output, urlEncoded.Length, base64.Length); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void SubstituteUrlCharsForDecoding(ref T urlEncoded, ref T base64, int urlEncodedLen, int base64Len, bool isFinalBlock = true) where T : struct - { - // Copy input into base64, fixing up '-' -> '+' and '_' -> '/' and add padding. - - var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations - var n = (IntPtr)urlEncodedLen; - var m = i; -#if !NET461 - if (Vector.IsHardwareAccelerated && (int*)n >= (int*)Vector.Count) - { - m = (IntPtr)((int)(int*)n & ~(Vector.Count - 1)); - for (; (int*)i < (int*)m; i += Vector.Count) - { - var vec = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref urlEncoded, i))); - - if (typeof(T) == typeof(byte)) - { - vec = Substitute(vec, (T)(object)(byte)'-', (T)(object)(byte)'+'); - vec = Substitute(vec, (T)(object)(byte)'_', (T)(object)(byte)'/'); - } - else if (typeof(T) == typeof(ushort)) - { - vec = Substitute(vec, (T)(object)(ushort)'-', (T)(object)(ushort)'+'); - vec = Substitute(vec, (T)(object)(ushort)'_', (T)(object)(ushort)'/'); - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - - Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref base64, i)), vec); - } - } -#endif - m = (IntPtr)((int)(int*)n & ~3); - for (; (int*)i < (int*)m; i += 4) - { - SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 0); - SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 1); - SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 2); - SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i + 3); - } - - for (; (int*)i < (int*)n; i += 1) - { - SubstituteUrlCharsForDecoding(ref urlEncoded, ref base64, i); - } - - if (isFinalBlock) - { - n = (IntPtr)base64Len; - - // There will be a maximum of 2 padding chars. - if ((int*)i < (int*)n) - { - Pad(ref base64, i); - - i += 1; - if ((int*)i < (int*)n) - { - Pad(ref base64, i); - } - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SubstituteUrlCharsForDecoding(ReadOnlySpan urlEncoded, Span base64) - { - // Copy input into base64, fixing up '-' -> '+' and '_' -> '/' and add padding. - - ref var input = ref MemoryMarshal.GetReference(urlEncoded); - ref var output = ref MemoryMarshal.GetReference(base64); - - var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations - var n = (IntPtr)urlEncoded.Length; - var m = i; -#if !NET461 - if (Vector.IsHardwareAccelerated && (int*)n >= (int*)(2 * Vector.Count)) - { - m = (IntPtr)((int)(int*)n & ~(2 * Vector.Count - 1)); - for (; (int*)i < (int*)m; i += 2 * Vector.Count) - { - ref var tmp = ref Unsafe.Add(ref input, i); - var charsVec1 = Unsafe.As>(ref tmp); - var charsVec2 = Unsafe.As>(ref Unsafe.Add(ref tmp, Vector.Count)); - var bytesVec = Vector.Narrow(charsVec1, charsVec2); - - bytesVec = Substitute(bytesVec, (byte)'-', (byte)'+'); - bytesVec = Substitute(bytesVec, (byte)'_', (byte)'/'); - - Unsafe.WriteUnaligned(ref Unsafe.Add(ref output, i), bytesVec); - } - } -#endif - m = (IntPtr)((int)(int*)n & ~3); - for (; (int*)i < (int*)m; i += 4) - { - SubstituteUrlCharsForDecoding(ref input, ref output, i + 0); - SubstituteUrlCharsForDecoding(ref input, ref output, i + 1); - SubstituteUrlCharsForDecoding(ref input, ref output, i + 2); - SubstituteUrlCharsForDecoding(ref input, ref output, i + 3); - } - - for (; (int*)i < (int*)n; i += 1) - { - SubstituteUrlCharsForDecoding(ref input, ref output, i); - } - - n = (IntPtr)base64.Length; - - // There will be a maximum of 2 padding chars. - if ((int*)i < (int*)n) - { - Pad(ref output, i); - - i += 1; - if ((int*)i < (int*)n) - { - Pad(ref output, i); - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int SubstituteUrlCharsForEncoding(Span base64, int count) - { - ref var r = ref MemoryMarshal.GetReference(base64); - - return SubstituteUrlCharsForEncoding(ref r, ref r, count); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int SubstituteUrlCharsForEncoding(ReadOnlySpan base64, Span urlEncoded) - { - ref var input = ref Unsafe.As(ref MemoryMarshal.GetReference(base64)); - ref var output = ref Unsafe.As(ref MemoryMarshal.GetReference(urlEncoded)); - - return SubstituteUrlCharsForEncoding(ref input, ref output, base64.Length); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int SubstituteUrlCharsForEncoding(ref T base64, ref T urlEncoded, int base64Length) where T : struct - { - // Use base64url encoding with no padding characters. See RFC 4648, Sec. 5. - - var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations - var n = (IntPtr)base64Length; - var m = i; -#if !NET461 - if (Vector.IsHardwareAccelerated && (int*)n >= (int*)Vector.Count) - { - m = (IntPtr)((int)(int*)n & ~(Vector.Count - 1)); - for (; (int*)i < (int*)m; i += Vector.Count) - { - var vec = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref base64, i))); - - if (typeof(T) == typeof(byte)) - { - if (Vector.EqualsAny(vec, new Vector((T)(object)(byte)'='))) break; - - vec = Substitute(vec, (T)(object)(byte)'+', (T)(object)(byte)'-'); - vec = Substitute(vec, (T)(object)(byte)'/', (T)(object)(byte)'_'); - } - else if (typeof(T) == typeof(ushort)) - { - if (Vector.EqualsAny(vec, new Vector((T)(object)(ushort)'='))) break; - - vec = Substitute(vec, (T)(object)(ushort)'+', (T)(object)(ushort)'-'); - vec = Substitute(vec, (T)(object)(ushort)'/', (T)(object)(ushort)'_'); - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - - Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref urlEncoded, i)), vec); - } - } -#endif - // n is always a multiple of 4 - Debug.Assert((int)n % 4 == 0); - for (; (int*)i < (int*)n; i += 4) - { - if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 0)) goto Exit0; - if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 1)) goto Exit1; - if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 2)) goto Exit2; - if (SubstituteUrlCharsForEncoding(ref base64, ref urlEncoded, i + 3)) goto Exit3; - } - goto Exit0; - - Exit3: i += 1; - Exit2: i += 1; - Exit1: i += 1; - Exit0: - return (int)(int*)i; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int SubstituteUrlCharsForEncoding(ReadOnlySpan base64, Span urlEncoded) - { - // A subset of the ASCII-range can be assumed, so no need - // to call the encoders necessary for byte -> char - - ref var input = ref MemoryMarshal.GetReference(base64); - ref var output = ref MemoryMarshal.GetReference(MemoryMarshal.Cast(urlEncoded)); - - var i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations - var n = (IntPtr)base64.Length; - var m = i; -#if !NET461 - if (Vector.IsHardwareAccelerated && (int*)n >= (int*)Vector.Count) - { - m = (IntPtr)((int)(int*)n & ~(Vector.Count - 1)); - for (; (int*)i < (int*)m; i += Vector.Count) - { - var bytesVec = Unsafe.As>(ref Unsafe.Add(ref input, i)); - - if (Vector.EqualsAny(bytesVec, new Vector((byte)'='))) - { - break; - } - - bytesVec = Substitute(bytesVec, (byte)'+', (byte)'-'); - bytesVec = Substitute(bytesVec, (byte)'/', (byte)'_'); - - Vector.Widen(bytesVec, out Vector charsVec1, out Vector charsVec2); - ref var tmp = ref Unsafe.Add(ref output, i); - Unsafe.WriteUnaligned(ref Unsafe.As(ref tmp), charsVec1); - Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref tmp, Vector.Count)), charsVec2); - } - } -#endif - // n is always a multiple of 4 - Debug.Assert((int)n == ((int)n & ~3)); - for (; (int*)i < (int*)n; i += 4) - { - if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 0)) goto Exit0; - if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 1)) goto Exit1; - if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 2)) goto Exit2; - if (SubstituteUrlCharsForEncoding(ref input, ref output, i + 3)) goto Exit3; - } - goto Exit0; - - Exit3: i += 1; - Exit2: i += 1; - Exit1: i += 1; - Exit0: - return (int)(int*)i; - } -#if !NET461 - private static Vector Substitute(Vector vector, T match, T substitution) where T : struct - => Vector.ConditionalSelect( - Vector.Equals(vector, new Vector(match)), - new Vector(substitution), - vector); -#endif - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void SubstituteUrlCharsForDecoding(ref TIn urlEncoded, ref TOut base64, IntPtr idx) - where TIn : struct - where TOut : struct - { - TIn tmp = Unsafe.Add(ref urlEncoded, idx); - int value = default; - - if (typeof(TIn) == typeof(byte)) - { - value = (byte)(object)tmp; - } - else if (typeof(TIn) == typeof(ushort)) - { - value = (ushort)(object)tmp; - } - else if (typeof(TIn) == typeof(char)) - { - value = (char)(object)tmp; - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - - var subst = value; - - if (value == '-') - { - subst = '+'; - } - else if (value == '_') - { - subst = '/'; - } - - if (typeof(TOut) == typeof(byte)) - { - Unsafe.Add(ref base64, idx) = (TOut)(object)(byte)subst; - } - else if (typeof(TOut) == typeof(ushort)) - { - Unsafe.Add(ref base64, idx) = (TOut)(object)(ushort)subst; - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Pad(ref T base64, IntPtr idx) where T : struct - { - if (typeof(T) == typeof(byte)) - { - Unsafe.Add(ref base64, idx) = (T)(object)(byte)'='; - } - else if (typeof(T) == typeof(ushort)) - { - Unsafe.Add(ref base64, idx) = (T)(object)(ushort)'='; - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool SubstituteUrlCharsForEncoding(ref TIn base64, ref TOut urlEncoded, IntPtr idx) - where TIn : struct - where TOut : struct - { - TIn tmp = Unsafe.Add(ref base64, idx); - int value = default; - - if (typeof(TIn) == typeof(byte)) - { - value = (byte)(object)tmp; - } - else if (typeof(TIn) == typeof(ushort)) - { - value = (ushort)(object)tmp; - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - - var subst = value; - - if (value == '+') - { - subst = '-'; - } - else if (value == '/') - { - subst = '_'; - } - else if (value == '=') - { - return true; - } - - if (typeof(TOut) == typeof(byte)) - { - Unsafe.Add(ref urlEncoded, idx) = (TOut)(object)(byte)subst; - } - else if (typeof(TOut) == typeof(ushort)) - { - Unsafe.Add(ref urlEncoded, idx) = (TOut)(object)(ushort)subst; - } - else - { - throw new NotSupportedException(); // just in case new types are introduced in the future - } - - return false; - } - } - private static class ThrowHelper { public static void ThrowArgumentNullException(ExceptionArgument argument) diff --git a/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs b/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs index 5f23bb83bfd..f5b687ef405 100644 --- a/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs +++ b/test/Microsoft.Extensions.Internal.Test/WebEncodersTests.cs @@ -238,87 +238,6 @@ public void GetArraySizeRequiredToEncode_InputLengthTooBig_Throws() Assert.Equal("count", exception.ParamName); } - // Taken from https://github.com/aspnet/HttpAbstractions/pull/926 - [Theory] - [InlineData("_", "/")] - [InlineData("-", "+")] - [InlineData("a-b-c", "a+b+c=")] - [InlineData("a_b_c_d", "a/b/c/d=")] - [InlineData("a-b_c", "a+b/c==")] - [InlineData("a-b_c-d", "a+b/c+d=")] - [InlineData("abcd", "abcd")] - public void SubstituteUrlCharsForDecoding_ReturnsValid_Base64String(string text, string expectedValue) - { -// To test the alternate code-path -#if NETCOREAPP2_1 - // Arrange - Span bytes = stackalloc byte[text.Length]; - Encoding.ASCII.GetBytes(text.AsSpan(), bytes); - Span expected = stackalloc byte[expectedValue.Length]; - Encoding.ASCII.GetBytes(expectedValue.AsSpan(), expected); - Span result = stackalloc byte[expectedValue.Length]; - - // Act - WebEncoders.UrlCharsHelper.SubstituteUrlCharsForDecoding(bytes, result); - - // Assert - Assert.True(expected.SequenceEqual(result)); -#else - // Arrange - var buffer = new char[expectedValue.Length]; - - // Act - WebEncoders.UrlCharsHelper.SubstituteUrlCharsForDecoding(text.AsSpan(), buffer); - - // Assert - Assert.Equal(expectedValue, new string(buffer)); -#endif - } - - // Taken from https://github.com/aspnet/HttpAbstractions/pull/926 - // Input length must be a multiple of 4 - [Theory] - [InlineData(" ", " ")] - [InlineData("+ ", "- ")] - [InlineData("/ ", "_ ")] - [InlineData("= ", "")] - [InlineData("== ", "")] - [InlineData("a+b+c+==", "a-b-c-")] - [InlineData("a/b/c== ", "a_b_c")] - [InlineData("a+b/c== ", "a-b_c")] - [InlineData("a+b/c ", "a-b_c ")] - [InlineData("abcd", "abcd")] - public void SubstituteUrlCharsForEncoding_Replaces_UrlEncodableCharacters(string base64EncodedValue, string expectedValue) - { -// To test the alternate code-path -#if NETCOREAPP2_1 - // Arrange - Span bytes = stackalloc byte[base64EncodedValue.Length]; - Encoding.ASCII.GetBytes(base64EncodedValue.AsSpan(), bytes); - Span expected = stackalloc byte[expectedValue.Length]; - Encoding.ASCII.GetBytes(expectedValue.AsSpan(), expected); - - // Act - var result = WebEncoders.UrlCharsHelper.SubstituteUrlCharsForEncoding(bytes, bytes.Length); - - // Assert - Assert.True(expected.SequenceEqual(bytes.Slice(0, result))); -#else - // Arrange - var buffer = base64EncodedValue.ToCharArray(); - - // Act - var result = WebEncoders.UrlCharsHelper.SubstituteUrlCharsForEncoding(buffer, buffer); - - // Assert - Assert.Equal(expectedValue.Length, result); - for (var i = 0; i < result; i++) - { - Assert.Equal(expectedValue[i], buffer[i]); - } -#endif - } - [Fact] public void Base64UrlDecode_BufferChain() { From a08b7a5d47658f48b2a4d0b7cb2bb2d95d5988a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Mon, 16 Apr 2018 15:55:22 +0200 Subject: [PATCH 5/8] Cleanup --- .../WebEncoders.cs | 74 +++++++++---------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index 17c46904548..24b3126d83b 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -3,16 +3,11 @@ using System; using System.Buffers; -using System.Buffers.Text; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Microsoft.Extensions.WebEncoders.Sources; -#if !NET461 -using System.Numerics; -#endif - #if WebEncoders_In_WebUtilities namespace Microsoft.AspNetCore.WebUtilities #else @@ -29,7 +24,9 @@ namespace Microsoft.Extensions.Internal #endif static class WebEncoders { +#if !NETCOREAPP2_1 private const int MaxStackallocBytes = 256; +#endif private const int MaxEncodedLength = (int.MaxValue / 4) * 3; // encode inflates the data by 4/3 private static readonly byte[] EmptyBytes = new byte[0]; @@ -94,7 +91,8 @@ public static byte[] Base64UrlDecode(ReadOnlySpan base64Url) var base64Len = GetBufferSizeRequiredToUrlDecode(base64Url.Length, out int dataLength); var data = new byte[dataLength]; - var written = Base64UrlDecodeCore(base64Url, data); + var status = Base64UrlDecodeCore(base64Url, data, out int consumed, out int written); + Debug.Assert(base64Url.Length == consumed); Debug.Assert(data.Length == written); return data; @@ -118,7 +116,11 @@ public static int Base64UrlDecode(ReadOnlySpan base64Url, Span data) return 0; } - return Base64UrlDecodeCore(base64Url, data); + var status = Base64UrlDecodeCore(base64Url, data, out int consumed, out int written); + Debug.Assert(base64Url.Length == consumed); + Debug.Assert(data.Length >= written); + + return written; } /// @@ -193,7 +195,8 @@ public static byte[] Base64UrlDecode(string input, int offset, char[] buffer, in } var data = new byte[dataLength]; - var written = Base64UrlDecodeCore(input.AsSpan(offset, count), data); + var status = Base64UrlDecodeCore(input.AsSpan(offset, count), data, out int consumed, out int written); + Debug.Assert(count == consumed); Debug.Assert(dataLength == written); return data; @@ -254,8 +257,9 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) return string.Create(base64UrlLen, (Ptr: (IntPtr)ptr, data.Length), (base64Url, state) => { var bytes = new ReadOnlySpan(state.Ptr.ToPointer(), state.Length); - var urlEncodedLen = Base64UrlEncodeCore(bytes, base64Url); - Debug.Assert(base64Url.Length == urlEncodedLen); + var status = Base64UrlEncodeCore(bytes, base64Url, out int consumed, out int written); + Debug.Assert(bytes.Length == consumed); + Debug.Assert(base64Url.Length == written); }); } #else @@ -272,12 +276,12 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) #else : arrayToReturnToPool = ArrayPool.Shared.Rent(base64UrlLen); #endif - var urlEncodedLen = Base64UrlEncodeCore(data, base64Url); - Debug.Assert(base64UrlLen == urlEncodedLen); + var status = Base64UrlEncodeCore(data, base64Url, out int consumed, out int written); + Debug.Assert(base64UrlLen == written); fixed (char* ptr = &MemoryMarshal.GetReference(base64Url)) { - return new string(ptr, 0, urlEncodedLen); + return new string(ptr, 0, written); } #if !NET461 } @@ -308,7 +312,11 @@ public static int Base64UrlEncode(ReadOnlySpan data, Span base64Url) return 0; } - return Base64UrlEncodeCore(data, base64Url); + var status = Base64UrlEncodeCore(data, base64Url, out int consumed, out int written); + Debug.Assert(data.Length == consumed); + Debug.Assert(base64Url.Length >= written); + + return written; } /// @@ -338,9 +346,7 @@ public static OperationStatus Base64UrlEncode(ReadOnlySpan data, Span.Encode(data, base64Url, out bytesConsumed, out bytesWritten, isFinalBlock); + return Base64UrlEncodeCore(data, base64Url, out bytesConsumed, out bytesWritten, isFinalBlock); } /// @@ -384,7 +390,11 @@ public static int Base64UrlEncode(byte[] input, int offset, char[] output, int o return 0; } - return Base64UrlEncodeCore(input.AsSpan(offset, count), output.AsSpan(outputOffset)); + var status = Base64UrlEncodeCore(input.AsSpan(offset, count), output.AsSpan(outputOffset), out int consumed, out int written); + Debug.Assert(count == consumed); + Debug.Assert(base64Len >= written); + + return written; } /// @@ -426,23 +436,9 @@ public static int GetArraySizeRequiredToEncode(int count) return count == 0 ? 0 : GetBufferSizeRequiredToBase64Encode(count); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data) + private static OperationStatus Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data, out int consumed, out int written, bool isFinalBlock = true) { - var status = UrlEncoder.Decode(base64Url, data, out int consumed, out int written); - - if (status != OperationStatus.Done) - { - ThrowHelper.ThrowOperationNotDone(status); - } - - return written; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static OperationStatus Base64UrlDecodeCore(ReadOnlySpan base64Url, Span data, out int consumed, out int written, bool isFinalBlock) - { - var status = UrlEncoder.Decode(base64Url, data, out consumed, out written, isFinalBlock); + var status = UrlEncoder.Decode(base64Url, data, out consumed, out written, isFinalBlock); if (status != OperationStatus.Done && status != OperationStatus.NeedMoreData) { @@ -452,17 +448,16 @@ private static OperationStatus Base64UrlDecodeCore(ReadOnlySpan base64Url, return status; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url) + private static OperationStatus Base64UrlEncodeCore(ReadOnlySpan data, Span base64Url, out int consumed, out int written, bool isFinalBlock = true) { - var status = UrlEncoder.Encode(data, base64Url, out int consumed, out int written); + var status = UrlEncoder.Encode(data, base64Url, out consumed, out written, isFinalBlock); - if (status != OperationStatus.Done) + if (status != OperationStatus.Done && status != OperationStatus.NeedMoreData) { ThrowHelper.ThrowOperationNotDone(status); } - return written; + return status; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -511,6 +506,7 @@ private static int GetNumBase64PaddingCharsToAddForDecode(int urlEncodedLen) return result; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int GetBufferSizeRequiredToBase64Encode(int count) { if ((uint)count > MaxEncodedLength) From 902e507426ce0e068a8ad2fe7bf2638bf9e96af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Mon, 16 Apr 2018 18:02:38 +0200 Subject: [PATCH 6/8] Code style --- .../WebEncoders.cs | 118 ++++++++++-------- 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index 24b3126d83b..b44818ad34f 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -582,28 +582,28 @@ internal static class UrlEncoder { public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data, out int consumed, out int written, bool isFinalBlock = true) { - ref T source = ref MemoryMarshal.GetReference(urlEncoded); - ref byte destBytes = ref MemoryMarshal.GetReference(data); + ref var source = ref MemoryMarshal.GetReference(urlEncoded); + ref var destBytes = ref MemoryMarshal.GetReference(data); - int base64Len = GetBufferSizeRequiredToUrlDecode(urlEncoded.Length, out int dataLength, isFinalBlock); - int srcLength = base64Len & ~0x3; // only decode input up to closest multiple of 4. - int destLength = data.Length; + var base64Len = GetBufferSizeRequiredToUrlDecode(urlEncoded.Length, out int dataLength, isFinalBlock); + var srcLength = base64Len & ~0x3; // only decode input up to closest multiple of 4. + var destLength = data.Length; - int sourceIndex = 0; - int destIndex = 0; + var sourceIndex = 0; + var destIndex = 0; if (urlEncoded.Length == 0) { goto DoneExit; } - ref sbyte decodingMap = ref s_decodingMap[0]; + ref var decodingMap = ref s_decodingMap[0]; // Last bytes could have padding characters, so process them separately and treat them as valid only if isFinalBlock is true. // If isFinalBlock is false, padding characters are considered invalid. - int skipLastChunk = isFinalBlock ? 4 : 0; + var skipLastChunk = isFinalBlock ? 4 : 0; - int maxSrcLength = 0; + var maxSrcLength = 0; if (destLength >= dataLength) { maxSrcLength = srcLength - skipLastChunk; @@ -617,7 +617,7 @@ public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data while (sourceIndex < maxSrcLength) { - int result = DecodeFour(ref Unsafe.Add(ref source, sourceIndex), ref decodingMap); + var result = DecodeFour(ref Unsafe.Add(ref source, sourceIndex), ref decodingMap); if (result < 0) goto InvalidExit; @@ -646,12 +646,12 @@ public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data // If isFinalBlock is false, we will never reach this point. // Handle last four bytes. There are 0, 1, 2 padding chars. - int numPaddingChars = base64Len - urlEncoded.Length; - ref T lastFourStart = ref Unsafe.Add(ref source, srcLength - 4); + var numPaddingChars = base64Len - urlEncoded.Length; + ref var lastFourStart = ref Unsafe.Add(ref source, srcLength - 4); if (numPaddingChars == 0) { - int result = DecodeFour(ref lastFourStart, ref decodingMap); + var result = DecodeFour(ref lastFourStart, ref decodingMap); if (result < 0) goto InvalidExit; if (destIndex > destLength - 3) goto DestinationSmallExit; @@ -662,10 +662,17 @@ public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data } else if (numPaddingChars == 1) { - int result = DecodeThree(ref lastFourStart, ref decodingMap); + var result = DecodeThree(ref lastFourStart, ref decodingMap); - if (result < 0) goto InvalidExit; - if (destIndex > destLength - 2) goto DestinationSmallExit; + if (result < 0) + { + goto InvalidExit; + } + + if (destIndex > destLength - 2) + { + goto DestinationSmallExit; + } WriteTwoLowOrderBytes(ref destBytes, destIndex, result); destIndex += 2; @@ -673,10 +680,17 @@ public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data } else { - int result = DecodeTwo(ref lastFourStart, ref decodingMap); + var result = DecodeTwo(ref lastFourStart, ref decodingMap); - if (result < 0) goto InvalidExit; - if (destIndex > destLength - 1) goto DestinationSmallExit; + if (result < 0) + { + goto InvalidExit; + } + + if (destIndex > destLength - 1) + { + goto DestinationSmallExit; + } WriteOneLowOrderByte(ref destBytes, destIndex, result); destIndex += 1; @@ -715,13 +729,13 @@ public static OperationStatus Decode(ReadOnlySpan urlEncoded, Span data public static OperationStatus Encode(ReadOnlySpan data, Span urlEncoded, out int consumed, out int written, bool isFinalBlock = true) { - ref byte srcBytes = ref MemoryMarshal.GetReference(data); - ref T destination = ref MemoryMarshal.GetReference(urlEncoded); + ref var srcBytes = ref MemoryMarshal.GetReference(data); + ref var destination = ref MemoryMarshal.GetReference(urlEncoded); - int srcLength = data.Length; - int destLength = urlEncoded.Length; + var srcLength = data.Length; + var destLength = urlEncoded.Length; - int maxSrcLength = -2; + var maxSrcLength = -2; if (srcLength <= MaxEncodedLength && destLength >= GetBufferSizeRequiredToBase64Encode(srcLength, out int numPaddingChars) - numPaddingChars) { maxSrcLength += srcLength; @@ -731,8 +745,8 @@ public static OperationStatus Encode(ReadOnlySpan data, Span urlEncoded maxSrcLength += (destLength >> 2) * 3; } - int sourceIndex = 0; - int destIndex = 0; + var sourceIndex = 0; + var destIndex = 0; ref byte encodingMap = ref s_encodingMap[0]; @@ -744,10 +758,14 @@ public static OperationStatus Encode(ReadOnlySpan data, Span urlEncoded } if (maxSrcLength != srcLength - 2) + { goto DestinationSmallExit; + } if (!isFinalBlock) + { goto NeedMoreDataExit; + } if (sourceIndex == srcLength - 1) { @@ -784,7 +802,7 @@ private static int DecodeFour(ref T encoded, ref sbyte decodingMap) if (typeof(T) == typeof(byte)) { - ref byte tmp = ref Unsafe.As(ref encoded); + ref var tmp = ref Unsafe.As(ref encoded); i0 = Unsafe.Add(ref tmp, 0); i1 = Unsafe.Add(ref tmp, 1); i2 = Unsafe.Add(ref tmp, 2); @@ -792,7 +810,7 @@ private static int DecodeFour(ref T encoded, ref sbyte decodingMap) } else if (typeof(T) == typeof(char)) { - ref char tmp = ref Unsafe.As(ref encoded); + ref var tmp = ref Unsafe.As(ref encoded); i0 = Unsafe.Add(ref tmp, 0); i1 = Unsafe.Add(ref tmp, 1); i2 = Unsafe.Add(ref tmp, 2); @@ -821,14 +839,14 @@ private static int DecodeThree(ref T encoded, ref sbyte decodingMap) if (typeof(T) == typeof(byte)) { - ref byte tmp = ref Unsafe.As(ref encoded); + ref var tmp = ref Unsafe.As(ref encoded); i0 = Unsafe.Add(ref tmp, 0); i1 = Unsafe.Add(ref tmp, 1); i2 = Unsafe.Add(ref tmp, 2); } else if (typeof(T) == typeof(char)) { - ref char tmp = ref Unsafe.As(ref encoded); + ref var tmp = ref Unsafe.As(ref encoded); i0 = Unsafe.Add(ref tmp, 0); i1 = Unsafe.Add(ref tmp, 1); i2 = Unsafe.Add(ref tmp, 2); @@ -854,13 +872,13 @@ private static int DecodeTwo(ref T encoded, ref sbyte decodingMap) if (typeof(T) == typeof(byte)) { - ref byte tmp = ref Unsafe.As(ref encoded); + ref var tmp = ref Unsafe.As(ref encoded); i0 = Unsafe.Add(ref tmp, 0); i1 = Unsafe.Add(ref tmp, 1); } else if (typeof(T) == typeof(char)) { - ref char tmp = ref Unsafe.As(ref encoded); + ref var tmp = ref Unsafe.As(ref encoded); i0 = Unsafe.Add(ref tmp, 0); i1 = Unsafe.Add(ref tmp, 1); } @@ -900,12 +918,12 @@ private static void WriteOneLowOrderByte(ref byte destination, int destIndex, in [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void EncodeThreeBytes(ref byte threeBytes, ref T encoded, ref byte encodingMap) { - int i = (threeBytes << 16) | (Unsafe.Add(ref threeBytes, 1) << 8) | Unsafe.Add(ref threeBytes, 2); + var i = (threeBytes << 16) | (Unsafe.Add(ref threeBytes, 1) << 8) | Unsafe.Add(ref threeBytes, 2); - int i0 = Unsafe.Add(ref encodingMap, i >> 18); - int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); - int i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); - int i3 = Unsafe.Add(ref encodingMap, i & 0x3F); + var i0 = Unsafe.Add(ref encodingMap, i >> 18); + var i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + var i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); + var i3 = Unsafe.Add(ref encodingMap, i & 0x3F); if (typeof(T) == typeof(byte)) { @@ -914,7 +932,7 @@ private static void EncodeThreeBytes(ref byte threeBytes, ref T encoded, ref byt } else if (typeof(T) == typeof(char)) { - ref char enc = ref Unsafe.As(ref encoded); + ref var enc = ref Unsafe.As(ref encoded); Unsafe.Add(ref enc, 0) = (char)i0; Unsafe.Add(ref enc, 1) = (char)i1; Unsafe.Add(ref enc, 2) = (char)i2; @@ -929,22 +947,22 @@ private static void EncodeThreeBytes(ref byte threeBytes, ref T encoded, ref byt [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void EncodeTwoBytes(ref byte twoBytes, ref T encoded, ref byte encodingMap) { - int i = (twoBytes << 16) | (Unsafe.Add(ref twoBytes, 1) << 8); + var i = (twoBytes << 16) | (Unsafe.Add(ref twoBytes, 1) << 8); - int i0 = Unsafe.Add(ref encodingMap, i >> 18); - int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); - int i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); + var i0 = Unsafe.Add(ref encodingMap, i >> 18); + var i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + var i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); if (typeof(T) == typeof(byte)) { - ref byte enc = ref Unsafe.As(ref encoded); + ref var enc = ref Unsafe.As(ref encoded); Unsafe.Add(ref enc, 0) = (byte)i0; Unsafe.Add(ref enc, 1) = (byte)i1; Unsafe.Add(ref enc, 2) = (byte)i2; } else if (typeof(T) == typeof(char)) { - ref char enc = ref Unsafe.As(ref encoded); + ref var enc = ref Unsafe.As(ref encoded); Unsafe.Add(ref enc, 0) = (char)i0; Unsafe.Add(ref enc, 1) = (char)i1; Unsafe.Add(ref enc, 2) = (char)i2; @@ -958,20 +976,20 @@ private static void EncodeTwoBytes(ref byte twoBytes, ref T encoded, ref byte en [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void EncodeOneByte(ref byte oneByte, ref T encoded, ref byte encodingMap) { - int i = (oneByte << 16); + var i = (oneByte << 16); - int i0 = Unsafe.Add(ref encodingMap, i >> 18); - int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + var i0 = Unsafe.Add(ref encodingMap, i >> 18); + var i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); if (typeof(T) == typeof(byte)) { - ref byte enc = ref Unsafe.As(ref encoded); + ref var enc = ref Unsafe.As(ref encoded); Unsafe.Add(ref enc, 0) = (byte)i0; Unsafe.Add(ref enc, 1) = (byte)i1; } else if (typeof(T) == typeof(char)) { - ref char enc = ref Unsafe.As(ref encoded); + ref var enc = ref Unsafe.As(ref encoded); Unsafe.Add(ref enc, 0) = (char)i0; Unsafe.Add(ref enc, 1) = (char)i1; } From db2297b3acd1ce3e698ce7b7a16bdf71460ed3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Tue, 19 Jun 2018 12:20:16 +0200 Subject: [PATCH 7/8] Included .NET Core 2.2 in multi-targeting The is netcoreapp2.2 + net461 (on win), but as the file will be added as source to "consumers" other target frameworks are masked in the source. net461 as target for desktop. netcoreapp2.0 so ArrayPool can be used. netcoreapp2.1 for string.Create netcoreapp2.2 for the current tfm (no other enhanced features used over netcoreapp2.1). --- .../Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index b44818ad34f..db8fee217eb 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -24,7 +24,7 @@ namespace Microsoft.Extensions.Internal #endif static class WebEncoders { -#if !NETCOREAPP2_1 +#if !NETCOREAPP2_2 && !NETCOREAPP2_1 private const int MaxStackallocBytes = 256; #endif private const int MaxEncodedLength = (int.MaxValue / 4) * 3; // encode inflates the data by 4/3 @@ -251,7 +251,8 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) var base64Len = GetBufferSizeRequiredToBase64Encode(data.Length, out int numPaddingChars); var base64UrlLen = base64Len - numPaddingChars; -#if NETCOREAPP2_1 + +#if NETCOREAPP2_2 || NETCOREAPP2_1 fixed (byte* ptr = &MemoryMarshal.GetReference(data)) { return string.Create(base64UrlLen, (Ptr: (IntPtr)ptr, data.Length), (base64Url, state) => From aa412661449d00a97807bffcdc394c7d656b9230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Foidl?= Date: Wed, 20 Jun 2018 14:29:31 +0200 Subject: [PATCH 8/8] Rearranged the #if to make it more readable Cf. https://github.com/aspnet/Common/pull/338#issuecomment-398645178 --- .../WebEncoders.cs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs index db8fee217eb..e1ae08426d8 100644 --- a/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs +++ b/shared/Microsoft.Extensions.WebEncoders.Sources/WebEncoders.cs @@ -263,20 +263,15 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) Debug.Assert(base64Url.Length == written); }); } -#else -#if !NET461 + +#elif NETCOREAPP2_0 char[] arrayToReturnToPool = null; try { -#endif - var base64Url = base64UrlLen <= MaxStackallocBytes / sizeof(char) ? stackalloc char[base64UrlLen] -#if NET461 - : new char[base64UrlLen]; -#else : arrayToReturnToPool = ArrayPool.Shared.Rent(base64UrlLen); -#endif + var status = Base64UrlEncodeCore(data, base64Url, out int consumed, out int written); Debug.Assert(base64UrlLen == written); @@ -284,7 +279,6 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) { return new string(ptr, 0, written); } -#if !NET461 } finally { @@ -293,7 +287,18 @@ public static unsafe string Base64UrlEncode(ReadOnlySpan data) ArrayPool.Shared.Return(arrayToReturnToPool); } } -#endif +#else + var base64Url = base64UrlLen <= MaxStackallocBytes / sizeof(char) + ? stackalloc char[base64UrlLen] + : new char[base64UrlLen]; + + var status = Base64UrlEncodeCore(data, base64Url, out int consumed, out int written); + Debug.Assert(base64UrlLen == written); + + fixed (char* ptr = &MemoryMarshal.GetReference(base64Url)) + { + return new string(ptr, 0, written); + } #endif }