Skip to content

Commit

Permalink
Fix the max token size threshold to correctly compute to 125MB for Ba…
Browse files Browse the repository at this point in the history 
…se64 bytes. (dotnet/corefx#40792)

* Fix the max token size threshold to correctly compute to 125MB for
Base64 bytes.

* Rename constant to fix transpose error: Base46 -> Base64

* Enable the outerloop tests for windows and osx only and update to use
platform specific new line.


Commit migrated from dotnet/corefx@1511f72
  • Loading branch information
@ahsonkhan
ahsonkhan authored Sep 4, 2019
1 parent 486a1fa commit 2614c3c
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 5 deletions.
2 changes: 1 addition & 1 deletion src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ internal static class JsonConstants

public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value.
public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes
public const int MaxBase46ValueTokenSize = (MaxEscapedTokenSize >> 2 * 3) / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes
public const int MaxBase64ValueTokenSize = (MaxEscapedTokenSize >> 2) * 3 / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes
public const int MaxCharacterTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 characters

public const int MaximumFormatInt64Length = 20; // 19 + sign (i.e. -9223372036854775808)
Expand Down
6 changes: 3 additions & 3 deletions src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public static void ValidateValue(ReadOnlySpan<byte> value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ValidateBytes(ReadOnlySpan<byte> bytes)
{
if (bytes.Length > JsonConstants.MaxBase46ValueTokenSize)
if (bytes.Length > JsonConstants.MaxBase64ValueTokenSize)
ThrowHelper.ThrowArgumentException_ValueTooLarge(bytes.Length);
}

Expand Down Expand Up @@ -123,14 +123,14 @@ public static void ValidatePropertyAndValue(ReadOnlySpan<char> propertyName, Rea
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ValidatePropertyAndBytes(ReadOnlySpan<char> propertyName, ReadOnlySpan<byte> bytes)
{
if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize)
if (propertyName.Length > JsonConstants.MaxCharacterTokenSize || bytes.Length > JsonConstants.MaxBase64ValueTokenSize)
ThrowHelper.ThrowArgumentException(propertyName, bytes);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ValidatePropertyAndBytes(ReadOnlySpan<byte> propertyName, ReadOnlySpan<byte> bytes)
{
if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || bytes.Length > JsonConstants.MaxBase46ValueTokenSize)
if (propertyName.Length > JsonConstants.MaxUnescapedTokenSize || bytes.Length > JsonConstants.MaxBase64ValueTokenSize)
ThrowHelper.ThrowArgumentException(propertyName, bytes);
}

Expand Down
139 changes: 138 additions & 1 deletion src/libraries/System.Text.Json/tests/Utf8JsonWriterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// See the LICENSE file in the project root for more information.

using System.Buffers;
using System.Buffers.Text;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
Expand Down Expand Up @@ -2464,6 +2465,11 @@ public void WritingTooLargeBase64Bytes(bool formatted, bool skipValidation)
var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation };
var output = new ArrayBufferWriter<byte>(1024);

using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
Assert.Throws<ArgumentException>(() => jsonUtf8.WriteBase64StringValue(value.AsSpan(0, 125_000_001)));
}

using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
Assert.Throws<ArgumentException>(() => jsonUtf8.WriteBase64StringValue(value));
Expand Down Expand Up @@ -2494,6 +2500,137 @@ public void WritingTooLargeBase64Bytes(bool formatted, bool skipValidation)
}
}

// NOTE: WritingTooLargeProperty test is constrained to run on Windows and MacOSX because it causes
// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can
// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the
// time the memory is accessed which triggers the full memory allocation.
[PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
[ConditionalTheory(nameof(IsX64))]
[OuterLoop]
[InlineData(true, true)]
[InlineData(true, false)]
[InlineData(false, true)]
[InlineData(false, false)]
public void WritingLargestPossibleBase64Bytes(bool formatted, bool skipValidation)
{
byte[] value;

try
{
value = new byte[125_000_000];
}
catch (OutOfMemoryException)
{
return;
}

value.AsSpan().Fill(168);

var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation };
var output = new ArrayBufferWriter<byte>(1024);

using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteBase64StringValue(value);
}

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String("foo", value);
jsonUtf8.WriteEndObject();
}

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String(Encoding.UTF8.GetBytes("foo"), value);
jsonUtf8.WriteEndObject();
}

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String("foo".AsSpan(), value);
jsonUtf8.WriteEndObject();
}

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String(JsonEncodedText.Encode("foo"), value);
jsonUtf8.WriteEndObject();
}
}

// https://github.com/dotnet/corefx/issues/40755
[Theory]
[InlineData(true, true)]
[InlineData(true, false)]
[InlineData(false, true)]
[InlineData(false, false)]
public void Writing3MBBase64Bytes(bool formatted, bool skipValidation)
{
byte[] value = new byte[3 * 1024 * 1024];

value.AsSpan().Fill(168);

byte[] base64StringUtf8 = new byte[Base64.GetMaxEncodedToUtf8Length(value.Length)];
Base64.EncodeToUtf8(value, base64StringUtf8, out _, out int bytesWritten);
string expectedValue = Encoding.UTF8.GetString(base64StringUtf8.AsSpan(0, bytesWritten).ToArray());

string expectedJson = formatted ? $"{{{Environment.NewLine} \"foo\": \"{expectedValue}\"{Environment.NewLine}}}" : $"{{\"foo\":\"{expectedValue}\"}}";

var options = new JsonWriterOptions { Indented = formatted, SkipValidation = skipValidation };
var output = new ArrayBufferWriter<byte>(1024);

using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteBase64StringValue(value);
}
JsonTestHelper.AssertContents($"\"{expectedValue}\"", output);

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String("foo", value);
jsonUtf8.WriteEndObject();
}
JsonTestHelper.AssertContents(expectedJson, output);

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String(Encoding.UTF8.GetBytes("foo"), value);
jsonUtf8.WriteEndObject();
}
JsonTestHelper.AssertContents(expectedJson, output);

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String("foo".AsSpan(), value);
jsonUtf8.WriteEndObject();
}
JsonTestHelper.AssertContents(expectedJson, output);

output.Clear();
using (var jsonUtf8 = new Utf8JsonWriter(output, options))
{
jsonUtf8.WriteStartObject();
jsonUtf8.WriteBase64String(JsonEncodedText.Encode("foo"), value);
jsonUtf8.WriteEndObject();
}
JsonTestHelper.AssertContents(expectedJson, output);
}

[Theory]
[InlineData(true, true)]
[InlineData(true, false)]
Expand Down Expand Up @@ -3932,7 +4069,7 @@ public void Utf8SurrogatePairReplacement_ValidPropertyName_ValidValue(bool skipV
JsonTestHelper.AssertContents("{" + ValidUtf8Expected + ":" + ValidUtf8Expected + "}", output);
}

private static readonly string s_InvalidUtf16Input = new string (new char[2] { (char)0xD801, 'a' });
private static readonly string s_InvalidUtf16Input = new string(new char[2] { (char)0xD801, 'a' });
private const string InvalidUtf16Expected = "\"\\uFFFDa\"";

private static readonly string s_ValidUtf16Input = new string(new char[2] { (char)0xD801, (char)0xDC37 }); // 0x10437
Expand Down

0 comments on commit 2614c3c

Please sign in to comment.