Skip to content

Commit

Permalink
Improve HttpClient POST tests stability (dotnet/corefx#36801)
Browse files Browse the repository at this point in the history 
Several of the HttpClient POST scenario tests were failing with HTTP status code 500. This was caused
by the Azure remote endpoint generating OutOfMemoryException at times. The tests are sending
large amounts of request body data to the Echo endpoint. That endpoint will respond back with all
the headers and request body data serialized into a JSON payload. The OOM exceptions were
coming from the Newtonsoft JSON serialization code currently used by the server endpoint.

These tests don't really require that the request body data be sent back. The purpose of the tests is
to verify that the request body payload was correctly sent by the client. We already have an endpoint,
VerifyUpload, that can do that without echo'ing back the large request body data.

This PR modifies the tests to use that endpoint. This should mitigate the OOM exceptions
currently being generated in the server-side code. Additional mitigations/fixes will be done later on the server-side code to improve robustness. But fixing the tests to be more streamlined is goodness and will result in these tests being faster and more stable.

Closes dotnet/corefx#36782

Commit migrated from dotnet/corefx@288ce93
  • Loading branch information
@davidsh
davidsh authored Apr 12, 2019
1 parent 0474ec9 commit 4484947
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 27 deletions.
1 change: 1 addition & 0 deletions src/libraries/Common/tests/System/Net/Configuration.Http.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public static partial class Http
public static readonly Uri RemoteVerifyUploadServer = new Uri("http://" + Host + "/" + VerifyUploadHandler);
public static readonly Uri SecureRemoteVerifyUploadServer = new Uri("https://" + SecureHost + "/" + VerifyUploadHandler);
public static readonly Uri Http2RemoteVerifyUploadServer = new Uri("https://" + Http2Host + "/" + VerifyUploadHandler);
public static readonly Uri[] VerifyUploadServerList = new Uri[] { RemoteVerifyUploadServer, SecureRemoteVerifyUploadServer, Http2RemoteVerifyUploadServer };

public static readonly Uri RemoteEmptyContentServer = new Uri("http://" + Host + "/" + EmptyContentHandler);
public static readonly Uri RemoteDeflateServer = new Uri("http://" + Host + "/" + DeflateHandler);
Expand Down
46 changes: 19 additions & 27 deletions src/libraries/System.Net.Http/tests/FunctionalTests/PostScenarioTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public abstract class PostScenarioTest : HttpClientHandlerTestBase
private readonly ITestOutputHelper _output;

public static readonly object[][] EchoServers = Configuration.Http.EchoServers;
public static readonly object[][] VerifyUploadServers = Configuration.Http.VerifyUploadServers;

public static readonly object[][] BasicAuthEchoServers =
new object[][]
Expand Down Expand Up @@ -67,79 +68,79 @@ public async Task PostRewindableStreamContentMultipleTimes_StreamContentFullySen
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostNoContentUsingContentLengthSemantics_Success(Uri serverUri)
{
await PostHelper(serverUri, string.Empty, null,
useContentLengthUpload: true, useChunkedEncodingUpload: false);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostEmptyContentUsingContentLengthSemantics_Success(Uri serverUri)
{
await PostHelper(serverUri, string.Empty, new StringContent(string.Empty),
useContentLengthUpload: true, useChunkedEncodingUpload: false);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostEmptyContentUsingChunkedEncoding_Success(Uri serverUri)
{
await PostHelper(serverUri, string.Empty, new StringContent(string.Empty),
useContentLengthUpload: false, useChunkedEncodingUpload: true);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostEmptyContentUsingConflictingSemantics_Success(Uri serverUri)
{
await PostHelper(serverUri, string.Empty, new StringContent(string.Empty),
useContentLengthUpload: true, useChunkedEncodingUpload: true);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostUsingContentLengthSemantics_Success(Uri serverUri)
{
await PostHelper(serverUri, ExpectedContent, new StringContent(ExpectedContent),
useContentLengthUpload: true, useChunkedEncodingUpload: false);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostUsingChunkedEncoding_Success(Uri serverUri)
{
await PostHelper(serverUri, ExpectedContent, new StringContent(ExpectedContent),
useContentLengthUpload: false, useChunkedEncodingUpload: true);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostSyncBlockingContentUsingChunkedEncoding_Success(Uri serverUri)
{
await PostHelper(serverUri, ExpectedContent, new SyncBlockingContent(ExpectedContent),
useContentLengthUpload: false, useChunkedEncodingUpload: true);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostRepeatedFlushContentUsingChunkedEncoding_Success(Uri serverUri)
{
await PostHelper(serverUri, ExpectedContent, new RepeatedFlushContent(ExpectedContent),
useContentLengthUpload: false, useChunkedEncodingUpload: true);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
public async Task PostUsingUsingConflictingSemantics_UsesChunkedSemantics(Uri serverUri)
{
await PostHelper(serverUri, ExpectedContent, new StringContent(ExpectedContent),
useContentLengthUpload: true, useChunkedEncodingUpload: true);
}

[OuterLoop("Uses external servers")]
[Theory, MemberData(nameof(EchoServers))]
[Theory, MemberData(nameof(VerifyUploadServers))]
[SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "netfx behaves differently and will buffer content and use 'Content-Length' semantics")]
[SkipOnTargetFramework(TargetFrameworkMonikers.Uap, "WinRT behaves differently and will use 'Content-Length' semantics")]
public async Task PostUsingNoSpecifiedSemantics_UsesChunkedSemantics(Uri serverUri)
Expand All @@ -148,9 +149,9 @@ public async Task PostUsingNoSpecifiedSemantics_UsesChunkedSemantics(Uri serverU
useContentLengthUpload: false, useChunkedEncodingUpload: false);
}

public static IEnumerable<object[]> EchoServersAndLargeContentSizes()
public static IEnumerable<object[]> VerifyUploadServersAndLargeContentSizes()
{
foreach (Uri uri in Configuration.Http.EchoServerList)
foreach (Uri uri in Configuration.Http.VerifyUploadServerList)
{
yield return new object[] { uri, 5 * 1024 };
yield return new object[] { uri, 63 * 1024 };
Expand All @@ -160,7 +161,7 @@ public static IEnumerable<object[]> EchoServersAndLargeContentSizes()

[OuterLoop("Uses external server")]
[Theory]
[MemberData(nameof(EchoServersAndLargeContentSizes))]
[MemberData(nameof(VerifyUploadServersAndLargeContentSizes))]
public async Task PostLargeContentUsingContentLengthSemantics_Success(Uri serverUri, int contentLength)
{
var rand = new Random(42);
Expand Down Expand Up @@ -246,8 +247,12 @@ private async Task PostHelper(
{
requestContent.Headers.ContentLength = null;
}

// Compute MD5 of request body data. This will be verified by the server when it
// receives the request.
requestContent.Headers.ContentMD5 = TestHelper.ComputeMD5Hash(requestBody);
}

if (useChunkedEncodingUpload)
{
client.DefaultRequestHeaders.TransferEncodingChunked = true;
Expand All @@ -256,19 +261,6 @@ private async Task PostHelper(
using (HttpResponseMessage response = await client.PostAsync(serverUri, requestContent))
{
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
string responseContent = await response.Content.ReadAsStringAsync();
_output.WriteLine(responseContent);

if (!useContentLengthUpload && !useChunkedEncodingUpload)
{
useChunkedEncodingUpload = true;
}

TestHelper.VerifyResponseBody(
responseContent,
response.Content.Headers.ContentMD5,
useChunkedEncodingUpload,
requestBody);
}
}
}
Expand Down

0 comments on commit 4484947

Please sign in to comment.