Skip to content

Commit

Permalink
Enable SocketHttpHandler to decompress zlib or deflate (dotnet#57862)
Browse files Browse the repository at this point in the history 
* Enable SocketHttpHandler to decompress zlib or deflate

Some servers incorrectly implement the deflate content-coding with the raw deflate algorithm rather than with deflate wrapped with a zlib header/footer.  Auto-detect whether to use ZLibStream or DeflateStream in order to accomodate both kinds of responses.

* Fix test build for WinHttpHandler on .NET Framework

* Apply suggestions from code review

* Add decompression test for empty response body

* Add decompression tests for multiple source content lengths
  • Loading branch information
@stephentoub
stephentoub authored Aug 23, 2021
1 parent c83b6ed commit b8a5a4a
Show file tree
Hide file tree
Showing 4 changed files with 324 additions and 56 deletions.
125 changes: 82 additions & 43 deletions src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Test.Common;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
Expand All @@ -26,19 +27,29 @@ public abstract class HttpClientHandler_Decompression_Test : HttpClientHandlerTe
#endif
public HttpClientHandler_Decompression_Test(ITestOutputHelper output) : base(output) { }

public static IEnumerable<object[]> DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData() =>
from compressionName in new[] { "gzip", "zlib", "deflate", "br" }
from all in new[] { false, true }
from copyTo in new[] { false, true }
from contentLength in new[] { 0, 1, 12345 }
select new object[] { compressionName, all, copyTo, contentLength };

[Theory]
[InlineData("gzip", false)]
[InlineData("gzip", true)]
[InlineData("deflate", false)]
[InlineData("deflate", true)]
[InlineData("br", false)]
[InlineData("br", true)]
[MemberData(nameof(DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData))]
[SkipOnPlatform(TestPlatforms.Browser, "AutomaticDecompression not supported on Browser")]
public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(string encodingName, bool all)
public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(string compressionName, bool all, bool useCopyTo, int contentLength)
{
if (IsWinHttpHandler &&
(compressionName == "br" || compressionName == "zlib"))
{
// brotli and zlib not supported on WinHttpHandler
return;
}

Func<Stream, Stream> compress;
DecompressionMethods methods;
switch (encodingName)
string encodingName = compressionName;
switch (compressionName)
{
case "gzip":
compress = s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true);
Expand All @@ -47,32 +58,27 @@ public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturn

#if !NETFRAMEWORK
case "br":
if (IsWinHttpHandler)
{
// Brotli only supported on SocketsHttpHandler.
return;
}

compress = s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true);
methods = all ? DecompressionMethods.Brotli : _all;
break;

case "deflate":
// WinHttpHandler continues to use DeflateStream as it doesn't have a newer build than netstandard2.0
// and doesn't have access to ZLibStream.
compress = IsWinHttpHandler ?
new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)) :
new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true));
case "zlib":
compress = s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true);
methods = all ? DecompressionMethods.Deflate : _all;
encodingName = "deflate";
break;
#endif

case "deflate":
compress = s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true);
methods = all ? DecompressionMethods.Deflate : _all;
break;

default:
Assert.Contains(encodingName, new[] { "br", "deflate", "gzip" });
return;
throw new Exception($"Unexpected compression: {compressionName}");
}

var expectedContent = new byte[12345];
var expectedContent = new byte[contentLength];
new Random(42).NextBytes(expectedContent);

await LoopbackServer.CreateClientAndServerAsync(async uri =>
Expand All @@ -81,7 +87,7 @@ await LoopbackServer.CreateClientAndServerAsync(async uri =>
using (HttpClient client = CreateHttpClient(handler))
{
handler.AutomaticDecompression = methods;
Assert.Equal<byte>(expectedContent, await client.GetByteArrayAsync(uri));
AssertExtensions.SequenceEqual(expectedContent, await client.GetByteArrayAsync(TestAsync, useCopyTo, uri));
}
}, async server =>
{
Expand All @@ -99,33 +105,39 @@ await server.AcceptConnectionAsync(async connection =>

public static IEnumerable<object[]> DecompressedResponse_MethodNotSpecified_OriginalContentReturned_MemberData()
{
yield return new object[]
foreach (bool useCopyTo in new[] { false, true })
{
"gzip",
new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
DecompressionMethods.None
};
yield return new object[]
{
"gzip",
new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
DecompressionMethods.None,
useCopyTo
};
#if !NETFRAMEWORK
yield return new object[]
{
"deflate",
new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true)),
DecompressionMethods.Brotli
};
yield return new object[]
{
"br",
new Func<Stream, Stream>(s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true)),
DecompressionMethods.Deflate | DecompressionMethods.GZip
};
yield return new object[]
{
"deflate",
new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true)),
DecompressionMethods.Brotli,
useCopyTo
};
yield return new object[]
{
"br",
new Func<Stream, Stream>(s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true)),
DecompressionMethods.Deflate | DecompressionMethods.GZip,
useCopyTo
};
#endif
}
}

[Theory]
[MemberData(nameof(DecompressedResponse_MethodNotSpecified_OriginalContentReturned_MemberData))]
[SkipOnPlatform(TestPlatforms.Browser, "AutomaticDecompression not supported on Browser")]
public async Task DecompressedResponse_MethodNotSpecified_OriginalContentReturned(
string encodingName, Func<Stream, Stream> compress, DecompressionMethods methods)
string encodingName, Func<Stream, Stream> compress, DecompressionMethods methods, bool useCopyTo)
{
var expectedContent = new byte[12345];
new Random(42).NextBytes(expectedContent);
Expand All @@ -143,7 +155,7 @@ await LoopbackServer.CreateClientAndServerAsync(async uri =>
using (HttpClient client = CreateHttpClient(handler))
{
handler.AutomaticDecompression = methods;
Assert.Equal<byte>(compressedContent, await client.GetByteArrayAsync(uri));
AssertExtensions.SequenceEqual(compressedContent, await client.GetByteArrayAsync(TestAsync, useCopyTo, uri));
}
}, async server =>
{
Expand All @@ -156,6 +168,33 @@ await server.AcceptConnectionAsync(async connection =>
});
}

[Theory]
[InlineData("gzip", DecompressionMethods.GZip)]
#if !NETFRAMEWORK
[InlineData("deflate", DecompressionMethods.Deflate)]
[InlineData("br", DecompressionMethods.Brotli)]
#endif
[SkipOnPlatform(TestPlatforms.Browser, "AutomaticDecompression not supported on Browser")]
public async Task DecompressedResponse_EmptyBody_Success(string encodingName, DecompressionMethods methods)
{
await LoopbackServer.CreateClientAndServerAsync(async uri =>
{
using (HttpClientHandler handler = CreateHttpClientHandler())
using (HttpClient client = CreateHttpClient(handler))
{
handler.AutomaticDecompression = methods;
Assert.Equal(Array.Empty<byte>(), await client.GetByteArrayAsync(TestAsync, useCopyTo: false, uri));
}
}, async server =>
{
await server.AcceptConnectionAsync(async connection =>
{
await connection.ReadRequestHeaderAsync();
await connection.WriteStringAsync($"HTTP/1.1 200 OK\r\nContent-Encoding: {encodingName}\r\n\r\n");
});
});
}

[Theory]
#if NETCOREAPP
[InlineData(DecompressionMethods.Brotli, "br", "")]
Expand Down
9 changes: 3 additions & 6 deletions src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.RemoteServer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1237,8 +1237,9 @@ public static IEnumerable<object[]> RemoteServersAndCompressionUris()
{
yield return new object[] { remoteServer, remoteServer.GZipUri };

// Remote deflate endpoint isn't correctly following the deflate protocol.
//yield return new object[] { remoteServer, remoteServer.DeflateUri };
// Remote deflate endpoint isn't correctly following the deflate protocol,
// but SocketsHttpHandler makes it work, anyway.
yield return new object[] { remoteServer, remoteServer.DeflateUri };
}
}

Expand Down Expand Up @@ -1271,10 +1272,6 @@ public async Task GetAsync_SetAutomaticDecompression_ContentDecompressed_GZip(Co
}
}

// The remote server endpoint was written to use DeflateStream, which isn't actually a correct
// implementation of the deflate protocol (the deflate protocol requires the zlib wrapper around
// deflate). Until we can get that updated (and deal with previous releases still testing it
// via a DeflateStream-based implementation), we utilize httpbin.org to help validate behavior.
[OuterLoop("Uses external servers")]
[Theory]
[InlineData("http://httpbin.org/deflate", "\"deflated\": true")]
Expand Down
42 changes: 42 additions & 0 deletions src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,5 +196,47 @@ public static Task<Stream> ReadAsStreamAsync(this HttpContent content, bool asyn
#endif
}
}

public static Task<byte[]> GetByteArrayAsync(this HttpClient client, bool async, bool useCopyTo, Uri uri)
{
#if NETCOREAPP
return Task.Run(async () =>
{
var m = new HttpRequestMessage(HttpMethod.Get, uri);
using HttpResponseMessage r = async ? await client.SendAsync(m, HttpCompletionOption.ResponseHeadersRead) : client.Send(m, HttpCompletionOption.ResponseHeadersRead);
using Stream s = async ? await r.Content.ReadAsStreamAsync() : r.Content.ReadAsStream();

var result = new MemoryStream();
if (useCopyTo)
{
if (async)
{
await s.CopyToAsync(result);
}
else
{
s.CopyTo(result);
}
}
else
{
byte[] buffer = new byte[100];
while (true)
{
int bytesRead = async ? await s.ReadAsync(buffer) : s.Read(buffer);
if (bytesRead == 0)
{
break;
}
result.Write(buffer.AsSpan(0, bytesRead));
}
}
return result.ToArray();
});
#else
// For WinHttpHandler on .NET Framework, we fall back to ignoring async and useCopyTo.
return client.GetByteArrayAsync(uri);
#endif
}
}
}
Loading

0 comments on commit b8a5a4a

Please sign in to comment.