Implement TranscodingStream (dotnet#35145)

This is a streaming analog to the one-shot Encoding.Convert method.

src/libraries/Common/tests/TestUtilities/System/IO/StreamExtensions.cs

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace System.IO
+{
+    public static class StreamExtensions
+    {
+        public static async Task<int> ReadByteAsync(this Stream stream, CancellationToken cancellationToken = default)
+        {
+            byte[] buffer = new byte[1];
+
+            int numBytesRead = await stream.ReadAsync(buffer, 0, 1, cancellationToken);
+            if (numBytesRead == 0)
+            {
+                return -1; // EOF
+            }
+
+            return buffer[0];
+        }
+    }
+}

src/libraries/Common/tests/TestUtilities/TestUtilities.csproj

@@ -11,6 +11,7 @@
   <ItemGroup>
     <Compile Include="System\AdminHelpers.cs" />
     <Compile Include="System\AssertExtensions.cs" />
+    <Compile Include="System\IO\StreamExtensions.cs" />
     <Compile Include="System\RetryHelper.cs" />
     <Compile Include="System\Buffers\BoundedMemory.cs" />
     <Compile Include="System\Buffers\BoundedMemory.Creation.cs" />

src/libraries/System.Net.Http.Json/src/System.Net.Http.Json.csproj

@@ -9,8 +9,6 @@
     <Compile Include="System\Net\Http\Json\HttpClientJsonExtensions.Put.cs" />
     <Compile Include="System\Net\Http\Json\HttpContentJsonExtensions.cs" />
     <Compile Include="System\Net\Http\Json\JsonContent.cs" />
-    <Compile Include="System\Net\Http\Json\TranscodingReadStream.cs" />
-    <Compile Include="System\Net\Http\Json\TranscodingWriteStream.cs" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == '$(NetCoreAppCurrent)'">
     <Compile Include="System\Net\Http\Json\JsonContent.netcoreapp.cs" />
@@ -21,6 +19,8 @@
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <Compile Include="System\ArraySegmentExtensions.netstandard.cs" />
+    <Compile Include="System\Net\Http\Json\TranscodingReadStream.cs" />
+    <Compile Include="System\Net\Http\Json\TranscodingWriteStream.cs" />
     <Reference Include="System.Buffers" />
   </ItemGroup>
   <ItemGroup>

src/libraries/System.Net.Http.Json/src/System/Net/Http/Json/HttpContentJsonExtensions.cs

@@ -38,7 +38,11 @@ public static Task<T> ReadFromJsonAsync<T>(this HttpContent content, JsonSeriali
             // Wrap content stream into a transcoding stream that buffers the data transcoded from the sourceEncoding to utf-8.
             if (sourceEncoding != null && sourceEncoding != Encoding.UTF8)
             {
+#if NETCOREAPP
+                contentStream = Encoding.CreateTranscodingStream(contentStream, innerStreamEncoding: sourceEncoding, outerStreamEncoding: Encoding.UTF8);
+#else
                 contentStream = new TranscodingReadStream(contentStream, sourceEncoding);
+#endif
             }
 
             using (contentStream)
@@ -54,7 +58,11 @@ private static async Task<T> ReadFromJsonAsyncCore<T>(HttpContent content, Encod
             // Wrap content stream into a transcoding stream that buffers the data transcoded from the sourceEncoding to utf-8.
             if (sourceEncoding != null && sourceEncoding != Encoding.UTF8)
             {
+#if NETCOREAPP
+                contentStream = Encoding.CreateTranscodingStream(contentStream, innerStreamEncoding: sourceEncoding, outerStreamEncoding: Encoding.UTF8);
+#else
                 contentStream = new TranscodingReadStream(contentStream, sourceEncoding);
+#endif
             }
 
             using (contentStream)

src/libraries/System.Net.Http.Json/src/System/Net/Http/Json/JsonContent.cs

@@ -67,6 +67,19 @@ private async Task SerializeToStreamAsyncCore(Stream targetStream, CancellationT
             // Wrap provided stream into a transcoding stream that buffers the data transcoded from utf-8 to the targetEncoding.
             if (targetEncoding != null && targetEncoding != Encoding.UTF8)
             {
+#if NETCOREAPP
+                Stream transcodingStream = Encoding.CreateTranscodingStream(targetStream, targetEncoding, Encoding.UTF8, leaveOpen: true);
+                try
+                {
+                    await JsonSerializer.SerializeAsync(transcodingStream, Value, ObjectType, _jsonSerializerOptions, cancellationToken).ConfigureAwait(false);
+                }
+                finally
+                {
+                    // DisposeAsync will flush any partial write buffers. In practice our partial write
+                    // buffers should be empty as we expect JsonSerializer to emit only well-formed UTF-8 data.
+                    await transcodingStream.DisposeAsync().ConfigureAwait(false);
+                }
+#else
                 using (TranscodingWriteStream transcodingStream = new TranscodingWriteStream(targetStream, targetEncoding))
                 {
                     await JsonSerializer.SerializeAsync(transcodingStream, Value, ObjectType, _jsonSerializerOptions, cancellationToken).ConfigureAwait(false);
@@ -75,6 +88,7 @@ private async Task SerializeToStreamAsyncCore(Stream targetStream, CancellationT
                     // acceptable to Flush a Stream (multiple times) prior to completion.
                     await transcodingStream.FinalWriteAsync(cancellationToken).ConfigureAwait(false);
                 }
+#endif
             }
             else
             {

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -874,6 +874,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringBuilder.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringBuilder.Debug.cs" Condition="'$(Configuration)' == 'Debug'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringRuneEnumerator.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Text\TranscodingStream.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Text\TrimType.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Text\Unicode\GraphemeClusterBreakType.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Text\Unicode\TextSegmentationUtility.cs" />

src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs

@@ -4,6 +4,7 @@
 
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Runtime.InteropServices;
 using System.Runtime.Serialization;
 
@@ -1040,6 +1041,50 @@ value is Encoding that &&
         public override int GetHashCode() =>
             _codePage + this.EncoderFallback.GetHashCode() + this.DecoderFallback.GetHashCode();
 
+        /// <summary>
+        /// Creates a <see cref="Stream"/> which serves to transcode data between an inner <see cref="Encoding"/>
+        /// and an outer <see cref="Encoding"/>, similar to <see cref="Convert"/>.
+        /// </summary>
+        /// <param name="innerStream">The <see cref="Stream"/> to wrap.</param>
+        /// <param name="innerStreamEncoding">The <see cref="Encoding"/> associated with <paramref name="innerStream"/>.</param>
+        /// <param name="outerStreamEncoding">The <see cref="Encoding"/> associated with the <see cref="Stream"/> returned
+        /// by this method.</param>
+        /// <param name="leaveOpen"><see langword="true"/> if disposing the <see cref="Stream"/> returned by this method
+        /// should <em>not</em> dispose <paramref name="innerStream"/>.</param>
+        /// <returns>A <see cref="Stream"/> which transcodes the contents of <paramref name="innerStream"/>
+        /// as <paramref name="outerStreamEncoding"/>.</returns>
+        /// <remarks>
+        /// The returned <see cref="Stream"/>'s <see cref="Stream.CanRead"/> and <see cref="Stream.CanWrite"/> properties
+        /// will reflect whether <paramref name="innerStream"/> is readable or writable. If <paramref name="innerStream"/>
+        /// is full-duplex, the returned <see cref="Stream"/> will be as well. However, the returned <see cref="Stream"/>
+        /// is not seekable, even if <paramref name="innerStream"/>'s <see cref="Stream.CanSeek"/> property returns <see langword="true"/>.
+        /// </remarks>
+        public static Stream CreateTranscodingStream(Stream innerStream, Encoding innerStreamEncoding, Encoding outerStreamEncoding, bool leaveOpen = false)
+        {
+            if (innerStream is null)
+            {
+                throw new ArgumentNullException(nameof(innerStream));
+            }
+
+            if (innerStreamEncoding is null)
+            {
+                throw new ArgumentNullException(nameof(innerStreamEncoding));
+            }
+
+            if (outerStreamEncoding is null)
+            {
+                throw new ArgumentNullException(nameof(outerStreamEncoding));
+            }
+
+            // We can't entirely optimize away the case where innerStreamEncoding == outerStreamEncoding. For example,
+            // the Encoding might perform a lossy conversion when it sees invalid data, so we still need to call it
+            // to perform basic validation. It's also possible that somebody subclassed one of the built-in types
+            // like ASCIIEncoding or UTF8Encoding and is running some non-standard logic. If this becomes a bottleneck
+            // we can consider targeted optimizations in a future release.
+
+            return new TranscodingStream(innerStream, innerStreamEncoding, outerStreamEncoding, leaveOpen);
+        }
+
         internal virtual char[] GetBestFitUnicodeToBytesData() =>
             // Normally we don't have any best fit data.
             Array.Empty<char>();