Support reseolve reference for typeof(object) on deserialize (dotnet#…

…38979)

src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs

@@ -317,7 +317,7 @@ internal bool TextEquals(int index, ReadOnlySpan<char> otherText, bool isPropert
                 Debug.Assert(status == OperationStatus.Done);
                 Debug.Assert(consumed == utf16Text.Length);
 
-                result = TextEquals(index, otherUtf8Text.Slice(0, written), isPropertyName);
+                result = TextEquals(index, otherUtf8Text.Slice(0, written), isPropertyName, shouldUnescape: true);
             }
 
             if (otherUtf8TextArray != null)
@@ -329,7 +329,7 @@ internal bool TextEquals(int index, ReadOnlySpan<char> otherText, bool isPropert
             return result;
         }
 
-        internal bool TextEquals(int index, ReadOnlySpan<byte> otherUtf8Text, bool isPropertyName)
+        internal bool TextEquals(int index, ReadOnlySpan<byte> otherUtf8Text, bool isPropertyName, bool shouldUnescape)
         {
             CheckNotDisposed();
 
@@ -344,12 +344,12 @@ internal bool TextEquals(int index, ReadOnlySpan<byte> otherUtf8Text, bool isPro
             ReadOnlySpan<byte> data = _utf8Json.Span;
             ReadOnlySpan<byte> segment = data.Slice(row.Location, row.SizeOrLength);
 
-            if (otherUtf8Text.Length > segment.Length)
+            if (otherUtf8Text.Length > segment.Length || (!shouldUnescape && otherUtf8Text.Length != segment.Length))
             {
                 return false;
             }
 
-            if (row.HasComplexChildren)
+            if (row.HasComplexChildren && shouldUnescape)
             {
                 if (otherUtf8Text.Length < segment.Length / JsonConstants.MaxExpansionFactorWhileEscaping)
                 {

src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs

@@ -1229,7 +1229,7 @@ public bool ValueEquals(ReadOnlySpan<byte> utf8Text)
                 return utf8Text == default;
             }
 
-            return TextEqualsHelper(utf8Text, isPropertyName: false);
+            return TextEqualsHelper(utf8Text, isPropertyName: false, shouldUnescape: true);
         }
 
         /// <summary>
@@ -1260,11 +1260,11 @@ public bool ValueEquals(ReadOnlySpan<char> text)
             return TextEqualsHelper(text, isPropertyName: false);
         }
 
-        internal bool TextEqualsHelper(ReadOnlySpan<byte> utf8Text, bool isPropertyName)
+        internal bool TextEqualsHelper(ReadOnlySpan<byte> utf8Text, bool isPropertyName, bool shouldUnescape)
         {
             CheckValidInstance();
 
-            return _parent.TextEquals(_idx, utf8Text, isPropertyName);
+            return _parent.TextEquals(_idx, utf8Text, isPropertyName, shouldUnescape);
         }
 
         internal bool TextEqualsHelper(ReadOnlySpan<char> text, bool isPropertyName)

src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonProperty.cs

@@ -66,7 +66,7 @@ public bool NameEquals(string? text)
         /// </remarks>
         public bool NameEquals(ReadOnlySpan<byte> utf8Text)
         {
-            return Value.TextEqualsHelper(utf8Text, isPropertyName: true);
+            return Value.TextEqualsHelper(utf8Text, isPropertyName: true, shouldUnescape: true);
         }
 
         /// <summary>
@@ -89,6 +89,11 @@ public bool NameEquals(ReadOnlySpan<char> text)
             return Value.TextEqualsHelper(text, isPropertyName: true);
         }
 
+        internal bool EscapedNameEquals(ReadOnlySpan<byte> utf8Text)
+        {
+            return Value.TextEqualsHelper(utf8Text, isPropertyName: true, shouldUnescape: false);
+        }
+
         /// <summary>
         ///   Write the property into the provided writer as a named JSON object property.
         /// </summary>

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs

@@ -149,6 +149,19 @@ internal bool TryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSeriali
                         ref reader);
                 }
 
+                if (CanBePolymorphic && options.ReferenceHandler != null)
+                {
+                    // Edge case where we want to lookup for a reference when parsing into typeof(object)
+                    // instead of return `value` as a JsonElement.
+                    Debug.Assert(TypeToConvert == typeof(object));
+                    Debug.Assert(value is JsonElement);
+
+                    if (JsonSerializer.TryGetReferenceFromJsonElement(ref state, (JsonElement)(object)value, out object? referenceValue))
+                    {
+                        value = (T)referenceValue;
+                    }
+                }
+
                 return true;
             }
 

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.HandleMetadata.cs

@@ -12,6 +12,9 @@ public static partial class JsonSerializer
         internal static readonly byte[] s_idPropertyName
             = new byte[] { (byte)'$', (byte)'i', (byte)'d' };
 
+        internal static ReadOnlySpan<byte> s_refPropertyName
+            => new byte[] { (byte)'$', (byte)'r', (byte)'e', (byte)'f' };
+
         /// <summary>
         /// Returns true if successful, false is the reader ran out of buffer.
         /// Sets state.Current.ReturnValue to the $ref target for MetadataRefProperty cases.
@@ -270,5 +273,46 @@ internal static MetadataPropertyName GetMetadataPropertyName(ReadOnlySpan<byte>
 
             return MetadataPropertyName.NoMetadata;
         }
+
+        internal static bool TryGetReferenceFromJsonElement(
+            ref ReadStack state,
+            JsonElement element,
+            out object? referenceValue)
+        {
+            bool refMetadataFound = false;
+            referenceValue = default;
+
+            if (element.ValueKind == JsonValueKind.Object)
+            {
+                int propertyCount = 0;
+                foreach (JsonProperty property in element.EnumerateObject())
+                {
+                    propertyCount++;
+                    if (refMetadataFound)
+                    {
+                        // There are more properties in an object with $ref.
+                        ThrowHelper.ThrowJsonException_MetadataReferenceObjectCannotContainOtherProperties();
+                    }
+                    else if (property.EscapedNameEquals(s_refPropertyName))
+                    {
+                        if (propertyCount > 1)
+                        {
+                            // $ref was found but there were other properties before.
+                            ThrowHelper.ThrowJsonException_MetadataReferenceObjectCannotContainOtherProperties();
+                        }
+
+                        if (property.Value.ValueKind != JsonValueKind.String)
+                        {
+                            ThrowHelper.ThrowJsonException_MetadataValueWasNotString(property.Value.ValueKind);
+                        }
+
+                        referenceValue = state.ReferenceResolver.ResolveReference(property.Value.GetString()!);
+                        refMetadataFound = true;
+                    }
+                }
+            }
+
+            return refMetadataFound;
+        }
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs

@@ -473,11 +473,25 @@ public static void ThrowJsonException_MetadataValueWasNotString(JsonTokenType to
             ThrowJsonException(SR.Format(SR.MetadataValueWasNotString, tokenType));
         }
 
+        [DoesNotReturn]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        public static void ThrowJsonException_MetadataValueWasNotString(JsonValueKind valueKind)
+        {
+            ThrowJsonException(SR.Format(SR.MetadataValueWasNotString, valueKind));
+        }
+
         [DoesNotReturn]
         [MethodImpl(MethodImplOptions.NoInlining)]
         public static void ThrowJsonException_MetadataReferenceObjectCannotContainOtherProperties(ReadOnlySpan<byte> propertyName, ref ReadStack state)
         {
             state.Current.JsonPropertyName = propertyName.ToArray();
+            ThrowJsonException_MetadataReferenceObjectCannotContainOtherProperties();
+        }
+
+        [DoesNotReturn]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        public static void ThrowJsonException_MetadataReferenceObjectCannotContainOtherProperties()
+        {
             ThrowJsonException(SR.MetadataReferenceCannotContainOtherProperties);
         }
 

src/libraries/System.Text.Json/tests/Serialization/ReferenceHandlerTests.cs

@@ -3,16 +3,17 @@
 
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.IO;
 using System.Text.Encodings.Web;
 using System.Text.Json.Tests;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Xunit;
 
 namespace System.Text.Json.Serialization.Tests
 {
     public static partial class ReferenceHandlerTests
     {
-
         [Fact]
         public static void ThrowByDefaultOnLoop()
         {
@@ -697,5 +698,98 @@ public override object ResolveReference(string referenceId)
             }
         }
         #endregion
+
+        [Fact]
+        public static void PreserveReferenceOfTypeObject()
+        {
+            var root = new ClassWithObjectProperty();
+            root.Child = new ClassWithObjectProperty();
+            root.Sibling = root.Child;
+
+            Assert.Same(root.Child, root.Sibling);
+
+            string json = JsonSerializer.Serialize(root, s_serializerOptionsPreserve);
+
+            ClassWithObjectProperty rootCopy = JsonSerializer.Deserialize<ClassWithObjectProperty>(json, s_serializerOptionsPreserve);
+            Assert.Same(rootCopy.Child, rootCopy.Sibling);
+        }
+
+        [Fact]
+        public static async Task PreserveReferenceOfTypeObjectAsync()
+        {
+            var root = new ClassWithObjectProperty();
+            root.Child = new ClassWithObjectProperty();
+            root.Sibling = root.Child;
+
+            Assert.Same(root.Child, root.Sibling);
+
+            var stream = new MemoryStream();
+            await JsonSerializer.SerializeAsync(stream, root, s_serializerOptionsPreserve);
+            stream.Position = 0;
+
+            ClassWithObjectProperty rootCopy = await JsonSerializer.DeserializeAsync<ClassWithObjectProperty>(stream, s_serializerOptionsPreserve);
+            Assert.Same(rootCopy.Child, rootCopy.Sibling);
+        }
+
+        [Fact]
+        public static void PreserveReferenceOfTypeOfObjectOnCollection()
+        {
+            var root = new ClassWithListOfObjectProperty();
+            root.Child = new ClassWithListOfObjectProperty();
+
+            root.ListOfObjects = new List<object>();
+            root.ListOfObjects.Add(root.Child);
+
+            Assert.Same(root.Child, root.ListOfObjects[0]);
+
+            string json = JsonSerializer.Serialize(root, s_serializerOptionsPreserve);
+            ClassWithListOfObjectProperty rootCopy = JsonSerializer.Deserialize<ClassWithListOfObjectProperty>(json, s_serializerOptionsPreserve);
+            Assert.Same(rootCopy.Child, rootCopy.ListOfObjects[0]);
+        }
+
+        [Fact]
+        public static void DoNotPreserveReferenceWhenRefPropertyIsAbsent()
+        {
+            string json = @"{""Child"":{""$id"":""1""},""Sibling"":{""foo"":""1""}}";
+            ClassWithObjectProperty root = JsonSerializer.Deserialize<ClassWithObjectProperty>(json);
+            Assert.IsType<JsonElement>(root.Sibling);
+
+            // $ref with any escaped character shall not be treated as metadata, hence Sibling must be JsonElement.
+            json = @"{""Child"":{""$id"":""1""},""Sibling"":{""\\u0024ref"":""1""}}";
+            root = JsonSerializer.Deserialize<ClassWithObjectProperty>(json);
+            Assert.IsType<JsonElement>(root.Sibling);
+        }
+
+        [Fact]
+        public static void VerifyValidationsOnPreservedReferenceOfTypeObject()
+        {
+            const string baseJson = @"{""Child"":{""$id"":""1""},""Sibling"":";
+
+            // A JSON object that contains a '$ref' metadata property must not contain any other properties.
+            string testJson = baseJson + @"{""foo"":""value"",""$ref"":""1""}}";
+            JsonException ex = Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<ClassWithObjectProperty>(testJson, s_serializerOptionsPreserve));
+            Assert.Equal("$.Sibling", ex.Path);
+
+            testJson = baseJson + @"{""$ref"":""1"",""bar"":""value""}}";
+            ex = Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<ClassWithObjectProperty>(testJson, s_serializerOptionsPreserve));
+            Assert.Equal("$.Sibling", ex.Path);
+
+            // The '$id' and '$ref' metadata properties must be JSON strings.
+            testJson = baseJson + @"{""$ref"":1}}";
+            ex = Assert.Throws<JsonException>(() => JsonSerializer.Deserialize<ClassWithObjectProperty>(testJson, s_serializerOptionsPreserve));
+            Assert.Equal("$.Sibling", ex.Path);
+        }
+
+        private class ClassWithObjectProperty
+        {
+            public ClassWithObjectProperty Child { get; set; }
+            public object Sibling { get; set; }
+        }
+
+        private class ClassWithListOfObjectProperty
+        {
+            public ClassWithListOfObjectProperty Child { get; set; }
+            public List<object> ListOfObjects { get; set; }
+        }
     }
 }

src/libraries/System.Text.Json/tests/Serialization/Stream.Collections.cs

@@ -60,21 +60,21 @@ private static async Task PerformSerialization<TElement>(object obj, Type type,
         {
             string expectedjson = JsonSerializer.Serialize(obj, options);
 
-            using (var memoryStream = new MemoryStream())
-            {
-                await JsonSerializer.SerializeAsync(memoryStream, obj, options);
-                string serialized = Encoding.UTF8.GetString(memoryStream.ToArray());
-                JsonTestHelper.AssertJsonEqual(expectedjson, serialized);
+            using var memoryStream = new MemoryStream();
+            await JsonSerializer.SerializeAsync(memoryStream, obj, options);
+            string serialized = Encoding.UTF8.GetString(memoryStream.ToArray());
+            JsonTestHelper.AssertJsonEqual(expectedjson, serialized);
 
-                memoryStream.Position = 0;
-                await TestDeserialization<TElement>(memoryStream, expectedjson, type, options);
-            }
+            memoryStream.Position = 0;
 
-            // Deserialize with extra whitespace
-            string jsonWithWhiteSpace = GetPayloadWithWhiteSpace(expectedjson);
-            using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonWithWhiteSpace)))
+            if (options.ReferenceHandler == null || !GetTypesNonRoundtrippableWithReferenceHandler().Contains(type))
             {
                 await TestDeserialization<TElement>(memoryStream, expectedjson, type, options);
+
+                // Deserialize with extra whitespace
+                string jsonWithWhiteSpace = GetPayloadWithWhiteSpace(expectedjson);
+                using var memoryStreamWithWhiteSpace = new MemoryStream(Encoding.UTF8.GetBytes(jsonWithWhiteSpace));
+                await TestDeserialization<TElement>(memoryStreamWithWhiteSpace, expectedjson, type, options);
             }
         }
 
@@ -353,6 +353,16 @@ private static IEnumerable<Type> DictionaryTypes<TElement>()
             typeof(GenericIReadOnlyDictionaryWrapper<string, TElement>)
         };
 
+        // Non-generic types cannot roundtrip when they contain a $ref written on serialization and they are the root type.
+        private static HashSet<Type> GetTypesNonRoundtrippableWithReferenceHandler() => new HashSet<Type>
+        {
+            typeof(Hashtable),
+            typeof(Queue),
+            typeof(Stack),
+            typeof(WrapperForIList),
+            typeof(WrapperForIEnumerable)
+        };
+
         private class ClassWithKVP
         {
             public KeyValuePair<string, SimpleStruct> MyKvp { get; set; }