diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 2c5113d9f7238..58fe300c6ba3c 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -421,21 +421,25 @@ - - - + + + + + + + + - @@ -1029,6 +1033,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 47678886ffa21..1dd36f6730661 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -339,7 +339,9 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly { // Do a quick search for the first element of "value". int relativeIndex = isLetter ? - SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : + PackedSpanHelpers.PackedIndexOfIsSupported + ? PackedSpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) + : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); if (relativeIndex < 0) { diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1Value.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1ByteValue.cs similarity index 61% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1Value.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1ByteValue.cs index 565af9cf42c70..3dd3a6aa4fcaf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1Value.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1ByteValue.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny1Value : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny1ByteValue : IndexOfAnyValues { - private readonly T _e0; + private readonly byte _e0; - public IndexOfAny1Value(ReadOnlySpan values) + public IndexOfAny1ByteValue(ReadOnlySpan values) { Debug.Assert(values.Length == 1); _e0 = values[0]; } - internal override T[] GetValues() => new[] { _e0 }; + internal override byte[] GetValues() => new[] { _e0 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOf(_e0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOf(_e0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs new file mode 100644 index 0000000000000..762f1872cb64c --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny1CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0; + + public IndexOfAny1CharValue(char value) => + _e0 = value; + + internal override char[] GetValues() => new[] { _e0 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) + : SpanHelpers.NonPackedIndexOfValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) + : SpanHelpers.NonPackedIndexOfValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOf(_e0); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2Values.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2ByteValues.cs similarity index 62% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2Values.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2ByteValues.cs index cb89fcfbc497e..76f8db819156f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2Values.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2ByteValues.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny2Values : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny2ByteValues : IndexOfAnyValues { - private readonly T _e0, _e1; + private readonly byte _e0, _e1; - public IndexOfAny2Values(ReadOnlySpan values) + public IndexOfAny2ByteValues(ReadOnlySpan values) { Debug.Assert(values.Length == 2); (_e0, _e1) = (values[0], values[1]); } - internal override T[] GetValues() => new[] { _e0, _e1 }; + internal override byte[] GetValues() => new[] { _e0, _e1 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0 || value == _e1; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAny(_e0, _e1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0, _e1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAny(_e0, _e1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0, _e1); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs new file mode 100644 index 0000000000000..f649ffc752657 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny2CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0, _e1; + + public IndexOfAny2CharValue(char value0, char value1) => + (_e0, _e1) = (value0, value1); + + internal override char[] GetValues() => new[] { _e0, _e1 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs similarity index 63% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs index f94509b19f6d6..ad569e18589ac 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny3Values : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny3ByteValues : IndexOfAnyValues { - private readonly T _e0, _e1, _e2; + private readonly byte _e0, _e1, _e2; - public IndexOfAny3Values(ReadOnlySpan values) + public IndexOfAny3ByteValues(ReadOnlySpan values) { Debug.Assert(values.Length == 3); (_e0, _e1, _e2) = (values[0], values[1], values[2]); } - internal override T[] GetValues() => new[] { _e0, _e1, _e2 }; + internal override byte[] GetValues() => new[] { _e0, _e1, _e2 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0 || value == _e1 || value == _e2; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAny(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAny(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0, _e1, _e2); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs new file mode 100644 index 0000000000000..5ae583b0f174b --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny3CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0, _e1, _e2; + + public IndexOfAny3CharValue(char value0, char value1, char value2) => + (_e0, _e1, _e2) = (value0, value1, value2); + + internal override char[] GetValues() => new[] { _e0, _e1, _e2 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1 || value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + Unsafe.As(ref _e2), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + Unsafe.As(ref _e2), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1, _e2); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1, _e2); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyByteValuesInRange.cs similarity index 52% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValuesInRange.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyByteValuesInRange.cs index c9c4f1a1f499a..4561a2170ec78 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValuesInRange.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyByteValuesInRange.cs @@ -1,58 +1,53 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAnyValuesInRange : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAnyByteValuesInRange : IndexOfAnyValues { - private readonly T _lowInclusive, _highInclusive; + private readonly byte _lowInclusive, _highInclusive; private readonly uint _lowUint, _highMinusLow; - public IndexOfAnyValuesInRange(T lowInclusive, T highInclusive) + public IndexOfAnyByteValuesInRange(byte lowInclusive, byte highInclusive) { - Debug.Assert(lowInclusive is byte or char); (_lowInclusive, _highInclusive) = (lowInclusive, highInclusive); - _lowUint = uint.CreateChecked(lowInclusive); - _highMinusLow = uint.CreateChecked(highInclusive - lowInclusive); + _lowUint = lowInclusive; + _highMinusLow = (uint)(highInclusive - lowInclusive); } - internal override T[] GetValues() + internal override byte[] GetValues() { - T[] values = new T[_highMinusLow + 1]; + byte[] values = new byte[_highMinusLow + 1]; - T element = _lowInclusive; + int low = _lowInclusive; for (int i = 0; i < values.Length; i++) { - values[i] = element; - element += T.One; + values[i] = (byte)(low + i); } return values; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => - uint.CreateChecked(value) - _lowUint <= _highMinusLow; + internal override bool ContainsCore(byte value) => + value - _lowUint <= _highMinusLow; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAnyInRange(_lowInclusive, _highInclusive); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExceptInRange(_lowInclusive, _highInclusive); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAnyInRange(_lowInclusive, _highInclusive); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExceptInRange(_lowInclusive, _highInclusive); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs new file mode 100644 index 0000000000000..018ef273b4839 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAnyCharValuesInRange : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _lowInclusive, _rangeInclusive, _highInclusive; + private readonly uint _lowUint, _highMinusLow; + + public IndexOfAnyCharValuesInRange(char lowInclusive, char highInclusive) + { + (_lowInclusive, _rangeInclusive, _highInclusive) = (lowInclusive, (char)(highInclusive - lowInclusive), highInclusive); + _lowUint = lowInclusive; + _highMinusLow = (uint)(highInclusive - lowInclusive); + } + + internal override char[] GetValues() + { + char[] values = new char[_rangeInclusive + 1]; + + int low = _lowInclusive; + for (int i = 0; i < values.Length; i++) + { + values[i] = (char)(low + i); + } + + return values; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value - _lowUint <= _highMinusLow; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _lowInclusive), + Unsafe.As(ref _highInclusive), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.IndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _lowInclusive), + Unsafe.As(ref _highInclusive), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAnyInRange(_lowInclusive, _highInclusive); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExceptInRange(_lowInclusive, _highInclusive); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs index 8ca2624750027..24c62bd92b969 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs @@ -8,6 +8,8 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +#pragma warning disable 8500 // address of managed types + namespace System.Buffers { /// @@ -31,7 +33,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 1) { - return new IndexOfAny1Value(values); + return new IndexOfAny1ByteValue(values); } // IndexOfAnyValuesInRange is slower than IndexOfAny1Value, but faster than IndexOfAny2Values @@ -45,8 +47,8 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) Debug.Assert(values.Length is 2 or 3 or 4 or 5); return values.Length switch { - 2 => new IndexOfAny2Values(values), - 3 => new IndexOfAny3Values(values), + 2 => new IndexOfAny2ByteValues(values), + 3 => new IndexOfAny3ByteValues(values), 4 => new IndexOfAny4Values(values), _ => new IndexOfAny5Values(values), }; @@ -77,7 +79,10 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 1) { - return new IndexOfAny1Value(values); + char value = values[0]; + return PackedSpanHelpers.CanUsePackedIndexOf(value) + ? new IndexOfAny1CharValue(value) + : new IndexOfAny1CharValue(value); } // IndexOfAnyValuesInRange is slower than IndexOfAny1Value, but faster than IndexOfAny2Values @@ -88,12 +93,21 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 2) { - return new IndexOfAny2Values(values); + char value0 = values[0]; + char value1 = values[1]; + return PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) + ? new IndexOfAny2CharValue(value0, value1) + : new IndexOfAny2CharValue(value0, value1); } if (values.Length == 3) { - return new IndexOfAny3Values(values); + char value0 = values[0]; + char value1 = values[1]; + char value2 = values[2]; + return PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2) + ? new IndexOfAny3CharValue(value0, value1, value2) + : new IndexOfAny3CharValue(value0, value1, value2); } // IndexOfAnyAsciiSearcher for chars is slower than IndexOfAny3Values, but faster than IndexOfAny4Values @@ -130,7 +144,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), return new IndexOfAnyCharValuesProbabilistic(values); } - private static IndexOfAnyValues? TryGetSingleRange(ReadOnlySpan values, out T maxInclusive) + private static unsafe IndexOfAnyValues? TryGetSingleRange(ReadOnlySpan values, out T maxInclusive) where T : struct, INumber, IMinMaxValue { T min = T.MaxValue; @@ -165,7 +179,30 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), return null; } - return (IndexOfAnyValues)(object)new IndexOfAnyValuesInRange(min, max); + if (typeof(T) == typeof(byte)) + { + return (IndexOfAnyValues)(object)new IndexOfAnyByteValuesInRange(byte.CreateChecked(min), byte.CreateChecked(max)); + } + + Debug.Assert(typeof(T) == typeof(char)); + return (IndexOfAnyValues)(object)(PackedSpanHelpers.CanUsePackedIndexOf(min) && PackedSpanHelpers.CanUsePackedIndexOf(max) + ? new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max) + : new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max)); + } + + internal interface IRuntimeConst + { + static abstract bool Value { get; } + } + + private readonly struct TrueConst : IRuntimeConst + { + public static bool Value => true; + } + + private readonly struct FalseConst : IRuntimeConst + { + public static bool Value => false; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 23c3e86ef0ad0..595b6969960dd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -558,7 +558,7 @@ public static int IndexOfAnyExcept(this Span span, IndexOfAnyValues val [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int IndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -609,7 +609,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -673,7 +673,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int IndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -864,7 +864,7 @@ public static int LastIndexOfAnyExcept(this Span span, IndexOfAnyValues [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int LastIndexOfAnyExcept(this ReadOnlySpan span, T value) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -915,7 +915,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -979,7 +979,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int LastIndexOfAnyExcept(this ReadOnlySpan span, T value0, T value1, T value2, T value3) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { if (sizeof(T) == sizeof(byte)) { @@ -3067,7 +3067,7 @@ public static void Sort(this Span keys, Span items, [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe void Replace(this Span span, T oldValue, T newValue) where T : IEquatable? { - if (SpanHelpers.CanVectorizeAndBenefit(span.Length)) + if (RuntimeHelpers.IsBitwiseEquatable()) { nuint length = (uint)span.Length; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs new file mode 100644 index 0000000000000..3347348ee187a --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -0,0 +1,873 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 + +#pragma warning disable 8500 // sizeof of managed types + +namespace System +{ + // This is a separate class instead of 'partial SpanHelpers' to hide the private helpers + // included in this file which are specific to the packed implementation. + internal static partial class PackedSpanHelpers + { + // We only do this optimization if we have support for X86 intrinsics (Sse2) as the packing is noticeably cheaper compared to ARM (AdvSimd). + // While the impact on the worst-case (match at the start) is minimal on X86, it's prohibitively large on ARM. + public static bool PackedIndexOfIsSupported => Sse2.IsSupported; + + // Not all values can benefit from packing the searchSpace. See comments in PackSources below. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe bool CanUsePackedIndexOf(T value) => + PackedIndexOfIsSupported && + RuntimeHelpers.IsBitwiseEquatable() && + sizeof(T) == sizeof(ushort) && + *(ushort*)&value - 1u < 254u; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOf(ref char searchSpace, char value, int length) => + IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAnyExcept(ref char searchSpace, char value, int length) => + IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAny(ref char searchSpace, char value0, char value1, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + IndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + IndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + public static bool Contains(ref short searchSpace, short value, int length) + { + Debug.Assert(CanUsePackedIndexOf(value)); + + if (length < Vector128.Count) + { + nuint offset = 0; + + if (length >= 4) + { + length -= 4; + + if (searchSpace == value || + Unsafe.Add(ref searchSpace, 1) == value || + Unsafe.Add(ref searchSpace, 2) == value || + Unsafe.Add(ref searchSpace, 3) == value) + { + return true; + } + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + if (Unsafe.Add(ref searchSpace, offset) == value) + { + return true; + } + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue = Vector256.Create((byte)value); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + + if (result != Vector256.Zero) + { + return true; + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we're only interested in whether any value matched. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + + if (result != Vector256.Zero) + { + return true; + } + } + } + else + { + Vector128 packedValue = Vector128.Create((byte)value); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + + if (result != Vector128.Zero) + { + return true; + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we're only interested in whether any value matched. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + + if (result != Vector128.Zero) + { + return true; + } + } + } + } + + return false; + } + + private static int IndexOf(ref short searchSpace, short value, int length) + where TNegator : struct, SpanHelpers.INegator + { + Debug.Assert(CanUsePackedIndexOf(value)); + + if (length < Vector128.Count) + { + nuint offset = 0; + + if (length >= 4) + { + length -= 4; + + if (TNegator.NegateIfNeeded(searchSpace == value)) return 0; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 1) == value)) return 1; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 2) == value)) return 2; + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 3) == value)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue = Vector256.Create((byte)value); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue = Vector128.Create((byte)value); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) + where TNegator : struct, SpanHelpers.INegator + { + Debug.Assert(CanUsePackedIndexOf(value0)); + Debug.Assert(CanUsePackedIndexOf(value1)); + + if (length < Vector128.Count) + { + nuint offset = 0; + short lookUp; + + if (length >= 4) + { + length -= 4; + + lookUp = searchSpace; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 0; + lookUp = Unsafe.Add(ref searchSpace, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 1; + lookUp = Unsafe.Add(ref searchSpace, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 2; + lookUp = Unsafe.Add(ref searchSpace, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue0 = Vector256.Create((byte)value0); + Vector256 packedValue1 = Vector256.Create((byte)value1); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue0 = Vector128.Create((byte)value0); + Vector128 packedValue1 = Vector128.Create((byte)value1); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) + where TNegator : struct, SpanHelpers.INegator + { + Debug.Assert(CanUsePackedIndexOf(value0)); + Debug.Assert(CanUsePackedIndexOf(value1)); + Debug.Assert(CanUsePackedIndexOf(value2)); + + if (length < Vector128.Count) + { + nuint offset = 0; + short lookUp; + + if (length >= 4) + { + length -= 4; + + lookUp = searchSpace; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 0; + lookUp = Unsafe.Add(ref searchSpace, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 1; + lookUp = Unsafe.Add(ref searchSpace, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 2; + lookUp = Unsafe.Add(ref searchSpace, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue0 = Vector256.Create((byte)value0); + Vector256 packedValue1 = Vector256.Create((byte)value1); + Vector256 packedValue2 = Vector256.Create((byte)value2); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue0 = Vector128.Create((byte)value0); + Vector128 packedValue1 = Vector128.Create((byte)value1); + Vector128 packedValue2 = Vector128.Create((byte)value2); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + private static int IndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) + where TNegator : struct, SpanHelpers.INegator + { + Debug.Assert(CanUsePackedIndexOf(lowInclusive)); + Debug.Assert(CanUsePackedIndexOf((short)(lowInclusive + rangeInclusive))); + Debug.Assert(rangeInclusive >= 0); + + if (length < Vector128.Count) + { + uint lowInclusiveUint = (uint)lowInclusive; + uint rangeInclusiveUint = (uint)rangeInclusive; + for (int i = 0; i < length; i++) + { + uint current = (uint)Unsafe.Add(ref searchSpace, i); + if (TNegator.NegateIfNeeded((current - lowInclusiveUint) <= rangeInclusiveUint)) + { + return i; + } + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 lowVector = Vector256.Create((byte)lowInclusive); + Vector256 rangeVector = Vector256.Create((byte)rangeInclusive); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 lowVector = Vector128.Create((byte)lowInclusive); + Vector128 rangeVector = Vector128.Create((byte)rangeInclusive); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.LessThanOrEqual(packedSource - lowVector, rangeVector); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 PackSources(Vector256 source0, Vector256 source1) + { + Debug.Assert(Avx2.IsSupported); + // Pack two vectors of characters into bytes. While the type is Vector256, these are really UInt16 characters. + // X86: Downcast every character using saturation. + // - Values <= 32767 result in min(value, 255). + // - Values > 32767 result in 0. Because of this we can't accept needles that contain 0. + return Avx2.PackUnsignedSaturate(source0, source1).AsByte(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 PackSources(Vector128 source0, Vector128 source1) + { + Debug.Assert(Sse2.IsSupported); + // Pack two vectors of characters into bytes. While the type is Vector128, these are really UInt16 characters. + // X86: Downcast every character using saturation. + // - Values <= 32767 result in min(value, 255). + // - Values > 32767 result in 0. Because of this we can't accept needles that contain 0. + return Sse2.PackUnsignedSaturate(source0, source1).AsByte(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 NegateIfNeeded(Vector128 result) + where TNegator : struct, SpanHelpers.INegator => + typeof(TNegator) == typeof(SpanHelpers.DontNegate) ? result : ~result; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 NegateIfNeeded(Vector256 result) + where TNegator : struct, SpanHelpers.INegator => + typeof(TNegator) == typeof(SpanHelpers.DontNegate) ? result : ~result; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndex(ref short searchSpace, ref short current, Vector128 equals) + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / sizeof(short)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndex(ref short searchSpace, ref short current, Vector256 equals) + { + uint notEqualsElements = FixUpPackedVector256Mask(equals.ExtractMostSignificantBits()); + int index = BitOperations.TrailingZeroCount(notEqualsElements); + return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / sizeof(short)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short current0, ref short current1, Vector128 equals) + { + uint notEqualsElements = equals.ExtractMostSignificantBits(); + int offsetInVector = BitOperations.TrailingZeroCount(notEqualsElements); + if (offsetInVector >= Vector128.Count) + { + // We matched within the second vector + current0 = ref current1; + offsetInVector -= Vector128.Count; + } + return offsetInVector + (int)(Unsafe.ByteOffset(ref searchSpace, ref current0) / sizeof(short)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short current0, ref short current1, Vector256 equals) + { + uint notEqualsElements = FixUpPackedVector256Mask(equals.ExtractMostSignificantBits()); + int offsetInVector = BitOperations.TrailingZeroCount(notEqualsElements); + if (offsetInVector >= Vector256.Count) + { + // We matched within the second vector + current0 = ref current1; + offsetInVector -= Vector256.Count; + } + return offsetInVector + (int)(Unsafe.ByteOffset(ref searchSpace, ref current0) / sizeof(short)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint FixUpPackedVector256Mask(uint mask) + { + Debug.Assert(Avx2.IsSupported); + // Avx2.PackUnsignedSaturate(Vector256.Create((short)1), Vector256.Create((short)2)) will result in + // 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 + // We want to swap the X and Y bits + // 1, 1, 1, 1, 1, 1, 1, 1, X, X, X, X, X, X, X, X, Y, Y, Y, Y, Y, Y, Y, Y, 2, 2, 2, 2, 2, 2, 2, 2 + const uint CorrectPositionsMask = 0xFF0000FF; + + return (mask & CorrectPositionsMask) | BinaryPrimitives.ReverseEndianness(mask & ~CorrectPositionsMask); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 66bf0618cab3a..d3d643f13ee64 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1304,33 +1304,18 @@ public static int SequenceCompareTo(ref T first, int firstLength, ref T secon } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe bool CanVectorizeAndBenefit(int length) where T : IEquatable? + internal static unsafe bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber { - if (Vector128.IsHardwareAccelerated && RuntimeHelpers.IsBitwiseEquatable()) + if (PackedSpanHelpers.CanUsePackedIndexOf(value)) { - if (sizeof(T) == sizeof(byte)) - { - return length >= Vector128.Count; - } - else if (sizeof(T) == sizeof(short)) - { - return length >= Vector128.Count; - } - else if (sizeof(T) == sizeof(int)) - { - return length >= Vector128.Count; - } - else if (sizeof(T) == sizeof(long)) - { - return length >= Vector128.Count; - } + return PackedSpanHelpers.Contains(ref Unsafe.As(ref searchSpace), *(short*)&value, length); } - return false; + return NonPackedContainsValueType(ref searchSpace, value, length); } [MethodImpl(MethodImplOptions.AggressiveOptimization)] - internal static bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber + internal static bool NonPackedContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber { Debug.Assert(length >= 0, "Expected non-negative length"); Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types"); @@ -1458,8 +1443,23 @@ internal static int IndexOfValueType(ref T searchSpace, T value, int length) internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value, int length) where T : struct, INumber => IndexOfValueType>(ref searchSpace, value, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfValueType(ref TValue searchSpace, TValue value, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.IndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) + : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); + } + + return NonPackedIndexOfValueType(ref searchSpace, value, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfValueType(ref TValue searchSpace, TValue value, int length) + internal static int NonPackedIndexOfValueType(ref TValue searchSpace, TValue value, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -1600,9 +1600,24 @@ internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1 internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber => IndexOfAnyValueType>(ref searchSpace, value0, value1, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.IndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) + : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); + } + + return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, length); + } + // having INumber constraint here allows to use == operator and get better perf compared to .Equals [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) + internal static int NonPackedIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -1762,8 +1777,23 @@ internal static int IndexOfAnyValueType(ref T searchSpace, T value0, T value1 internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.IndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length) + : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length); + } + + return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, value2, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + internal static int NonPackedIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -3050,7 +3080,26 @@ internal static int IndexOfAnyExceptInRangeUnsignedNumber(ref T searchSpace, where T : struct, IUnsignedNumber, IComparisonOperators => IndexOfAnyInRangeUnsignedNumber>(ref searchSpace, lowInclusive, highInclusive, length); - private static int IndexOfAnyInRangeUnsignedNumber(ref T searchSpace, T lowInclusive, T highInclusive, int length) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyInRangeUnsignedNumber(ref T searchSpace, T lowInclusive, T highInclusive, int length) + where T : struct, IUnsignedNumber, IComparisonOperators + where TNegator : struct, INegator + { + if (PackedSpanHelpers.CanUsePackedIndexOf(lowInclusive) && PackedSpanHelpers.CanUsePackedIndexOf(highInclusive) && highInclusive >= lowInclusive) + { + ref char charSearchSpace = ref Unsafe.As(ref searchSpace); + char charLowInclusive = *(char*)&lowInclusive; + char charRange = (char)(*(char*)&highInclusive - charLowInclusive); + + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.IndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) + : PackedSpanHelpers.IndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); + } + + return NonPackedIndexOfAnyInRangeUnsignedNumber(ref searchSpace, lowInclusive, highInclusive, length); + } + + internal static int NonPackedIndexOfAnyInRangeUnsignedNumber(ref T searchSpace, T lowInclusive, T highInclusive, int length) where T : struct, IUnsignedNumber, IComparisonOperators where TNegator : struct, INegator { diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index df95295a116fa..331fd3392e988 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -78,7 +78,9 @@ private int IndexOfCharOrdinalIgnoreCase(char value) { char valueUc = (char)(value | 0x20); char valueLc = (char)(value & ~0x20); - return SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); + return PackedSpanHelpers.PackedIndexOfIsSupported + ? PackedSpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length) + : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } return SpanHelpers.IndexOfChar(ref _firstChar, value, Length);