Change Signature telemetry

alexcovington · Apr 20, 2020 · e941fa8 · e941fa8
1 parent cf881c6
commit e941fa8
Show file tree

Hide file tree

Showing 7 changed files with 303 additions and 1 deletion.
diff --git a/src/Features/Core/Portable/ChangeSignature/AbstractChangeSignatureService.cs b/src/Features/Core/Portable/ChangeSignature/AbstractChangeSignatureService.cs
@@ -252,6 +252,8 @@ private static async Task<ImmutableArray<ReferencedSymbol>> FindChangeSignatureR
         private bool TryCreateUpdatedSolution(
             ChangeSignatureAnalysisSucceededContext context, ChangeSignatureOptionsResult options, CancellationToken cancellationToken, [NotNullWhen(true)] out Solution? updatedSolution)
         {
+            var telemetryTimer = Stopwatch.StartNew();
+
             updatedSolution = null;
 
             var currentSolution = context.Solution;
@@ -268,6 +270,9 @@ private bool TryCreateUpdatedSolution(
 
             var declaredSymbolParametersCount = declaredSymbol.GetParameters().Length;
 
+            int telemetryNumberOfDeclarationsToUpdate = 0;
+            int telemetryNumberOfReferencesToUpdate = 0;
+
             foreach (var symbol in symbols)
             {
                 var methodSymbol = symbol.Definition as IMethodSymbol;
@@ -348,6 +353,7 @@ private bool TryCreateUpdatedSolution(
                             nodesToUpdate.Add(documentId, new List<SyntaxNode>());
                         }
 
+                        telemetryNumberOfDeclarationsToUpdate++;
                         AddUpdatableNodeToDictionaries(nodesToUpdate, documentId, nodeToUpdate, definitionToUse, symbolWithSemanticParameters);
                     }
                 }
@@ -371,6 +377,7 @@ private bool TryCreateUpdatedSolution(
                         nodesToUpdate.Add(documentId2, new List<SyntaxNode>());
                     }
 
+                    telemetryNumberOfReferencesToUpdate++;
                     AddUpdatableNodeToDictionaries(nodesToUpdate, documentId2, nodeToUpdate2, definitionToUse, symbolWithSemanticParameters);
                 }
             }
@@ -433,6 +440,9 @@ private bool TryCreateUpdatedSolution(
                 currentSolution = currentSolution.WithDocumentSyntaxRoot(docId, formattedDoc.GetSyntaxRootSynchronously(cancellationToken)!);
             }
 
+            telemetryTimer.Stop();
+            ChangeSignatureLogger.LogCommitInformation(telemetryNumberOfDeclarationsToUpdate, telemetryNumberOfReferencesToUpdate, (int)telemetryTimer.ElapsedMilliseconds);
+
             updatedSolution = currentSolution;
             return true;
         }

diff --git a/src/Features/Core/Portable/ChangeSignature/ChangeSignatureTelemetryLogger.cs b/src/Features/Core/Portable/ChangeSignature/ChangeSignatureTelemetryLogger.cs
@@ -0,0 +1,213 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.CodeAnalysis.Internal.Log;
+
+namespace Microsoft.CodeAnalysis
+{
+    internal class ChangeSignatureLogger
+    {
+        private const string Maximum = nameof(Maximum);
+        private const string Minimum = nameof(Minimum);
+        private const string Mean = nameof(Mean);
+
+        private static readonly LogAggregator s_logAggregator = new LogAggregator();
+        private static readonly StatisticLogAggregator s_statisticLogAggregator = new StatisticLogAggregator();
+        private static readonly HistogramLogAggregator s_histogramLogAggregator = new HistogramLogAggregator(bucketSize: 50, maxBucketValue: 1000);
+
+        internal enum ActionInfo
+        {
+            // Calculate % of successful dialog launches
+            ChangeSignatureDialogLaunched,
+            ChangeSignatureDialogCommitted,
+            ChangeSignatureCommitCompleted,
+
+            // Calculate % of successful dialog launches
+            AddParameterDialogLaunched,
+            AddParameterDialogCommitted,
+
+            // Which transformations were done
+            CommittedSessionAddedRemovedReordered,
+            CommittedSessionAddedRemovedOnly,
+            CommittedSessionAddedReorderedOnly,
+            CommittedSessionRemovedReorderedOnly,
+            CommittedSessionAddedOnly,
+            CommittedSessionRemovedOnly,
+            CommittedSessionReorderedOnly,
+
+            // Signature change specification details
+            CommittedSession_OriginalParameterCount,
+            CommittedSessionWithRemoved_NumberRemoved,
+            CommittedSessionWithAdded_NumberAdded,
+
+            // Signature change commit information
+            CommittedSessionNumberOfDeclarationsUpdated,
+            CommittedSessionNumberOfCallSitesUpdated,
+            CommittedSessionCommitElapsedMS,
+
+            // Added parameter binds or doesn't bind
+            AddedParameterTypeBinds,
+
+            // Added parameter required or optional w/default
+            AddedParameterRequired,
+
+            // Added parameter callsite value options
+            AddedParameterValueExplicit,
+            AddedParameterValueExplicitNamed,
+            AddedParameterValueTODO,
+            AddedParameterValueOmitted
+        }
+
+        internal static void LogChangeSignatureDialogLaunched() =>
+            s_logAggregator.IncreaseCount((int)ActionInfo.ChangeSignatureDialogLaunched);
+
+        internal static void LogChangeSignatureDialogCommitted() =>
+            s_logAggregator.IncreaseCount((int)ActionInfo.ChangeSignatureDialogCommitted);
+
+        internal static void LogAddParameterDialogLaunched() =>
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddParameterDialogLaunched);
+
+        internal static void LogAddParameterDialogCommitted() =>
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddParameterDialogCommitted);
+
+        internal static void LogTransformationInformation(int numOriginalParameters, int numParametersAdded, int numParametersRemoved, bool anyParametersReordered)
+        {
+            LogTransformationCombination(numParametersAdded > 0, numParametersRemoved > 0, anyParametersReordered);
+
+            s_logAggregator.IncreaseCountBy((int)ActionInfo.CommittedSession_OriginalParameterCount, numOriginalParameters);
+
+            if (numParametersAdded > 0)
+            {
+                s_logAggregator.IncreaseCountBy((int)ActionInfo.CommittedSessionWithAdded_NumberAdded, numParametersAdded);
+            }
+
+            if (numParametersRemoved > 0)
+            {
+                s_logAggregator.IncreaseCountBy((int)ActionInfo.CommittedSessionWithRemoved_NumberRemoved, numParametersRemoved);
+            }
+        }
+
+        private static void LogTransformationCombination(bool parametersAdded, bool parametersRemoved, bool parametersReordered)
+        {
+            // All three transformations
+            if (parametersAdded && parametersRemoved && parametersReordered)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionAddedRemovedReordered);
+                return;
+            }
+
+            // Two transformations
+            if (parametersAdded && parametersRemoved)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionAddedRemovedOnly);
+                return;
+            }
+
+            if (parametersAdded && parametersReordered)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionAddedReorderedOnly);
+                return;
+            }
+
+            if (parametersRemoved && parametersReordered)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionRemovedReorderedOnly);
+                return;
+            }
+
+            // One transformation
+            if (parametersAdded)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionAddedOnly);
+                return;
+            }
+
+            if (parametersRemoved)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionRemovedOnly);
+                return;
+            }
+
+            if (parametersReordered)
+            {
+                s_logAggregator.IncreaseCount((int)ActionInfo.CommittedSessionReorderedOnly);
+                return;
+            }
+        }
+
+        internal static void LogCommitInformation(int numDeclarationsUpdated, int numCallSitesUpdated, int elapsedMS)
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.ChangeSignatureCommitCompleted);
+
+            s_logAggregator.IncreaseCountBy((int)ActionInfo.CommittedSessionNumberOfDeclarationsUpdated, numDeclarationsUpdated);
+            s_logAggregator.IncreaseCountBy((int)ActionInfo.CommittedSessionNumberOfCallSitesUpdated, numCallSitesUpdated);
+
+            s_statisticLogAggregator.AddDataPoint((int)ActionInfo.CommittedSessionCommitElapsedMS, elapsedMS);
+            s_histogramLogAggregator.IncreaseCount((int)ActionInfo.CommittedSessionCommitElapsedMS, elapsedMS);
+        }
+
+        internal static void LogAddedParameterTypeBinds()
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddedParameterTypeBinds);
+        }
+
+        internal static void LogAddedParameterRequired()
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddedParameterRequired);
+        }
+
+        internal static void LogAddedParameter_ValueExplicit()
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddedParameterValueExplicit);
+        }
+
+        internal static void LogAddedParameter_ValueExplicitNamed()
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddedParameterValueExplicitNamed);
+        }
+
+        internal static void LogAddedParameter_ValueTODO()
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddedParameterValueTODO);
+        }
+
+        internal static void LogAddedParameter_ValueOmitted()
+        {
+            s_logAggregator.IncreaseCount((int)ActionInfo.AddedParameterValueOmitted);
+        }
+
+        internal static void ReportTelemetry()
+        {
+            Logger.Log(FunctionId.ChangeSignature_Data, KeyValueLogMessage.Create(m =>
+            {
+                foreach (var kv in s_logAggregator)
+                {
+                    var info = ((ActionInfo)kv.Key).ToString("f");
+                    m[info] = kv.Value.GetCount();
+                }
+
+                foreach (var kv in s_statisticLogAggregator)
+                {
+                    var info = ((ActionInfo)kv.Key).ToString("f");
+                    var statistics = kv.Value.GetStatisticResult();
+
+                    m[CreateProperty(info, Maximum)] = statistics.Maximum;
+                    m[CreateProperty(info, Minimum)] = statistics.Minimum;
+                    m[CreateProperty(info, Mean)] = statistics.Mean;
+                }
+
+                foreach (var kv in s_histogramLogAggregator)
+                {
+                    var info = ((ActionInfo)kv.Key).ToString("f");
+                    m[$"{info}.BucketSize"] = kv.Value.BucketSize;
+                    m[$"{info}.MaxBucketValue"] = kv.Value.MaxBucketValue;
+                    m[$"{info}.Buckets"] = kv.Value.GetBucketsAsString();
+                }
+            }));
+        }
+
+        private static string CreateProperty(string parent, string child)
+            => parent + "." + child;
+    }
+}
diff --git a/src/Features/Core/Portable/ChangeSignature/SignatureChange.cs b/src/Features/Core/Portable/ChangeSignature/SignatureChange.cs
@@ -6,7 +6,10 @@
 
 using System;
 using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
 using Microsoft.CodeAnalysis.Shared.Extensions;
+using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.ChangeSignature
 {
@@ -55,5 +58,66 @@ public SignatureChange(ParameterConfiguration originalConfiguration, ParameterCo
 
         internal SignatureChange WithoutAddedParameters()
             => new SignatureChange(OriginalConfiguration, UpdatedConfiguration.WithoutAddedParameters());
+
+        internal void LogTelemetry()
+        {
+            var originalListOfParameters = OriginalConfiguration.ToListOfParameters();
+            var updatedListOfParameters = UpdatedConfiguration.ToListOfParameters();
+
+            ChangeSignatureLogger.LogTransformationInformation(
+                numOriginalParameters: originalListOfParameters.Length,
+                numParametersAdded: updatedListOfParameters.Count(p => p is AddedParameter),
+                numParametersRemoved: originalListOfParameters.Count(p => !updatedListOfParameters.Contains(p)),
+                anyParametersReordered: AnyParametersReordered(originalListOfParameters, updatedListOfParameters));
+
+            foreach (var addedParameter in updatedListOfParameters.OfType<AddedParameter>())
+            {
+                if (addedParameter.IsRequired)
+                {
+                    ChangeSignatureLogger.LogAddedParameterRequired();
+                }
+
+                if (addedParameter.TypeBinds)
+                {
+                    ChangeSignatureLogger.LogAddedParameterTypeBinds();
+                }
+
+                if (addedParameter.IsCallsiteTodo)
+                {
+                    ChangeSignatureLogger.LogAddedParameter_ValueTODO();
+                }
+                else if (addedParameter.IsCallsiteOmitted)
+                {
+                    ChangeSignatureLogger.LogAddedParameter_ValueOmitted();
+                }
+                else
+                {
+                    if (addedParameter.UseNamedArguments)
+                    {
+                        ChangeSignatureLogger.LogAddedParameter_ValueExplicitNamed();
+                    }
+                    else
+                    {
+                        ChangeSignatureLogger.LogAddedParameter_ValueExplicit();
+                    }
+                }
+            }
+        }
+
+        private bool AnyParametersReordered(ImmutableArray<Parameter> originalListOfParameters, ImmutableArray<Parameter> updatedListOfParameters)
+        {
+            var originalListWithoutRemovedOrAdded = originalListOfParameters.Where(p => updatedListOfParameters.Contains(p)).ToImmutableArray();
+            var updatedListWithoutRemovedOrAdded = updatedListOfParameters.Where(p => originalListOfParameters.Contains(p)).ToImmutableArray();
+
+            for (int i = 0; i < originalListWithoutRemovedOrAdded.Length; i++)
+            {
+                if (originalListWithoutRemovedOrAdded[i] != updatedListWithoutRemovedOrAdded[i])
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
     }
 }
diff --git a/src/VisualStudio/Core/Def/Implementation/ChangeSignature/ChangeSignatureDialog.xaml.cs b/src/VisualStudio/Core/Def/Implementation/ChangeSignature/ChangeSignatureDialog.xaml.cs
@@ -7,6 +7,7 @@
 using System.Windows.Controls;
 using System.Windows.Input;
 using System.Windows.Media;
+using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.ChangeSignature;
 using Microsoft.VisualStudio.PlatformUI;
 
@@ -133,8 +134,12 @@ private void Add_Click(object sender, RoutedEventArgs e)
             var dialog = new AddParameterDialog(addParameterViewModel);
             var result = dialog.ShowModal();
 
+            ChangeSignatureLogger.LogAddParameterDialogLaunched();
+
             if (result.HasValue && result.Value)
             {
+                ChangeSignatureLogger.LogAddParameterDialogCommitted();
+
                 var addedParameter = new AddedParameter(
                     addParameterViewModel.TypeSymbol,
                     addParameterViewModel.TypeName,

diff --git a/...udio/Core/Def/Implementation/ChangeSignature/VisualStudioChangeSignatureOptionsService.cs b/...udio/Core/Def/Implementation/ChangeSignature/VisualStudioChangeSignatureOptionsService.cs
@@ -44,12 +44,19 @@ public VisualStudioChangeSignatureOptionsService(
                 _classificationFormatMap,
                 _classificationTypeMap);
 
+            ChangeSignatureLogger.LogChangeSignatureDialogLaunched();
+
             var dialog = new ChangeSignatureDialog(viewModel);
             var result = dialog.ShowModal();
 
             if (result.HasValue && result.Value)
             {
-                return new ChangeSignatureOptionsResult(new SignatureChange(parameters, viewModel.GetParameterConfiguration()), previewChanges: viewModel.PreviewChanges);
+                ChangeSignatureLogger.LogChangeSignatureDialogCommitted();
+
+                var signatureChange = new SignatureChange(parameters, viewModel.GetParameterConfiguration());
+                signatureChange.LogTelemetry();
+
+                return new ChangeSignatureOptionsResult(signatureChange, previewChanges: viewModel.PreviewChanges);
             }
 
             return null;

diff --git a/src/VisualStudio/Core/Def/RoslynPackage.cs b/src/VisualStudio/Core/Def/RoslynPackage.cs
@@ -271,6 +271,7 @@ private void ReportSessionWideTelemetry()
             SolutionLogger.ReportTelemetry();
             AsyncCompletionLogger.ReportTelemetry();
             CompletionProvidersLogger.ReportTelemetry();
+            ChangeSignatureLogger.ReportTelemetry();
             SyntacticLspLogger.ReportTelemetry();
         }
 

diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/Log/FunctionId.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/Log/FunctionId.cs
@@ -479,5 +479,7 @@ internal enum FunctionId
         CodeFixes_AddExplicitCast = 384,
 
         ToolsOptions_GenerateEditorconfig = 385,
+
+        ChangeSignature_Data = 386,
     }
 }