Release/1.2 (#245)

* Avoid recompiling the same rule on every call * Update ScriptedRuleWrapper.cs fix compilation error * Fix IdentyRef parsing * Support "only links" filter option when adding a mapping * Added tests to #242 * changelog * dependencies updated * release log Co-authored-by: Alexander Omelchuk <[email protected]> Co-authored-by: aok-foss <[email protected]>
tfsaggregator · Sep 4, 2021 · 56a546c · 56a546c
1 parent 332c159
commit 56a546c
Show file tree

Hide file tree

Showing 19 changed files with 224 additions and 87 deletions.
diff --git a/Next-Release-ChangeLog.md b/Next-Release-ChangeLog.md
@@ -1,31 +1,30 @@
-This release fixes a number of issues and introduce an additional CLI command.
+This release has a few fixes and a new feature.
 
 
 CLI Commands and Options
 ========================
-- Fixes spurious upgrade message noted in #225.
+New `--filterOnlyLinks` option to `map.rule` command; coincide with the _Links are added or removed_ filter of the Azure DevOps subscription (#244).
 
 
 Docker and Azure Function Hosting
 ========================
-No changes.
+Fix an off-by-one error when getting unique name from the string (#243).
+Fix invalid cast exception when the `IdentityRef` could not be parsed (#243).
 
 
 Rule Language
 ========================
-- Added support for `.bypassrules` directive (#83, #228).
-- Fixes #231 (Directive check revision false missing after `update.rule`).
+No changes.
 
 
 Rule Interpreter Engine
 ========================
-- Fixes #229 (Updating a work item field with impersonation enabled fails with the message: _Remove requires Value to be null_).
-- Fixes #234 (_Object reference not set_ when removing a work item link).
+Improved performance and memory profile through caching compiled rules (#242).
 
 
 Build, Test, Documentation
 ========================
-- SonarCloud now requires Java 11.
+Updated NuGet dependencies.
 
 
 File Hashes

diff --git a/src/aggregator-cli/Mappings/AggregatorMappings.cs b/src/aggregator-cli/Mappings/AggregatorMappings.cs
@@ -383,6 +383,7 @@ internal class EventFilters
         public string Type { get; set; }
         public string Tag { get; set; }
         public IEnumerable<string> Fields { get; set; }
+        public bool OnlyLinks { get; set; }
     }
 
     internal static class EventFiltersExtension
@@ -412,6 +413,11 @@ public static IEnumerable<KeyValuePair<string, string>> ToInputs(this EventFilte
                 // Filter events to include only work items with the specified field(s) changed
                 yield return new KeyValuePair<string, string>("changedFields", string.Join(',', filters.Fields));
             }
+            if (filters.OnlyLinks)
+            {
+                // Filter events to include only work items with added or removed links
+                yield return new KeyValuePair<string, string>("linksChanged", "true");
+            }
         }
 
 

diff --git a/src/aggregator-cli/Mappings/MapLocalRuleCommand.cs b/src/aggregator-cli/Mappings/MapLocalRuleCommand.cs
@@ -38,6 +38,9 @@ class MapLocalRuleCommand : CommandBase
         [ShowInTelemetry]
         [Option("filterFields", Required = false, HelpText = "Filter Azure DevOps event to include only work items with the specified Field(s) changed.")]
         public IEnumerable<string> FilterFields { get; set; }
+        [ShowInTelemetry]
+        [Option("filterOnlyLinks", Required = false, HelpText = "Filter Azure DevOps event to include only work items with links added or removed.")]
+        public bool FilterOnlyLinks { get; set; }
 
         internal override async Task<int> RunAsync(CancellationToken cancellationToken)
         {
@@ -57,7 +60,8 @@ internal override async Task<int> RunAsync(CancellationToken cancellationToken)
                 AreaPath = FilterAreaPath,
                 Type = FilterType,
                 Tag = FilterTag,
-                Fields = FilterFields
+                Fields = FilterFields,
+                OnlyLinks = FilterOnlyLinks,
             };
 
             var targetUrl = new Uri(TargetUrl);

diff --git a/src/aggregator-cli/Mappings/MapRuleCommand.cs b/src/aggregator-cli/Mappings/MapRuleCommand.cs
@@ -42,6 +42,9 @@ class MapRuleCommand : CommandBase
         [ShowInTelemetry]
         [Option("filterFields", Required = false, HelpText = "Filter Azure DevOps event to include only work items with the specified Field(s) changed.")]
         public IEnumerable<string> FilterFields { get; set; }
+        [ShowInTelemetry]
+        [Option("filterOnlyLinks", Required = false, HelpText = "Filter Azure DevOps event to include only work items with links added or removed.")]
+        public bool FilterOnlyLinks { get; set; }
 
         internal override async Task<int> RunAsync(CancellationToken cancellationToken)
         {
@@ -63,7 +66,8 @@ internal override async Task<int> RunAsync(CancellationToken cancellationToken)
                 AreaPath = FilterAreaPath,
                 Type = FilterType,
                 Tag = FilterTag,
-                Fields = FilterFields
+                Fields = FilterFields,
+                OnlyLinks = FilterOnlyLinks,
             };
 
             var instance = context.Naming.Instance(Instance, ResourceGroup);

diff --git a/src/aggregator-cli/aggregator-cli.csproj b/src/aggregator-cli/aggregator-cli.csproj
@@ -36,19 +36,19 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.ApplicationInsights" Version="2.16.0" />
-    <PackageReference Include="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel" Version="2.16.0" />
-    <PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Azure.Management.Fluent" Version="1.36.0" />
-    <PackageReference Include="Microsoft.Azure.Management.ResourceManager.Fluent" Version="1.36.0" />
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.0" />
-    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.InteractiveClient" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.153.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights" Version="2.18.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel" Version="2.18.0" />
+    <PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="5.0.9" />
+    <PackageReference Include="Microsoft.Azure.Management.Fluent" Version="1.37.1" />
+    <PackageReference Include="Microsoft.Azure.Management.ResourceManager.Fluent" Version="1.37.1" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.2" />
+    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.InteractiveClient" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.170.0" />
     <PackageReference Include="CommandLineParser" Version="2.8.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="Octokit" Version="0.48.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Octokit" Version="0.50.0" />
     <PackageReference Include="semver" Version="2.0.6" />
   </ItemGroup>
 

diff --git a/src/aggregator-function/aggregator-function.csproj b/src/aggregator-function/aggregator-function.csproj
@@ -32,21 +32,21 @@
   </ItemGroup>
   <ItemGroup>
     <!-- resolves MSB3277 -->
-    <PackageReference Include="System.Text.Json" Version="5.0.0" />
+    <PackageReference Include="System.Text.Json" Version="5.0.2" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.WebApiCompatShim" Version="2.2.0" />
-    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.23" />
+    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.27" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="3.1.6" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
     <!-- explicit reference -->
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.6" />
-    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.3" />
-    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ExtensionManagement.WebApi" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.153.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
+    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.13" />
+    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ExtensionManagement.WebApi" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.170.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
   <ItemGroup>
     <None Update="aggregator-manifest.ini">

diff --git a/src/aggregator-host/aggregator-host.csproj b/src/aggregator-host/aggregator-host.csproj
@@ -21,7 +21,7 @@
 
   <ItemGroup>
     <!-- resolves MSB3277 -->
-    <PackageReference Include="System.Text.Json" Version="5.0.0" />
+    <PackageReference Include="System.Text.Json" Version="5.0.2" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
   </ItemGroup>
 

diff --git a/src/aggregator-ruleng/ScriptedRuleWrapper.cs b/src/aggregator-ruleng/ScriptedRuleWrapper.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
@@ -17,6 +18,8 @@ namespace aggregator.Engine
     /// </summary>
     public class ScriptedRuleWrapper : IRule
     {
+        private static readonly ConcurrentDictionary<string, Script<string>> _scriptCache = new ConcurrentDictionary<string, Script<string>>();
+
         private Script<string> _roslynScript;
         private readonly IAggregatorLogger _logger;
         private readonly bool? _ruleFileParseSuccess;
@@ -34,7 +37,7 @@ public class ScriptedRuleWrapper : IRule
 
         private ScriptedRuleWrapper(string ruleName, IAggregatorLogger logger)
         {
-            _logger = logger;
+            _logger = logger ?? new NullLogger();
             Name = ruleName;
         }
 
@@ -43,7 +46,7 @@ private ScriptedRuleWrapper(string ruleName, IAggregatorLogger logger)
         /// </summary>
         /// <param name="ruleName"></param>
         /// <param name="ruleCode"></param>
-        internal ScriptedRuleWrapper(string ruleName, string[] ruleCode) : this(ruleName, new NullLogger())
+        internal ScriptedRuleWrapper(string ruleName, string[] ruleCode, IAggregatorLogger logger = null) : this(ruleName, logger)
         {
             (IPreprocessedRule preprocessedRule, bool parseSuccess) = RuleFileParser.Read(ruleCode);
             _ruleFileParseSuccess = parseSuccess;
@@ -80,21 +83,14 @@ private void Initialize(IPreprocessedRule preprocessedRule)
 
             Settings = preprocessedRule.Settings;
 
-            var references = new HashSet<Assembly>(DefaultAssemblyReferences().Concat(RuleDirectives.LoadAssemblyReferences()));
-            var imports = new HashSet<string>(DefaultImports().Concat(RuleDirectives.Imports));
-
-            var scriptOptions = ScriptOptions.Default
-                .WithEmitDebugInformation(true)
-                .WithReferences(references)
-                // Add namespaces
-                .WithImports(imports);
-
             if (RuleDirectives.IsCSharp())
             {
-                _roslynScript = CSharpScript.Create<string>(
-                    code: RuleDirectives.GetRuleCode(),
-                    options: scriptOptions,
-                    globalsType: typeof(RuleExecutionContext));
+                string ruleKey = string.Join('\n', RuleFileParser.Write(RuleDirectives));
+#if DEBUG
+                bool cached = _scriptCache.ContainsKey(ruleKey);                
+                _logger.WriteVerbose(cached ? $"Rule {Name} found in cache": $"Rule {Name} was not in cache: compiling");
+#endif
+                _roslynScript = _scriptCache.GetOrAdd(ruleKey, CreateRoslynScript, RuleDirectives);
             }
             else
             {
@@ -130,6 +126,26 @@ private static IEnumerable<string> DefaultImports()
             return imports;
         }
 
+        private static Script<string> CreateRoslynScript(string key, IPreprocessedRule rule)
+        {
+            var references = new HashSet<Assembly>(DefaultAssemblyReferences().Concat(rule.LoadAssemblyReferences()));
+            var imports = new HashSet<string>(DefaultImports().Concat(rule.Imports));
+
+            var scriptOptions = ScriptOptions.Default
+                .WithEmitDebugInformation(true)
+                .WithReferences(references)
+                // Add namespaces
+                .WithImports(imports);
+
+            var script = CSharpScript.Create<string>(
+                code: rule.GetRuleCode(),
+                options: scriptOptions,
+                globalsType: typeof(RuleExecutionContext));
+            script.Compile();
+
+            return script;
+        }
+
         /// <inheritdoc />
         public async Task<IRuleResult> ApplyAsync(RuleExecutionContext executionContext, CancellationToken cancellationToken)
         {

diff --git a/src/aggregator-ruleng/aggregator-ruleng.csproj b/src/aggregator-ruleng/aggregator-ruleng.csproj
@@ -28,13 +28,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.CodeAnalysis.Scripting" Version="3.8.0" />
+    <PackageReference Include="Microsoft.CodeAnalysis.Scripting" Version="3.11.0" />
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="3.1.6" />
-    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ExtensionManagement.WebApi" Version="16.153.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="5.0.1" />
+    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ExtensionManagement.WebApi" Version="16.170.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/aggregator-shared/aggregator-shared.csproj b/src/aggregator-shared/aggregator-shared.csproj
@@ -28,11 +28,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.ApplicationInsights" Version="2.16.0" />
-    <PackageReference Include="Microsoft.Azure.Management.Fluent" Version="1.36.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights" Version="2.18.0" />
+    <PackageReference Include="Microsoft.Azure.Management.Fluent" Version="1.37.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.153.0" />
+    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.170.0" />
   </ItemGroup>
 
 </Project>
diff --git a/src/aggregator-webshared/RequestHelper.cs b/src/aggregator-webshared/RequestHelper.cs
@@ -108,12 +108,10 @@ static IdentityRef ConvertOrDefault(string input)
                     return null;
                 }
 
-                var uniqueNameLength = uniqueNameEndIndex - uniqueNameStartIndex + 1;
-
                 return new IdentityRef()
                 {
-                    DisplayName = input.Substring(0, uniqueNameStartIndex).Trim(),
-                    UniqueName = new string(input.Skip(uniqueNameStartIndex + 1).Take(uniqueNameLength).ToArray())
+                    DisplayName = input[0..uniqueNameStartIndex].Trim(),
+                    UniqueName = input[(uniqueNameStartIndex + 1)..uniqueNameEndIndex]
                 };
             }
 
@@ -125,13 +123,15 @@ static IdentityRef ConvertOrDefault(string input)
 
             foreach (var identityField in workItem.Fields.Where(field => identityFieldReferenceNameEndings.Any(name => field.Key.EndsWith(name))).ToList())
             {
-                if (identityField.Value is string identityString)
+                IdentityRef identityRef = identityField.Value switch
                 {
-                    workItem.Fields[identityField.Key] = ConvertOrDefault(identityString) ?? identityField.Value;
-                }
-                else if (identityField.Value is JsonElement identityElement)
+                    string identityString => ConvertOrDefault(identityString),
+                    JsonElement identityElement => ConvertOrDefault(identityElement.GetString()),
+                    _ => null
+                };
+                if (identityRef != null)
                 {
-                    workItem.Fields[identityField.Key] = ConvertOrDefault(identityElement.GetString()) ?? identityField.Value;
+                    workItem.Fields[identityField.Key] = identityRef;
                 }
             }
         }

diff --git a/src/aggregator-webshared/aggregator-webshared.csproj b/src/aggregator-webshared/aggregator-webshared.csproj
@@ -22,14 +22,14 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="3.1.6" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
     <!-- explicit reference -->
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.6" />    
-    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ExtensionManagement.WebApi" Version="16.153.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.153.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />    
+    <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ExtensionManagement.WebApi" Version="16.170.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Services.ServiceHooks.WebApi" Version="16.170.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/integrationtests-cli/integrationtests-cli.csproj b/src/integrationtests-cli/integrationtests-cli.csproj
@@ -8,11 +8,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="1.3.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>

diff --git a/src/unittests-core/unittests-core.csproj b/src/unittests-core/unittests-core.csproj
@@ -9,11 +9,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="1.3.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
     <DotNetCliToolReference Include="dotnet-xunit" Version="2.3.1" />