Skip to content

Commit

Permalink
Add support to retry function execution on invocation failures (Azure…
Browse files Browse the repository at this point in the history
  • Loading branch information
pragnagopa authored Sep 24, 2020
1 parent 3a68115 commit dd062a5
Show file tree
Hide file tree
Showing 22 changed files with 413 additions and 36 deletions.
27 changes: 27 additions & 0 deletions WebJobs.Script.sln
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,29 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger", "HttpTrigger"
sample\CustomHandler\HttpTrigger\function.json = sample\CustomHandler\HttpTrigger\function.json
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger-Retry", "HttpTrigger-Retry", "{821D5B92-2C3E-44F0-AA92-8B996DCB8E6C}"
ProjectSection(SolutionItems) = preProject
sample\Node\HttpTrigger-Retry\function.json = sample\Node\HttpTrigger-Retry\function.json
sample\Node\HttpTrigger-Retry\index.js = sample\Node\HttpTrigger-Retry\index.js
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NodeRetry", "NodeRetry", "{EEBAC197-FAD8-4214-9A12-76334BB3021A}"
ProjectSection(SolutionItems) = preProject
sample\NodeRetry\host.json = sample\NodeRetry\host.json
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger-FunctionJsonRetry", "HttpTrigger-FunctionJsonRetry", "{7935A7A4-191A-4A9B-B8DD-173968309EBF}"
ProjectSection(SolutionItems) = preProject
sample\NodeRetry\HttpTrigger-RetryFunctionJson\function.json = sample\NodeRetry\HttpTrigger-RetryFunctionJson\function.json
sample\NodeRetry\HttpTrigger-RetryFunctionJson\index.js = sample\NodeRetry\HttpTrigger-RetryFunctionJson\index.js
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger-HostJsonRetry", "HttpTrigger-HostJsonRetry", "{CF6D9CDA-2290-46DF-B162-2D422477288A}"
ProjectSection(SolutionItems) = preProject
sample\NodeRetry\HttpTrigger-RetryHostJson\function.json = sample\NodeRetry\HttpTrigger-RetryHostJson\function.json
sample\NodeRetry\HttpTrigger-RetryHostJson\index.js = sample\NodeRetry\HttpTrigger-RetryHostJson\index.js
EndProjectSection
EndProject
Global
GlobalSection(SharedMSBuildProjectFiles) = preSolution
test\WebJobs.Script.Tests.Shared\WebJobs.Script.Tests.Shared.projitems*{35c9ccb7-d8b6-4161-bb0d-bcfa7c6dcffb}*SharedItemsImports = 13
Expand Down Expand Up @@ -427,6 +450,10 @@ Global
{9A522D9D-2D86-4572-B7D1-ECBFBFAF312C} = {16351B76-87CA-4A8C-80A1-3DD83A0C4AA6}
{A59D3F65-53E5-4666-AEFE-882987A39A49} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
{209DC34B-762E-4B1B-B094-EBD7C4B972C2} = {A59D3F65-53E5-4666-AEFE-882987A39A49}
{821D5B92-2C3E-44F0-AA92-8B996DCB8E6C} = {9D87C796-7914-4A43-B843-579562393E10}
{EEBAC197-FAD8-4214-9A12-76334BB3021A} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
{7935A7A4-191A-4A9B-B8DD-173968309EBF} = {EEBAC197-FAD8-4214-9A12-76334BB3021A}
{CF6D9CDA-2290-46DF-B162-2D422477288A} = {EEBAC197-FAD8-4214-9A12-76334BB3021A}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {85400884-5FFD-4C27-A571-58CB3C8CAAC5}
Expand Down
6 changes: 5 additions & 1 deletion release_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- Update PowerShell Worker to 3.0.552 (PS6) [Release Note](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v3.0.552) and 3.0.549 (PS7) [Release Note](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v3.0.549)
- Update Java Worker to 1.8.0 [Release Note](https://github.com/Azure/azure-functions-java-worker/releases/tag/1.8.0)
- Update Java Library to 1.4.0 [Release Note](https://github.com/Azure/azure-functions-java-library)
- Added support for function execution retry on invocation failures [#6664](https://github.com/Azure/azure-functions-host/issues/6664)
- **[BreakingChange]** Fixes [#400](https://github.com/Azure/azure-functions-java-worker/issues/400) which was a regression from the 1.7.1 release.
There is potential of impact if the function code has taken a dependency on a feature in gson 2.8.6 as the dependency `gson-2.8.5.jar` is now included in the class path of the worker and will take precedence over the function's lib folder.
- **Breaking Changes in CustomHandler**
Expand All @@ -27,4 +28,7 @@
If you are using these properties, please ensure your app is able to detect and handle the new schema.

**Release sprint:** Sprint 84
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Afeature+is%3Aclosed) ]
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Afeature+is%3Aclosed) ]

**Release sprint:** Sprint 85
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+85%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+85%22+label%3Afeature+is%3Aclosed) ]
23 changes: 23 additions & 0 deletions sample/NodeRetry/HttpTrigger-RetryFunctionJson/function.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"bindings": [
{
"type": "httpTrigger",
"direction": "in",
"name": "req",
"methods": [
"get",
"post"
]
},
{
"type": "http",
"direction": "out",
"name": "res"
}
],
"retry": {
"strategy": "fixedDelay",
"maxRetryCount": 4,
"delayInterval": "00:00:03"
}
}
18 changes: 18 additions & 0 deletions sample/NodeRetry/HttpTrigger-RetryFunctionJson/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
var invocationCount = 0;

module.exports = async function (context, req) {
const reset = req.query.reset;
invocationCount = reset ? 0 : invocationCount

context.log('JavaScript HTTP trigger function processed a request.invocationCount: ' + invocationCount);

invocationCount = invocationCount + 1;
const responseMessage = "invocationCount: " + invocationCount;
if (invocationCount < 4) {
throw new Error('An error occurred');
}
context.res = {
// status: 200, /* Defaults to 200 */
body: responseMessage
};
}
18 changes: 18 additions & 0 deletions sample/NodeRetry/HttpTrigger-RetryHostJson/function.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"bindings": [
{
"type": "httpTrigger",
"direction": "in",
"name": "req",
"methods": [
"get",
"post"
]
},
{
"type": "http",
"direction": "out",
"name": "res"
}
]
}
18 changes: 18 additions & 0 deletions sample/NodeRetry/HttpTrigger-RetryHostJson/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
var invocationCount = 0;

module.exports = async function (context, req) {
const reset = req.query.reset;
invocationCount = reset ? 0 : invocationCount

context.log('JavaScript HTTP trigger function processed a request.invocationCount: ' + invocationCount);

invocationCount = invocationCount + 1;
const responseMessage = "invocationCount: " + invocationCount;
if (invocationCount < 2) {
throw new Error('An error occurred');
}
context.res = {
// status: 200, /* Defaults to 200 */
body: responseMessage
};
}
8 changes: 8 additions & 0 deletions sample/NodeRetry/host.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"version": "2.0",
"retry": {
"strategy": "fixedDelay",
"maxRetryCount": 2,
"delayInterval": "00:00:03"
}
}
2 changes: 1 addition & 1 deletion src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@
<PackageReference Include="Microsoft.Azure.AppService.Proxy.Client" Version="2.0.11020001-fabe022e" />
<PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.0.3" />
<PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.7" />
<PackageReference Include="Microsoft.Azure.WebJobs.Host.Storage" Version="4.0.1" />
<PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.23-11785" />
<PackageReference Include="Microsoft.Azure.WebJobs.Host.Storage" Version="4.0.1" />
<PackageReference Include="Microsoft.Azure.KeyVault" Version="3.0.3" />
<PackageReference Include="Microsoft.Azure.WebJobs.Logging" Version="4.0.1" />
<PackageReference Include="Microsoft.Azure.WebSites.DataProtection" Version="2.1.91-alpha" />
Expand Down
1 change: 1 addition & 0 deletions src/WebJobs.Script/Config/ConfigurationSectionNames.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,6 @@ public static class ConfigurationSectionNames
public const string Hsts = Http + ":hsts";
public const string CustomHttpHeaders = Http + ":customHeaders";
public const string EasyAuth = "easyauth";
public const string Retry = "retry";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider
{
private static readonly string[] WellKnownHostJsonProperties = new[]
{
"version", "functionTimeout", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",
"version", "functionTimeout", "retry", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",
"eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies",
"customHandler", "httpWorker"
};
Expand Down
1 change: 1 addition & 0 deletions src/WebJobs.Script/Config/ScriptHostOptionsSetup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public void Configure(ScriptJobHostOptions options)
{
options.FileLoggingMode = fileLoggingMode.Value;
}
Utility.ValidateRetryOptions(options.Retry);
}

// FunctionTimeout
Expand Down
6 changes: 6 additions & 0 deletions src/WebJobs.Script/Config/ScriptJobHostOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using Microsoft.Azure.WebJobs.Script.Description;

namespace Microsoft.Azure.WebJobs.Script
{
Expand Down Expand Up @@ -106,5 +107,10 @@ public string RootScriptPath
/// locally or via CLI.
/// </summary>
public bool IsSelfHost { get; set; }

/// <summary>
/// Gets or sets retry options to use on function executions on function invocation failures.
/// </summary>
public RetryOptions Retry { get; set; }
}
}
61 changes: 61 additions & 0 deletions src/WebJobs.Script/CustomAttributeBuilderUtility.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.Reflection;
using System.Reflection.Emit;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.Azure.WebJobs.Script.Description;

namespace Microsoft.Azure.WebJobs.Script
{
internal static class CustomAttributeBuilderUtility
{
internal static CustomAttributeBuilder GetRetryCustomAttributeBuilder(RetryOptions functionRetry)
{
switch (functionRetry.Strategy)
{
case RetryStrategy.FixedDelay:
Type fixedDelayRetryType = typeof(FixedDelayRetryAttribute);
ConstructorInfo fixedDelayRetryCtorInfo = fixedDelayRetryType.GetConstructor(new[] { typeof(int), typeof(string) });
CustomAttributeBuilder fixedDelayRetryBuilder = new CustomAttributeBuilder(
fixedDelayRetryCtorInfo,
new object[] { functionRetry.MaxRetryCount, functionRetry.DelayInterval.ToString() });
return fixedDelayRetryBuilder;
case RetryStrategy.ExponentialBackoff:
Type exponentialBackoffRetryType = typeof(ExponentialBackoffRetryAttribute);
ConstructorInfo exponentialBackoffDelayRetryCtorInfo = exponentialBackoffRetryType.GetConstructor(new[] { typeof(int), typeof(string), typeof(string) });
CustomAttributeBuilder exponentialBackoffRetryBuilder = new CustomAttributeBuilder(
exponentialBackoffDelayRetryCtorInfo,
new object[] { functionRetry.MaxRetryCount, functionRetry.MinimumInterval.ToString(), functionRetry.MaximumInterval.ToString() });
return exponentialBackoffRetryBuilder;
}
return null;
}

internal static CustomAttributeBuilder GetTimeoutCustomAttributeBuilder(TimeSpan functionTimeout)
{
Type timeoutType = typeof(TimeoutAttribute);
ConstructorInfo ctorInfo = timeoutType.GetConstructor(new[] { typeof(string) });

PropertyInfo[] propertyInfos = new[]
{
timeoutType.GetProperty("ThrowOnTimeout"),
timeoutType.GetProperty("TimeoutWhileDebugging")
};

// Hard-code these for now. Eventually elevate to config
object[] propertyValues = new object[]
{
true,
true
};

return new CustomAttributeBuilder(
ctorInfo,
new object[] { functionTimeout.ToString() },
propertyInfos,
propertyValues);
}
}
}
11 changes: 10 additions & 1 deletion src/WebJobs.Script/Description/FunctionDescriptorProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Script.Binding;
using Microsoft.Azure.WebJobs.Script.Configuration;
using Microsoft.Azure.WebJobs.Script.Extensibility;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq;
Expand Down Expand Up @@ -226,6 +225,16 @@ protected static void ApplyMethodLevelAttributes(FunctionMetadata functionMetada
CustomAttributeBuilder attributeBuilder = new CustomAttributeBuilder(ctorInfo, new object[0]);
methodAttributes.Add(attributeBuilder);
}

// apply the retry settings from function.json
if (functionMetadata.Retry != null)
{
CustomAttributeBuilder retryCustomAttributeBuilder = CustomAttributeBuilderUtility.GetRetryCustomAttributeBuilder(functionMetadata.Retry);
if (retryCustomAttributeBuilder != null)
{
methodAttributes.Add(retryCustomAttributeBuilder);
}
}
}

protected abstract IFunctionInvoker CreateFunctionInvoker(string scriptFilePath, BindingMetadata triggerMetadata, FunctionMetadata functionMetadata, Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings);
Expand Down
6 changes: 6 additions & 0 deletions src/WebJobs.Script/Host/FunctionMetadataProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.IO;
using System.IO.Abstractions;
using System.Linq;
using Microsoft.Azure.WebJobs.Script.Configuration;
using Microsoft.Azure.WebJobs.Script.Description;
using Microsoft.Azure.WebJobs.Script.Diagnostics;
using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
Expand Down Expand Up @@ -157,6 +158,11 @@ private FunctionMetadata ParseFunctionMetadata(string functionName, JObject conf
functionMetadata.Language = ParseLanguage(functionMetadata.ScriptFile, workerConfigs);
}
functionMetadata.EntryPoint = (string)configMetadata["entryPoint"];

//Retry
functionMetadata.Retry = configMetadata.Property(ConfigurationSectionNames.Retry)?.Value?.ToObject<RetryOptions>();
Utility.ValidateRetryOptions(functionMetadata.Retry);

return functionMetadata;
}

Expand Down
33 changes: 11 additions & 22 deletions src/WebJobs.Script/Host/ScriptHost.cs
Original file line number Diff line number Diff line change
Expand Up @@ -356,30 +356,19 @@ internal static Collection<CustomAttributeBuilder> CreateTypeAttributes(ScriptJo
// apply the timeout settings to our type
if (scriptConfig.FunctionTimeout != null)
{
Type timeoutType = typeof(TimeoutAttribute);
ConstructorInfo ctorInfo = timeoutType.GetConstructor(new[] { typeof(string) });

PropertyInfo[] propertyInfos = new[]
{
timeoutType.GetProperty("ThrowOnTimeout"),
timeoutType.GetProperty("TimeoutWhileDebugging")
};

// Hard-code these for now. Eventually elevate to config
object[] propertyValues = new object[]
{
true,
true
};

CustomAttributeBuilder timeoutBuilder = new CustomAttributeBuilder(
ctorInfo,
new object[] { scriptConfig.FunctionTimeout.ToString() },
propertyInfos,
propertyValues);

var timeoutBuilder = CustomAttributeBuilderUtility.GetTimeoutCustomAttributeBuilder(scriptConfig.FunctionTimeout.Value);
customAttributes.Add(timeoutBuilder);
}
// apply retry settings for function execution
if (scriptConfig.Retry != null)
{
// apply the retry settings from host.json
var retryCustomAttributeBuilder = CustomAttributeBuilderUtility.GetRetryCustomAttributeBuilder(scriptConfig.Retry);
if (retryCustomAttributeBuilder != null)
{
customAttributes.Add(retryCustomAttributeBuilder);
}
}

return customAttributes;
}
Expand Down
Loading

0 comments on commit dd062a5

Please sign in to comment.