Skip to content

Commit

Permalink
Add option to consume burst metrics from external service, inject as …
Browse files Browse the repository at this point in the history
…header (retaildevcrews#61)

* Add option to use burst service

* address design review comments

* Update option defaults

* Make burstmetricsservice static

* Address review comments

* Add burst service vars as args/envvaroptions
  • Loading branch information
sivamu authored Aug 30, 2021
1 parent 980bec6 commit 6bd2d68
Show file tree
Hide file tree
Showing 7 changed files with 225 additions and 63 deletions.
3 changes: 1 addition & 2 deletions src/Controllers/HealthzController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
using Microsoft.Extensions.Logging;
using Ngsa.Application.DataAccessLayer;
using Ngsa.Application.Model;
using Ngsa.Middleware;

namespace Ngsa.Application.Controllers
{
Expand Down Expand Up @@ -60,7 +59,7 @@ public async Task<IActionResult> RunHealthzAsync()
StatusCode = res.Status == HealthStatus.Unhealthy ? (int)System.Net.HttpStatusCode.ServiceUnavailable : (int)System.Net.HttpStatusCode.OK,
};

CpuCounter.AddBurstHeader(Response.HttpContext);
BurstMetricsService.InjectBurstMetricsHeader(Response.HttpContext);

return result;
}
Expand Down
177 changes: 177 additions & 0 deletions src/Core/BurstMetricsService.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using System;
using System.Net.Http;
using System.Threading;
using System.Timers;
using Microsoft.AspNetCore.Http;
using Ngsa.Middleware;

/// <summary>
/// This service consumes burst metrics from an external service and will also
/// inject these metrics into a response header.
/// </summary>
namespace Ngsa.Application
{
public class BurstMetricsService : IDisposable
{
private const int ClientTimeout = 5;
private const int MetricsRefreshFrequency = 5;
private const string CapacityHeader = "X-Load-Feedback";

private static readonly NgsaLog Logger = new NgsaLog { Name = typeof(BurstMetricsService).FullName };
private static HttpClient client;
private static System.Timers.Timer timer;
private static string burstMetricsPath;
private static string burstMetricsResult;
private static CancellationToken Token { get; set; }

/// <summary>
/// Initializes burst metrics service
/// </summary>
public static void Init(CancellationToken token)
{
burstMetricsResult = string.Empty;
Token = token;
burstMetricsPath = $"{App.Config.BurstServiceNs}/{App.Config.BurstServiceHPA}";

// ensure version is set for client's user-agent
VersionExtension.Init();

// setup http client
client = OpenHTTPClient(App.Config.BurstServiceEndpoint);
}

/// <summary>
/// Start consuming data from burst metrics service
/// </summary>
public static void Start()
{
// run timed burst metrics service
timer = new ()
{
Enabled = true,
Interval = MetricsRefreshFrequency * 1000,
};
timer.Elapsed += TimerWork;

// Run once before the timer
TimerWork(null, null);

// Start the timer, it will be called after Interval
timer.Start();
}

/// <summary>
/// Stop collecting burst metrics
/// </summary>
public static void Stop()
{
if (timer != null)
{
timer.Stop();
timer.Dispose();
timer = null;
}

if (client != null)
{
client.Dispose();
client = null;
}
}

/// <summary>
/// Return burst metrics
/// </summary>
public static string GetBurstMetrics()
{
return burstMetricsResult;
}

/// <summary>
/// Inject burst metrics, if present, into the response header
/// </summary>
/// <param name="context">response context</param>
public static void InjectBurstMetricsHeader(HttpContext context)
{
if (App.Config.BurstHeader && !string.IsNullOrEmpty(burstMetricsResult))
{
context.Response.Headers.Add(CapacityHeader, burstMetricsResult);
}
}

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
{
Stop();
}
}

private static HttpClient OpenHTTPClient(string baseAddress)
{
try
{
// Create a http client
HttpClient client = new ()
{
Timeout = new TimeSpan(0, 0, ClientTimeout),
BaseAddress = new Uri(baseAddress),
};
client.DefaultRequestHeaders.Add("User-Agent", $"ngsa/{VersionExtension.ShortVersion}");
return client;
}
catch (Exception ex)
{
throw new Exception("Unable to setup client to burst service endpoint.", ex);
}
}

private static async void TimerWork(object state, ElapsedEventArgs e)
{
// exit if cancelled
if (Token.IsCancellationRequested)
{
Stop();
return;
}

// verify http client
if (client == null)
{
Logger.LogError("BurstMetricsServiceTimer", "Burst Metrics Service HTTP Client is null. Attempting to recreate.");

// recreate http client
client = OpenHTTPClient(App.Config.BurstServiceEndpoint);
return;
}

try
{
// process the response
using HttpResponseMessage resp = await client.GetAsync(burstMetricsPath).ConfigureAwait(false);
if (resp.IsSuccessStatusCode)
{
burstMetricsResult = await resp.Content.ReadAsStringAsync().ConfigureAwait(false);
}
else
{
burstMetricsResult = string.Empty;
Logger.LogWarning("BurstMetricsServiceTimer", "Received error status code from burst service.", new LogEventId((int)resp.StatusCode, resp.StatusCode.ToString()));
}
}
catch (Exception ex)
{
Logger.LogError("BurstMetricsServiceTimer", "Failed to get response from burst service.", ex: ex);
}
}
}
}
57 changes: 30 additions & 27 deletions src/Core/CommandLine.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ public static async Task<int> RunApp(Config config)
// log startup messages
LogStartup(logger);

// start burst metrics service
if (config.BurstHeader)
{
BurstMetricsService.Init(ctCancel.Token);
BurstMetricsService.Start();
}

// start the webserver
Task w = host.RunAsync();

Expand Down Expand Up @@ -98,10 +105,10 @@ public static RootCommand BuildRootCommand()
root.AddOption(EnvVarOption(new string[] { "--url-prefix" }, "URL prefix for ingress mapping", string.Empty));
root.AddOption(EnvVarOption(new string[] { "--port" }, "Listen Port", 8080, 1, (64 * 1024) - 1));
root.AddOption(EnvVarOption(new string[] { "--cache-duration", "-d" }, "Cache for duration (seconds)", 300, 1));
root.AddOption(EnvVarOption(new string[] { "--burst-header" }, "Enable burst metrics header in healthz", false));
root.AddOption(EnvVarOption(new string[] { "--burst-service" }, "Service name for bursting metrics", string.Empty));
root.AddOption(EnvVarOption(new string[] { "--burst-target" }, "Target level for bursting metrics (int)", 60, 1, 100));
root.AddOption(EnvVarOption(new string[] { "--burst-max" }, "Max level for bursting metrics (int)", 80, 1, 100));
root.AddOption(EnvVarOption(new string[] { "--burst-header" }, "Enable burst metrics header in health and version endpoints. If true, the other burst-service* args/env must be set.", false));
root.AddOption(EnvVarOption(new string[] { "--burst-service-endpoint" }, "Burst metrics service endpoint", string.Empty));
root.AddOption(EnvVarOption(new string[] { "--burst-service-ns" }, "Namespace parameter for burst metrics service", string.Empty));
root.AddOption(EnvVarOption(new string[] { "--burst-service-hpa" }, "HPA name parameter for burst metrics service", string.Empty));
root.AddOption(EnvVarOption(new string[] { "--retries" }, "Cosmos 429 retries", 10, 0));
root.AddOption(EnvVarOption(new string[] { "--timeout" }, "Request timeout", 10, 1));
root.AddOption(EnvVarOption(new string[] { "--data-service", "-s" }, "Data Service URL", string.Empty));
Expand Down Expand Up @@ -135,9 +142,12 @@ private static string ValidateDependencies(CommandResult result)
string secrets = result.Children.FirstOrDefault(c => c.Symbol.Name == "secrets-volume") is OptionResult secretsRes ? secretsRes.GetValueOrDefault<string>() : string.Empty;
string dataService = result.Children.FirstOrDefault(c => c.Symbol.Name == "data-service") is OptionResult dsRes ? dsRes.GetValueOrDefault<string>() : string.Empty;
string urlPrefix = result.Children.FirstOrDefault(c => c.Symbol.Name == "urlPrefix") is OptionResult urlRes ? urlRes.GetValueOrDefault<string>() : string.Empty;
string burstService = result.Children.FirstOrDefault(c => c.Symbol.Name == "burst-service") is OptionResult bsRes ? bsRes.GetValueOrDefault<string>() : string.Empty;
string bsEndpoint = result.Children.FirstOrDefault(c => c.Symbol.Name == "burst-service-endpoint") is OptionResult bsEndpointRes ? bsEndpointRes.GetValueOrDefault<string>() : string.Empty;
string bsNamespace = result.Children.FirstOrDefault(c => c.Symbol.Name == "burst-service-ns") is OptionResult bsNamespaceRes ? bsNamespaceRes.GetValueOrDefault<string>() : string.Empty;
string bsHpa = result.Children.FirstOrDefault(c => c.Symbol.Name == "burst-service-hpa") is OptionResult bsHpaRes ? bsHpaRes.GetValueOrDefault<string>() : string.Empty;
bool inMemory = result.Children.FirstOrDefault(c => c.Symbol.Name == "in-memory") is OptionResult inMemoryRes && inMemoryRes.GetValueOrDefault<bool>();
bool noCache = result.Children.FirstOrDefault(c => c.Symbol.Name == "no-cache") is OptionResult noCacheRes && noCacheRes.GetValueOrDefault<bool>();
bool burstHeader = result.Children.FirstOrDefault(c => c.Symbol.Name == "burst-header") is OptionResult burstHeaderRes && burstHeaderRes.GetValueOrDefault<bool>();

// validate url-prefix
if (!string.IsNullOrWhiteSpace(urlPrefix))
Expand Down Expand Up @@ -185,21 +195,6 @@ private static string ValidateDependencies(CommandResult result)
}
}

// validate burst-service
if (!string.IsNullOrWhiteSpace(burstService))
{
burstService = burstService.Trim();

if (burstService.Length > 64 ||
burstService.Contains('\n') ||
burstService.Contains('\t') ||
burstService.Contains(' ') ||
burstService.Contains('\r'))
{
msg += "--burst-service is invalid";
}
}

// validate secrets volume
if (!inMemory && appType == AppType.App)
{
Expand All @@ -224,6 +219,21 @@ private static string ValidateDependencies(CommandResult result)
}
}

// validate burst headers
if (burstHeader)
{
if (string.IsNullOrWhiteSpace(bsEndpoint) ||
string.IsNullOrWhiteSpace(bsNamespace) ||
string.IsNullOrWhiteSpace(bsHpa))
{
msg += "burst metrics service variable(s) cannot be empty\n";
}
else if (!Uri.IsWellFormedUriString($"{bsEndpoint}/{bsNamespace}/{bsHpa}", UriKind.Absolute))
{
msg += "burst metrics service endpoint is not a valid URI\n";
}
}

// invalid combination
if (inMemory && noCache)
{
Expand Down Expand Up @@ -404,13 +414,6 @@ private static void SetConfig(Config config)
}
}

// set burst headers service name
if (string.IsNullOrWhiteSpace(Config.BurstService))
{
VersionExtension.Init();
Config.BurstService = VersionExtension.Name;
}

SetLoggerConfig();
}

Expand Down
12 changes: 6 additions & 6 deletions src/Core/Config.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ public class Config
public InMemoryDal CacheDal { get; set; }
public IDAL CosmosDal { get; set; }
public bool BurstHeader { get; set; }
public string BurstService { get; set; }
public int BurstTarget { get; set; }
public int BurstMax { get; set; }
public string BurstServiceEndpoint { get; set; } = string.Empty;
public string BurstServiceNs { get; set; } = string.Empty;
public string BurstServiceHPA { get; set; } = string.Empty;
public string UrlPrefix { get; set; }

public void SetConfig(Config config)
Expand All @@ -57,9 +57,9 @@ public void SetConfig(Config config)
CacheDal = config.CacheDal;
CosmosDal = config.CosmosDal;
BurstHeader = config.BurstHeader;
BurstService = config.BurstService;
BurstMax = config.BurstMax;
BurstTarget = config.BurstTarget;
BurstServiceEndpoint = config.BurstServiceEndpoint;
BurstServiceNs = config.BurstServiceNs;
BurstServiceHPA = config.BurstServiceHPA;
UrlPrefix = string.IsNullOrWhiteSpace(config.UrlPrefix) ? string.Empty : config.UrlPrefix;

// remove trailing / if present
Expand Down
13 changes: 0 additions & 13 deletions src/Core/CpuCounter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ namespace Ngsa.Application
/// </summary>
public class CpuCounter : IDisposable
{
public const string CapacityHeader = "X-Load-Feedback";

private static readonly Process Proc = Process.GetCurrentProcess();
private static long lastTicks = Environment.TickCount64;
private static long lastCpu = Proc.TotalProcessorTime.Ticks;
Expand Down Expand Up @@ -73,17 +71,6 @@ public void Dispose()
GC.SuppressFinalize(this);
}

/// <summary>
/// Insert bursting headers
/// </summary>
public static void AddBurstHeader(HttpContext context)
{
if (App.Config.BurstHeader)
{
context.Response.Headers.Add(CapacityHeader, $"service={App.Config.BurstService}, current-load={CpuPercent}, target-load={App.Config.BurstTarget}, max-load={App.Config.BurstMax}");
}
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
Expand Down
Loading

0 comments on commit 6bd2d68

Please sign in to comment.