Skip to content

Commit

Permalink
Handle empty response for secrets from Kubernetes
Browse files Browse the repository at this point in the history
  • Loading branch information
@ahmelsayed
ahmelsayed committed Jan 26, 2021
1 parent c833bfe commit 2efc384
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 14 deletions.
15 changes: 10 additions & 5 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,26 @@
FROM mcr.microsoft.com/dotnet/core/sdk:3.0 AS installer-env
FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS installer-env

ENV PublishWithAspNetCoreTargetManifest false

COPY . /workingdir

RUN cd workingdir && \
dotnet build WebJobs.Script.sln && \
dotnet publish src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj --output /azure-functions-host

# Runtime image
FROM mcr.microsoft.com/azure-functions/python:2.0
FROM mcr.microsoft.com/azure-functions/python:3.0

RUN apt-get update && \
apt-get install -y gnupg && \
curl -sL https://deb.nodesource.com/setup_8.x | bash - && \
curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
apt-get update && \
apt-get install -y nodejs dotnet-sdk-3.0
apt-get install -y nodejs dotnet-sdk-3.1

# Install the dependencies for Visual Studio Remote Debugger
RUN apt-get update && apt-get install -y --no-install-recommends unzip procps

# Install Visual Studio Remote Debugger
RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg

COPY --from=installer-env ["/azure-functions-host", "/azure-functions-host"]

Expand Down
2 changes: 1 addition & 1 deletion src/WebJobs.Script.WebHost/Security/KeyManagement/DefaultSecretManagerProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ internal ISecretsRepository CreateSecretsRepository()
}
else if (secretStorageType != null && secretStorageType.Equals("kubernetes", StringComparison.OrdinalIgnoreCase))
{
return new KubernetesSecretsRepository(_environment, new SimpleKubernetesClient(_environment));
return new KubernetesSecretsRepository(_environment, new SimpleKubernetesClient(_environment, _loggerFactory.CreateLogger<SimpleKubernetesClient>()));
}
else if (secretStorageSas != null)
{
Expand Down
46 changes: 38 additions & 8 deletions src/WebJobs.Script.WebHost/Security/KeyManagement/SimpleKubernetesClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Script.IO;
using Microsoft.Extensions.Logging;
using Microsoft.Net.Http.Headers;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
Expand All @@ -27,18 +28,20 @@ public class SimpleKubernetesClient : IKubernetesClient, IDisposable
private const string KubernetesSecretsDir = "/run/secrets/functions-keys";
private readonly HttpClient _httpClient;
private readonly IEnvironment _environment;
private readonly ILogger _logger;
private Action _watchCallback;
private bool _disposed;
private AutoRecoveringFileSystemWatcher _fileWatcher;

public SimpleKubernetesClient(IEnvironment environment) : this(environment, CreateHttpClient())
public SimpleKubernetesClient(IEnvironment environment, ILogger logger) : this(environment, CreateHttpClient(), logger)
{ }

// for testing
internal SimpleKubernetesClient(IEnvironment environment, HttpClient client)
internal SimpleKubernetesClient(IEnvironment environment, HttpClient client, ILogger logger)
{
_httpClient = client;
_environment = environment;
_logger = logger;
Task.Run(() => RunWatcher());
}

Expand All @@ -58,7 +61,9 @@ public async Task<IDictionary<string, string>> GetSecrets()
}
else
{
throw new InvalidOperationException($"{nameof(KubernetesSecretsRepository)} requires setting {EnvironmentSettingNames.AzureWebJobsKubernetesSecretName} or mounting secrets to {KubernetesSecretsDir}");
var exception = new InvalidOperationException($"{nameof(KubernetesSecretsRepository)} requires setting {EnvironmentSettingNames.AzureWebJobsKubernetesSecretName} or mounting secrets to {KubernetesSecretsDir}");
_logger.LogError(exception, "Error geting secrets");
throw exception;
}
}

Expand All @@ -74,6 +79,12 @@ public async Task UpdateSecrets(IDictionary<string, string> data)
using (var request = await GetRequest(HttpMethod.Patch, url, new[] { new { op = "replace", path = "/data", value = data } }, "application/json-patch+json"))
{
var response = await _httpClient.SendAsync(request);
if (!response.IsSuccessStatusCode)
{
_logger.LogError("Error updating Kubernetes secrets. {StatusCode}: {Content}",
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
response.EnsureSuccessStatusCode();
}
}
Expand All @@ -84,6 +95,15 @@ public void OnSecretChange(Action callback)
}

private async Task RunWatcher()
{
while (!_disposed)
{
// watch API requests terminate after 4 minutes
await RunWatcherInternal();
}
}

private async Task RunWatcherInternal()
{
if (string.IsNullOrEmpty(KubernetesObjectName) && FileUtility.DirectoryExists(KubernetesSecretsDir))
{
Expand Down Expand Up @@ -124,13 +144,15 @@ private async Task<IDictionary<string, string>> GetFromApiServer(string objectNa
if (response.IsSuccessStatusCode)
{
var obj = await response.Content.ReadAsAsync<JObject>();
return obj["data"]
?.ToObject<IDictionary<string, string>>()
?.ToDictionary(
return obj.ContainsKey("data")
? obj["data"]
.ToObject<IDictionary<string, string>>()
.ToDictionary(
k => k.Key,
v => decode
? Encoding.UTF8.GetString(Convert.FromBase64String(v.Value))
: v.Value);
: v.Value)
: new Dictionary<string, string>();
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
Expand All @@ -139,7 +161,9 @@ private async Task<IDictionary<string, string>> GetFromApiServer(string objectNa
}
else
{
throw new HttpRequestException($"Error calling GET {url}, Status: {response.StatusCode}, Content: {await response.Content.ReadAsStringAsync()}");
var exception = new HttpRequestException($"Error calling GET {url}, Status: {response.StatusCode}, Content: {await response.Content.ReadAsStringAsync()}");
_logger.LogError(exception, "Error reading secrets from Kubernetes");
throw exception;
}
}
}
Expand All @@ -165,6 +189,12 @@ private async Task CreateIfDoesntExist(string url, bool isSecret)
using (var createRequest = await GetRequest(HttpMethod.Post, url, payload))
{
var createResponse = await _httpClient.SendAsync(createRequest);
if (!createResponse.IsSuccessStatusCode)
{
_logger.LogError("Error creating Kubernetes secrets. {StatusCode}: {Content}",
createResponse.StatusCode,
await createResponse.Content.ReadAsStringAsync());
}
createResponse.EnsureSuccessStatusCode();
}
}
Expand Down
90 changes: 90 additions & 0 deletions test/WebJobs.Script.Tests/Security/SimpleKubernetesClientTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.IO;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Script.WebHost;
using Microsoft.Extensions.Logging;
using Microsoft.WebJobs.Script.Tests;
using Moq;
using Moq.Protected;
using Xunit;

namespace Microsoft.Azure.WebJobs.Script.Tests
{
public class SimpleKubernetesClientTests : IDisposable
{
[Theory]
[InlineData(HttpStatusCode.OK, "{}", 0)]
[InlineData(HttpStatusCode.OK, "{'data': {}}", 0)]
[InlineData(HttpStatusCode.OK, "{'data': {'key': 'dmFsdWU='}}", 1)]
public async Task Get_From_ApiServer_No_Data(HttpStatusCode statusCode, string content, int length)
{
var environment = new TestEnvironment();
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsKubernetesSecretName, "test");
environment.SetEnvironmentVariable(EnvironmentSettingNames.KubernetesServiceHost, "127.0.0.1");
environment.SetEnvironmentVariable(EnvironmentSettingNames.KubernetesServiceHttpsPort, "443");

var fullFileSystem = new FileSystem();
var fileSystem = new Mock<IFileSystem>();
var fileBase = new Mock<FileBase>();
var directoryBase = new Mock<DirectoryBase>();

fileSystem.SetupGet(f => f.Path).Returns(fullFileSystem.Path);
fileSystem.SetupGet(f => f.File).Returns(fileBase.Object);
fileSystem.SetupGet(f => f.Directory).Returns(directoryBase.Object);
fileBase.Setup(f => f.Exists("/run/secrets/kubernetes.io/serviceaccount/namespace")).Returns(true);
fileBase.Setup(f => f.Exists("/run/secrets/kubernetes.io/serviceaccount/token")).Returns(true);
fileBase.Setup(f => f.Exists("/run/secrets/kubernetes.io/serviceaccount/ca.crt")).Returns(true);

fileBase
.Setup(f => f.Open("/run/secrets/kubernetes.io/serviceaccount/token", It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>()))
.Returns(() =>
{
var token = new MemoryStream(Encoding.UTF8.GetBytes("test_token"));
token.Position = 0;
return token;
});
fileBase
.Setup(f => f.Open("/run/secrets/kubernetes.io/serviceaccount/namespace", It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>()))
.Returns(() =>
{
var ns = new MemoryStream(Encoding.UTF8.GetBytes("namespace"));
ns.Position = 0;
return ns;
});

FileUtility.Instance = fileSystem.Object;

var loggerFactory = new LoggerFactory();
var loggerProvider = new TestLoggerProvider();
loggerFactory.AddProvider(loggerProvider);

var handlerMock = new Mock<HttpMessageHandler>(MockBehavior.Strict);
handlerMock.Protected().Setup<Task<HttpResponseMessage>>("SendAsync", ItExpr.IsAny<HttpRequestMessage>(), ItExpr.IsAny<CancellationToken>()).ReturnsAsync(new HttpResponseMessage
{
StatusCode = statusCode,

Content = new StringContent(content, Encoding.UTF8, "application/json")
});

var client = new SimpleKubernetesClient(environment, new HttpClient(handlerMock.Object), loggerFactory.CreateLogger<SimpleKubernetesClient>());
var secrets = await client.GetSecrets();

Assert.NotNull(secrets);
Assert.Equal(secrets.Count, length);
}

public void Dispose()
{
FileUtility.Instance = null;
}
}
}

0 comments on commit 2efc384

Please sign in to comment.