Skip to content

Commit

Permalink
Merge pull request kubernetes#59360 from immutableT/envelop_encryptio…
Browse files Browse the repository at this point in the history 
…n_benchmark

Automatic merge from submit-queue (batch tested with PRs 59190, 59360). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adding benchmarks to envelop encryption integration tests

**What this PR does / why we need it**:
Adding benchmarks for envelop encryption integration tests.
Allows to estimate how envelop encryption may impact the performance of KubeAPI server.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```
  • Loading branch information
Kubernetes Submit Queue authored Feb 8, 2018
2 parents 3aa7094 + cfdea23 commit c15ae2f
Show file tree
Hide file tree
Showing 2 changed files with 166 additions and 79 deletions.
18 changes: 12 additions & 6 deletions cmd/kube-apiserver/app/testing/testserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"io/ioutil"
"net"
"os"
"testing"
"time"

pflag "github.com/spf13/pflag"
Expand All @@ -46,13 +45,20 @@ type TestServer struct {
TmpDir string // Temp Dir used, by the apiserver
}

// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
type Logger interface {
Errorf(format string, args ...interface{})
Fatalf(format string, args ...interface{})
Logf(format string, args ...interface{})
}

// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func,
// and location of the tmpdir are returned.
//
// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy
// files that becaues Golang testing's call to os.Exit will not give a stop channel go routine
// enough time to remove temporariy files.
func StartTestServer(t *testing.T, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
// files that because Golang testing's call to os.Exit will not give a stop channel go routine
// enough time to remove temporary files.
func StartTestServer(t Logger, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {

// TODO : Remove TrackStorageCleanup below when PR
// https://github.com/kubernetes/kubernetes/pull/50690
Expand Down Expand Up @@ -137,7 +143,7 @@ func StartTestServer(t *testing.T, customFlags []string, storageConfig *storageb
}

// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
func StartTestServerOrDie(t *testing.T, flags []string, storageConfig *storagebackend.Config) *TestServer {
func StartTestServerOrDie(t Logger, flags []string, storageConfig *storagebackend.Config) *TestServer {

result, err := StartTestServer(t, flags, storageConfig)
if err == nil {
Expand Down
227 changes: 154 additions & 73 deletions test/integration/master/secrets_enveloping_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"strings"
"testing"

Expand All @@ -47,6 +49,8 @@ const (
testNamespace = "secret-encryption-test"
testSecret = "test-secret"

encryptionConfigFileName = "encryption.conf"

aesGCMPrefix = "k8s:enc:aesgcm:v1:key1:"
aesCBCPrefix = "k8s:enc:aescbc:v1:key1:"

Expand Down Expand Up @@ -79,117 +83,199 @@ resources:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
`

identityConfigYAML = `
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- identity: {}
`
)

type unSealSecret func(cipherText []byte, ctx value.Context, config encryptionconfig.ProviderConfig) ([]byte, error)

// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and ECTD that checks:
// 1. Secrets are encrypted on write
// 2. Secrets are decrypted on read
// when EncryptionConfig is passed to KubeAPI server.
func TestSecretsShouldBeEnveloped(t *testing.T) {
var testCases = []struct {
transformerConfigContent string
transformerPrefix string
unSealFunc unSealSecret
}{
{aesGCMConfigYAML, aesGCMPrefix, unSealWithGCMTransformer},
{aesCBCConfigYAML, aesCBCPrefix, unSealWithCBCTransformer},
// TODO: add secretbox
}
for _, tt := range testCases {
runEnvelopeTest(t, tt.unSealFunc, tt.transformerConfigContent, tt.transformerPrefix)
}
type envelopTest struct {
logger kubeapiservertesting.Logger
storageConfig *storagebackend.Config
configDir string
transformerConfig string
kubeAPIServer kubeapiservertesting.TestServer
restClient *kubernetes.Clientset
ns *corev1.Namespace
secret *corev1.Secret
}

func runEnvelopeTest(t *testing.T, unSealSecretFunc unSealSecret, transformerConfigYAML, expectedEnvelopePrefix string) {
transformerConfig := parseTransformerConfigOrDie(t, transformerConfigYAML)
func newEnvelopeTest(l kubeapiservertesting.Logger, transformerConfigYAML string) (*envelopTest, error) {
e := envelopTest{
logger: l,
transformerConfig: transformerConfigYAML,
storageConfig: framework.SharedEtcd(),
}

storageConfig := framework.SharedEtcd()
kubeAPIServer, err := startKubeApiWithEncryption(t, storageConfig, transformerConfigYAML)
if err != nil {
t.Error(err)
return
var err error
if transformerConfigYAML != "" {
if e.configDir, err = createKubeAPIServerEncryptionConfig(transformerConfigYAML); err != nil {
return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err)
}
}
defer kubeAPIServer.TearDownFn()

client, err := kubernetes.NewForConfig(kubeAPIServer.ClientConfig)
if err != nil {
t.Fatalf("error while creating client: %v", err)
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, e.getKubeAPIServerEncryptionOptions(), e.storageConfig); err != nil {
return nil, fmt.Errorf("failed to start KubeAPI server: %v", err)
}

ns, err := createTestNamespace(client, testNamespace)
if err != nil {
t.Error(err)
return
if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil {
return nil, fmt.Errorf("error while creating rest client: %v", err)
}
defer func() {
client.CoreV1().Namespaces().Delete(ns.Name, metav1.NewDeleteOptions(0))
}()

_, err = createTestSecret(client, testSecret, ns.Name)
if err != nil {
t.Error(err)
return
if e.ns, err = createTestNamespace(e.restClient, testNamespace); err != nil {
return nil, err
}

etcdPath := getETCDPath(storageConfig.Prefix)
response, err := readRawRecordFromETCD(kubeAPIServer, etcdPath)
if e.secret, err = createTestSecret(e.restClient, testSecret, e.ns.Name); err != nil {
return nil, err
}

return &e, nil
}

func (e *envelopTest) cleanUp() {
os.RemoveAll(e.configDir)
e.restClient.CoreV1().Namespaces().Delete(e.ns.Name, metav1.NewDeleteOptions(0))
e.kubeAPIServer.TearDownFn()
}

func (e *envelopTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefix string) {
response, err := readRawRecordFromETCD(&e.kubeAPIServer, e.getETCDPath())
if err != nil {
t.Error(err)
e.logger.Errorf("failed to read from etcd: %v", err)
return
}

if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) {
t.Errorf("expected secret to be enveloped by %s, but got %s",
e.logger.Errorf("expected secret to be enveloped by %s, but got %s",
expectedEnvelopePrefix, response.Kvs[0].Value)
return
}

// etcd path of the key is used as authenticated context - need to pass it to decrypt
ctx := value.DefaultContext([]byte(etcdPath))
// etcd path of the key is used as the authenticated context - need to pass it to decrypt
ctx := value.DefaultContext([]byte(e.getETCDPath()))
// Envelope header precedes the payload
sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
v, err := unSealSecretFunc(sealedData, ctx, transformerConfig)
transformerConfig, err := parseTransformerConfig(e.transformerConfig)
if err != nil {
e.logger.Errorf("failed to parse transformer config: %v", err)
}
v, err := unSealSecretFunc(sealedData, ctx, *transformerConfig)
if err != nil {
t.Error(err)
e.logger.Errorf("failed to unseal secret: %v", err)
return
}
if !strings.Contains(string(v), secretVal) {
t.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
e.logger.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
}

// Secrets should be un-enveloped on direct reads from Kube API Server.
s, err := client.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{})
s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{})
if secretVal != string(s.Data[secretKey]) {
t.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
e.logger.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
}
}

func startKubeApiWithEncryption(t *testing.T, storageConfig *storagebackend.Config,
transformerConfig string) (*kubeapiservertesting.TestServer, error) {
tempDir, err := ioutil.TempDir("", "secrets-encryption-test")
if err != nil {
return nil, fmt.Errorf("failed to create temp directory: %v", err)
func (e *envelopTest) benchmark(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := createTestSecret(e.restClient, e.secret.Name+strconv.Itoa(i), e.ns.Name)
if err != nil {
b.Fatalf("failed to create a secret: %v", err)
}
}
defer os.RemoveAll(tempDir)
}

encryptionConfig, err := ioutil.TempFile(tempDir, "encryption-config")
if err != nil {
return nil, fmt.Errorf("error while creating temp file for encryption config %v", err)
func (e *envelopTest) getETCDPath() string {
return fmt.Sprintf("/%s/secrets/%s/%s", e.storageConfig.Prefix, e.ns.Name, e.secret.Name)
}

func (e *envelopTest) getKubeAPIServerEncryptionOptions() []string {
if e.transformerConfig != "" {
return []string{"--experimental-encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName)}
}

return nil
}

// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and etcd that checks:
// 1. Secrets are encrypted on write
// 2. Secrets are decrypted on read
// when EncryptionConfig is passed to KubeAPI server.
func TestSecretsShouldBeEnveloped(t *testing.T) {
var testCases = []struct {
transformerConfigContent string
transformerPrefix string
unSealFunc unSealSecret
}{
{aesGCMConfigYAML, aesGCMPrefix, unSealWithGCMTransformer},
{aesCBCConfigYAML, aesCBCPrefix, unSealWithCBCTransformer},
// TODO: add secretbox
}
for _, tt := range testCases {
test, err := newEnvelopeTest(t, tt.transformerConfigContent)
if err != nil {
test.cleanUp()
t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
continue
}
test.run(tt.unSealFunc, tt.transformerPrefix)
test.cleanUp()
}
}

// Baseline (no enveloping) - use to contrast with enveloping benchmarks.
func BenchmarkBase(b *testing.B) {
runBenchmark(b, "")
}

if _, err := encryptionConfig.Write([]byte(transformerConfig)); err != nil {
return nil, fmt.Errorf("error while writing encryption config: %v", err)
// Identity transformer is a NOOP (crypto-wise) - use to contrast with AESGCM and AESCBC benchmark results.
func BenchmarkIdentityWrite(b *testing.B) {
runBenchmark(b, identityConfigYAML)
}

func BenchmarkAESGCMEnvelopeWrite(b *testing.B) {
runBenchmark(b, aesGCMConfigYAML)
}

func BenchmarkAESCBCEnvelopeWrite(b *testing.B) {
runBenchmark(b, aesCBCConfigYAML)
}

func runBenchmark(b *testing.B, transformerConfig string) {
b.StopTimer()
test, err := newEnvelopeTest(b, transformerConfig)
defer test.cleanUp()
if err != nil {
b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err)
}

kubeAPIOptions := []string{"--experimental-encryption-provider-config", encryptionConfig.Name()}
server, err := kubeapiservertesting.StartTestServer(t, kubeAPIOptions, storageConfig)
b.StartTimer()
test.benchmark(b)
b.StopTimer()
}

func createKubeAPIServerEncryptionConfig(transformerConfig string) (string, error) {
tempDir, err := ioutil.TempDir("", "secrets-encryption-test")
if err != nil {
return nil, fmt.Errorf("failed to start KubeAPI Server %v", err)
return "", fmt.Errorf("failed to create temp directory: %v", err)
}

return &server, nil
encryptionConfig := path.Join(tempDir, encryptionConfigFileName)

if err := ioutil.WriteFile(encryptionConfig, []byte(transformerConfig), 0644); err != nil {
os.RemoveAll(tempDir)
return "", fmt.Errorf("error while writing encryption config: %v", err)
}

return tempDir, nil
}

func createTestNamespace(client *kubernetes.Clientset, name string) (*corev1.Namespace, error) {
Expand Down Expand Up @@ -224,7 +310,6 @@ func createTestSecret(client *kubernetes.Clientset, name, namespace string) (*co
}

func readRawRecordFromETCD(kubeAPIServer *kubeapiservertesting.TestServer, path string) (*clientv3.GetResponse, error) {
// Reading secret directly from etcd - expect data to be enveloped and the payload encrypted.
etcdClient, err := integration.GetEtcdKVClient(kubeAPIServer.ServerOpts.Etcd.StorageConfig)
if err != nil {
return nil, fmt.Errorf("failed to create etcd client: %v", err)
Expand All @@ -237,18 +322,14 @@ func readRawRecordFromETCD(kubeAPIServer *kubeapiservertesting.TestServer, path
return response, nil
}

func getETCDPath(prefix string) string {
return fmt.Sprintf("/%s/secrets/%s/%s", prefix, testNamespace, testSecret)
}

func parseTransformerConfigOrDie(t *testing.T, configContent string) encryptionconfig.ProviderConfig {
func parseTransformerConfig(configContent string) (*encryptionconfig.ProviderConfig, error) {
var config encryptionconfig.EncryptionConfig
err := yaml.Unmarshal([]byte(configContent), &config)
if err != nil {
t.Errorf("failed to extract transformer key: %v", err)
return nil, fmt.Errorf("failed to extract transformer key: %v", err)
}

return config.Resources[0].Providers[0]
return &config.Resources[0].Providers[0], nil
}

func unSealWithGCMTransformer(cipherText []byte, ctx value.Context,
Expand Down

0 comments on commit c15ae2f

Please sign in to comment.