Fix TestNSCAndICMPWebhook* is unstable (networkservicemesh#1828)

* fix cert generation Signed-off-by: Denis Tingajkin <[email protected]> * update admission-webhook-secret.tpl Signed-off-by: Denis Tingajkin <[email protected]>
zhaohuabing · Nov 5, 2019 · 8dcb3ff · 8dcb3ff
1 parent f0264b3
commit 8dcb3ff
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 53 deletions.
diff --git a/deployments/helm/nsm/charts/admission-webhook/templates/admission-webhook-secret.tpl b/deployments/helm/nsm/charts/admission-webhook/templates/admission-webhook-secret.tpl
@@ -11,8 +11,8 @@ metadata:
   namespace: {{ .Release.Namespace }}
 type: Opaque
 data:
-  key.pem: {{ $cert.Key | b64enc }}
-  cert.pem: {{ $cert.Cert | b64enc }}
+  tls.key: {{ $cert.Key | b64enc }}
+  tls.crt: {{ $cert.Cert | b64enc }}
 ---
 apiVersion: apps/v1
 kind: Deployment

diff --git a/k8s/cmd/admission-webhook/const.go b/k8s/cmd/admission-webhook/const.go
@@ -1,5 +1,7 @@
 package main
 
+import v1 "k8s.io/api/core/v1"
+
 const (
 	emptyBody            = "empty body"
 	mutateMethod         = "/mutate"
@@ -19,8 +21,8 @@ const (
 	initContainerDefault = "nsm-init"
 	tagDefault           = "latest"
 	initContainerName    = "nsm-init-container"
-	certFile             = "/etc/webhook/certs/cert.pem"
-	keyFile              = "/etc/webhook/certs/key.pem"
+	certFile             = "/etc/webhook/certs/" + v1.TLSCertKey
+	keyFile              = "/etc/webhook/certs/" + v1.TLSPrivateKeyKey
 	initContainersPath   = "/spec/initContainers"
 	unsupportedKind      = "kind %v is not supported"
 	deploymentSubPath    = "/spec/template"

diff --git a/test/kubetest/utils.go b/test/kubetest/utils.go
@@ -2,17 +2,24 @@ package kubetest
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/rsa"
+	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/pem"
 	"fmt"
+	"math/big"
 	"net"
 	"os"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+
 	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/networkservicemesh/pkg/security"
@@ -29,8 +36,6 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/client-go/util/cert"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 
 	"github.com/networkservicemesh/networkservicemesh/controlplane/api/registry"
 	nsmd2 "github.com/networkservicemesh/networkservicemesh/controlplane/pkg/nsmd"
@@ -501,72 +506,74 @@ func DeleteAdmissionWebhook(k8s *K8s, secretName string,
 }
 
 // CreateAdmissionWebhookSecret - Create admission webhook secret
-func CreateAdmissionWebhookSecret(k8s *K8s, name, namespace string) (*v1.Secret, []byte) {
-	caCertSpec := &cert.Config{
-		CommonName: "admission-controller-ca",
-		Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
-	}
-	caCert, caKey, err := pkiutil.NewCertificateAuthority(caCertSpec)
-	k8s.g.Expect(err).To(BeNil())
-
-	certSpec := &cert.Config{
-		CommonName: name + "-svc",
-		AltNames: cert.AltNames{
-			DNSNames: []string{
-				name + "-svc." + namespace,
-				name + "-svc." + namespace + ".svc",
-			},
+func CreateAdmissionWebhookSecret(k8s *K8s, name, namespace string) (*v1.Secret, tls.Certificate) {
+	now := time.Now()
+	crt := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName: "admission-controller-ca",
+		},
+		NotBefore:    now.Add(-time.Hour * 24 * 365),
+		NotAfter:     now.Add(time.Hour * 24 * 365),
+		SerialNumber: big.NewInt(now.Unix()),
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageServerAuth,
+		},
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		DNSNames: []string{
+			name + "-svc." + namespace,
+			name + "-svc." + namespace + ".svc",
 		},
-		Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
 	}
-	cer, key, err := pkiutil.NewCertAndKey(caCert, caKey, certSpec)
-	k8s.g.Expect(err).To(BeNil())
+	// Generate a private key.
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	k8s.g.Expect(err).Should(BeNil())
 
-	block := &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(key.(*rsa.PrivateKey)),
-	}
-	keyPem := pem.EncodeToMemory(block)
+	// PEM-encode the private key.
+	keyBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  keyutil.RSAPrivateKeyBlockType,
+		Bytes: x509.MarshalPKCS1PrivateKey(key),
+	})
+	// Self-sign the certificate using the private key.
+	sig, err := x509.CreateCertificate(rand.Reader, &crt, &crt, key.Public(), key)
+	k8s.g.Expect(err).Should(BeNil())
 
-	block = &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: cer.Raw,
-	}
-	certPem := pem.EncodeToMemory(block)
+	// PEM-encode the signed certificate
+	sigBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  cert.CertificateBlockType,
+		Bytes: sig,
+	})
 
 	secret := &v1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind: "Secret",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      name + "-certs",
+			Name:      "nsm-admission-webhook-certs",
 			Namespace: namespace,
 		},
-		Type: v1.SecretTypeOpaque,
+		Type: v1.SecretTypeTLS,
 		Data: map[string][]byte{
-			"key.pem":  keyPem,
-			"cert.pem": certPem,
+			v1.TLSCertKey:       sigBytes,
+			v1.TLSPrivateKeyKey: keyBytes,
 		},
 	}
-
-	awSecret, err := k8s.CreateSecret(secret, namespace)
-	k8s.g.Expect(err).To(BeNil())
-
-	block = &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: caCert.Raw,
-	}
-	caCertPem := pem.EncodeToMemory(block)
-
-	return awSecret, caCertPem
+	result, err := tls.X509KeyPair(sigBytes, keyBytes)
+	k8s.g.Expect(err).Should(BeNil())
+	secret, err = k8s.CreateSecret(secret, namespace)
+	k8s.g.Expect(err).Should(BeNil())
+	return secret, result
 }
 
 // CreateMutatingWebhookConfiguration - Setup Mutating webhook configuration
-func CreateMutatingWebhookConfiguration(k8s *K8s, certPem []byte, name, namespace string) *arv1beta1.MutatingWebhookConfiguration {
+func CreateMutatingWebhookConfiguration(k8s *K8s, c tls.Certificate, name, namespace string) *arv1beta1.MutatingWebhookConfiguration {
 	servicePath := "/mutate"
-
+	caBundle := pem.EncodeToMemory(&pem.Block{
+		Type:  cert.CertificateBlockType,
+		Bytes: c.Certificate[0],
+	})
 	mutatingWebhookConf := &arv1beta1.MutatingWebhookConfiguration{
-
 		TypeMeta: metav1.TypeMeta{
 			Kind: "MutatingWebhookConfiguration",
 		},
@@ -585,7 +592,7 @@ func CreateMutatingWebhookConfiguration(k8s *K8s, certPem []byte, name, namespac
 						Name:      name + "-svc",
 						Path:      &servicePath,
 					},
-					CABundle: certPem,
+					CABundle: caBundle,
 				},
 				Rules: []arv1beta1.RuleWithOperations{
 					{