From 2dd359ba192aedf37a9789992b46f7d9f0468fec Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 24 Jul 2017 16:40:11 -0500 Subject: [PATCH 1/3] kubeadm: add pubkeypin package (public key pinning hash implementation). This change adds a `k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin` package which implements x509 public key pinning in the style of RFC7469. This is the public key hash format used by the new `kubeadm join --discovery-token-ca-cert-hash` flag. Hashes are namespaced with a short type, with "sha256" being the only currently-supported format. Type "sha256" is a hex-encoded SHA-256 hash over the Subject Public Key Info (SPKI) object in DER-encoded ASN.1. --- cmd/kubeadm/app/util/BUILD | 1 + cmd/kubeadm/app/util/pubkeypin/BUILD | 35 ++++ cmd/kubeadm/app/util/pubkeypin/pubkeypin.go | 108 ++++++++++++ .../app/util/pubkeypin/pubkeypin_test.go | 158 ++++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100644 cmd/kubeadm/app/util/pubkeypin/BUILD create mode 100644 cmd/kubeadm/app/util/pubkeypin/pubkeypin.go create mode 100644 cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 2b3d9eabe1d32..9cda2c92684df 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -55,6 +55,7 @@ filegroup( "//cmd/kubeadm/app/util/apiclient:all-srcs", "//cmd/kubeadm/app/util/config:all-srcs", "//cmd/kubeadm/app/util/kubeconfig:all-srcs", + "//cmd/kubeadm/app/util/pubkeypin:all-srcs", "//cmd/kubeadm/app/util/token:all-srcs", ], tags = ["automanaged"], diff --git a/cmd/kubeadm/app/util/pubkeypin/BUILD b/cmd/kubeadm/app/util/pubkeypin/BUILD new file mode 100644 index 0000000000000..195b85c928777 --- /dev/null +++ b/cmd/kubeadm/app/util/pubkeypin/BUILD @@ -0,0 +1,35 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["pubkeypin_test.go"], + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["pubkeypin.go"], + tags = ["automanaged"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go b/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go new file mode 100644 index 0000000000000..1766e95755525 --- /dev/null +++ b/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go @@ -0,0 +1,108 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package pubkeypin provides primitives for x509 public key pinning in the +// style of RFC7469. +package pubkeypin + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "fmt" + "strings" +) + +const ( + // formatSHA256 is the prefix for pins that are full-length SHA-256 hashes encoded in base 16 (hex) + formatSHA256 = "sha256" +) + +// Set is a set of pinned x509 public keys. +type Set struct { + sha256Hashes map[string]bool +} + +// NewSet returns a new, empty PubKeyPinSet +func NewSet() *Set { + return &Set{make(map[string]bool)} +} + +// Allow adds an allowed public key hash to the Set +func (s *Set) Allow(pubKeyHashes ...string) error { + for _, pubKeyHash := range pubKeyHashes { + parts := strings.Split(pubKeyHash, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid public key hash, expected \"format:value\"") + } + format, value := parts[0], parts[1] + + switch strings.ToLower(format) { + case "sha256": + return s.allowSHA256(value) + default: + return fmt.Errorf("unknown hash format %q", format) + } + } + return nil +} + +// Check if a certificate matches one of the public keys in the set +func (s *Set) Check(certificate *x509.Certificate) error { + if s.checkSHA256(certificate) { + return nil + } + return fmt.Errorf("public key %s not pinned", Hash(certificate)) +} + +// Empty returns true if the Set contains no pinned public keys. +func (s *Set) Empty() bool { + return len(s.sha256Hashes) == 0 +} + +// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI) +// object in an x509 certificate (in DER encoding). It returns the full hash as a +// hex encoded string (suitable for passing to Set.Allow). +func Hash(certificate *x509.Certificate) string { + spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) + return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:])) +} + +// allowSHA256 validates a "sha256" format hash and adds a canonical version of it into the Set +func (s *Set) allowSHA256(hash string) error { + // validate that the hash is the right length to be a full SHA-256 hash + hashLength := hex.DecodedLen(len(hash)) + if hashLength != sha256.Size { + return fmt.Errorf("expected a %d byte SHA-256 hash, found %d bytes", sha256.Size, hashLength) + } + + // validate that the hash is valid hex + _, err := hex.DecodeString(hash) + if err != nil { + return err + } + + // in the end, just store the original hex string in memory (in lowercase) + s.sha256Hashes[strings.ToLower(hash)] = true + return nil +} + +// checkSHA256 returns true if the certificate's "sha256" hash is pinned in the Set +func (s *Set) checkSHA256(certificate *x509.Certificate) bool { + actualHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) + actualHashHex := strings.ToLower(hex.EncodeToString(actualHash[:])) + return s.sha256Hashes[actualHashHex] +} diff --git a/cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go b/cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go new file mode 100644 index 0000000000000..4e578a4bdb9b4 --- /dev/null +++ b/cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pubkeypin + +import ( + "crypto/x509" + "encoding/pem" + "strings" + "testing" +) + +// testCertPEM is a simple self-signed test certificate issued with the openssl CLI: +// openssl req -new -newkey rsa:2048 -days 36500 -nodes -x509 -keyout /dev/null -out test.crt +const testCertPEM = ` +-----BEGIN CERTIFICATE----- +MIIDRDCCAiygAwIBAgIJAJgVaCXvC6HkMA0GCSqGSIb3DQEBBQUAMB8xHTAbBgNV +BAMTFGt1YmVhZG0ta2V5cGlucy10ZXN0MCAXDTE3MDcwNTE3NDMxMFoYDzIxMTcw +NjExMTc0MzEwWjAfMR0wGwYDVQQDExRrdWJlYWRtLWtleXBpbnMtdGVzdDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0ba8mHU9UtYlzM1Own2Fk/XGjR +J4uJQvSeGLtz1hID1IA0dLwruvgLCPadXEOw/f/IWIWcmT+ZmvIHZKa/woq2iHi5 ++HLhXs7aG4tjKGLYhag1hLjBI7icqV7ovkjdGAt9pWkxEzhIYClFMXDjKpMSynu+ +YX6nZ9tic1cOkHmx2yiZdMkuriRQnpTOa7bb03OC1VfGl7gHlOAIYaj4539WCOr8 ++ACTUMJUFEHcRZ2o8a/v6F9GMK+7SC8SJUI+GuroXqlMAdhEv4lX5Co52enYaClN ++D9FJLRpBv2YfiCQdJRaiTvCBSxEFz6BN+PtP5l2Hs703ZWEkOqCByM6HV8CAwEA +AaOBgDB+MB0GA1UdDgQWBBRQgUX8MhK2rWBWQiPHWcKzoWDH5DBPBgNVHSMESDBG +gBRQgUX8MhK2rWBWQiPHWcKzoWDH5KEjpCEwHzEdMBsGA1UEAxMUa3ViZWFkbS1r +ZXlwaW5zLXRlc3SCCQCYFWgl7wuh5DAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEB +BQUAA4IBAQCaAUif7Pfx3X0F08cxhx8/Hdx4jcJw6MCq6iq6rsXM32ge43t8OHKC +pJW08dk58a3O1YQSMMvD6GJDAiAfXzfwcwY6j258b1ZlI9Ag0VokvhMl/XfdCsdh +AWImnL1t4hvU5jLaImUUMlYxMcSfHBGAm7WJIZ2LdEfg6YWfZh+WGbg1W7uxLxk6 +y4h5rWdNnzBHWAGf7zJ0oEDV6W6RSwNXtC0JNnLaeIUm/6xdSddJlQPwUv8YH4jX +c1vuFqTnJBPcb7W//R/GI2Paicm1cmns9NLnPR35exHxFTy+D1yxmGokpoPMdife +aH+sfuxT8xeTPb3kjzF9eJTlnEquUDLM +-----END CERTIFICATE-----` + +// expectedHash can be verified using the openssl CLI: +// openssl x509 -pubkey -in test.crt openssl rsa -pubin -outform der 2>&/dev/null | openssl dgst -sha256 -hex +const expectedHash = `sha256:345959acb2c3b2feb87d281961c893f62a314207ef02599f1cc4a5fb255480b3` + +// testCert2PEM is a second test cert generated the same way as testCertPEM +const testCert2PEM = ` +-----BEGIN CERTIFICATE----- +MIID9jCCAt6gAwIBAgIJAN5MXZDic7qYMA0GCSqGSIb3DQEBBQUAMFkxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCXRlc3RDZXJ0MjAgFw0xNzA3MjQxNjA0 +MDFaGA8yMTE3MDYzMDE2MDQwMVowWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNv +bWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAG +A1UEAxMJdGVzdENlcnQyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +0brwpJYN2ytPWzRBtZSVc3dhkQlA59AzxzqeLLkano0Pxo9NIc3T/y58nnRI8uaS +I1P7BzUfJTiUEvmAtX8NggqKK4ld/gPrU+IRww1CUYS4KCkA/0d0ctPy0JwBCjD+ +b57G3rmNE8c+0jns6J96ZzNtqmv6N+ZlFBAXm1p4S+k0kGi5+hoQ6H7SYXjk2lG+ +r/8jPQEjy/NSdw1dcCA0Nc6o+hPr32927dS6J9KOhBeXNYUNdbuDDmroM9/gN2e/ +YMSA1olLeDPQ7Xvhk0PIyEDnHh83AffPCx5yM3htVRGddjIsPAVUJEL3z5leJtxe +fzyPghOhHJY0PXqznDQTcwIDAQABo4G+MIG7MB0GA1UdDgQWBBRP0IJqv/5rQ4Uf +SByl77dJeEapRDCBiwYDVR0jBIGDMIGAgBRP0IJqv/5rQ4UfSByl77dJeEapRKFd +pFswWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoT +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJdGVzdENlcnQyggkA +3kxdkOJzupgwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEA0RIMHc10 +wHHPMh9UflqBgDMF7gfbOL0juJfGloAOcohWWfMZBBJ0CQKMy3xRyoK3HmbW1eeb +iATjesw7t4VEAwf7mgKAd+eTfWYB952uq5qYJ2TI28mSofEq1Wz3RmrNkC1KCBs1 +u+YMFGwyl6necV9zKCeiju4jeovI1GA38TvH7MgYln6vMJ+FbgOXj7XCpek7dQiY +KGaeSSH218mGNQaWRQw2Sm3W6cFdANoCJUph4w18s7gjtFpfV63s80hXRps+vEyv +jEQMEQpG8Ss7HGJLGLBw/xAmG0e//XS/o2dDonbGbvzToFByz8OGxjMhk6yV6hdd ++iyvsLAw/MYMSA== +-----END CERTIFICATE----- +` + +// testCert is a small helper to get a test x509.Certificate from the PEM constants +func testCert(t *testing.T, pemString string) *x509.Certificate { + // Decode the example certificate from a PEM file into a PEM block + pemBlock, _ := pem.Decode([]byte(pemString)) + if pemBlock == nil { + t.Fatal("failed to parse test certificate PEM") + return nil + } + + // Parse the PEM block into an x509.Certificate + result, err := x509.ParseCertificate(pemBlock.Bytes) + if err != nil { + t.Fatalf("failed to parse test certificate: %v", err) + return nil + } + return result +} + +func TestSet(t *testing.T) { + s := NewSet() + if !s.Empty() { + t.Error("expected a new set to be empty") + return + } + err := s.Allow("xyz") + if err == nil || !s.Empty() { + t.Error("expected allowing junk to fail") + return + } + + err = s.Allow("0011223344") + if err == nil || !s.Empty() { + t.Error("expected allowing something too short to fail") + return + } + + err = s.Allow(expectedHash + expectedHash) + if err == nil || !s.Empty() { + t.Error("expected allowing something too long to fail") + return + } + + err = s.Check(testCert(t, testCertPEM)) + if err == nil { + t.Error("expected test cert to not be allowed (yet)") + return + } + + err = s.Allow(strings.ToUpper(expectedHash)) + if err != nil || s.Empty() { + t.Error("expected allowing uppercase expectedHash to succeed") + return + } + + err = s.Check(testCert(t, testCertPEM)) + if err != nil { + t.Errorf("expected test cert to be allowed, but got back: %v", err) + return + } + + err = s.Check(testCert(t, testCert2PEM)) + if err == nil { + t.Error("expected the second test cert to be disallowed") + return + } +} + +func TestHash(t *testing.T) { + actualHash := Hash(testCert(t, testCertPEM)) + if actualHash != expectedHash { + t.Errorf( + "failed to Hash() to the expected value\n\texpected: %q\n\t actual: %q", + expectedHash, + actualHash, + ) + } +} From 1be639d6b05f314513c982087ee26fcc4ed30d79 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 24 Jul 2017 16:34:36 -0500 Subject: [PATCH 2/3] kubeadm: implement TLS discovery root CA pinning. This change adds the `--discovery-token-ca-cert-hash` and `--discovery-token-unsafe-skip-ca-verification` flags for `kubeadm join` and corresponding fields on the kubeadm NodeConfiguration struct. These flags configure enhanced TLS validation for token-based discovery. The enhanced TLS validation works by pinning the public key hashes of the cluster CA. This is done by connecting to the `cluster-info` endpoint initially using an unvalidated/unsafe TLS connection. After the cluster info has been loaded, parsed, and validated with the existing symmetric signature/MAC scheme, the root CA is validated against the pinned public key set. A second request is made using validated/safe TLS using the newly-known CA and the result is validated to make sure the same `cluster-info` was returned from both requests. This validation prevents a class of attacks where a leaked bootstrap token (such as from a compromised worker node) allows an attacker to impersonate the API server. This change also update `kubeadm init` to print the correct `--discovery-token-ca-cert-hash` flag in the example `kubeadm join` command it prints at the end of initialization. --- cmd/kubeadm/app/apis/kubeadm/types.go | 15 +++ .../app/apis/kubeadm/v1alpha1/types.go | 15 +++ .../app/apis/kubeadm/validation/validation.go | 11 ++ cmd/kubeadm/app/cmd/BUILD | 2 + cmd/kubeadm/app/cmd/init.go | 11 +- cmd/kubeadm/app/cmd/join.go | 22 ++++ cmd/kubeadm/app/discovery/discovery.go | 2 +- cmd/kubeadm/app/discovery/token/BUILD | 1 + cmd/kubeadm/app/discovery/token/token.go | 120 +++++++++++++++--- cmd/kubeadm/app/discovery/token/token_test.go | 54 ++++++++ 10 files changed, 235 insertions(+), 18 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index b6baf01ee6437..93ad680334447 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -100,6 +100,21 @@ type NodeConfiguration struct { NodeName string TLSBootstrapToken string Token string + + // DiscoveryTokenCACertHashes specifies a set of public key pins to verify + // when token-based discovery is used. The root CA found during discovery + // must match one of these values. Specifying an empty set disables root CA + // pinning, which can be unsafe. Each hash is specified as ":", + // where the only currently supported type is "sha256". This is a hex-encoded + // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded + // ASN.1. These hashes can be calculated using, for example, OpenSSL: + // openssl x509 -pubkey -in ca.crt openssl rsa -pubin -outform der 2>&/dev/null | openssl dgst -sha256 -hex + DiscoveryTokenCACertHashes []string + + // DiscoveryTokenUnsafeSkipCAVerification allows token-based discovery + // without CA verification via DiscoveryTokenCACertHashes. This can weaken + // the security of kubeadm since other nodes can impersonate the master. + DiscoveryTokenUnsafeSkipCAVerification bool } func (cfg *MasterConfiguration) GetMasterEndpoint() string { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index 38c2e77b1230c..f7c1cdf7ac194 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -98,4 +98,19 @@ type NodeConfiguration struct { NodeName string `json:"nodeName"` TLSBootstrapToken string `json:"tlsBootstrapToken"` Token string `json:"token"` + + // DiscoveryTokenCACertHashes specifies a set of public key pins to verify + // when token-based discovery is used. The root CA found during discovery + // must match one of these values. Specifying an empty set disables root CA + // pinning, which can be unsafe. Each hash is specified as ":", + // where the only currently supported type is "sha256". This is a hex-encoded + // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded + // ASN.1. These hashes can be calculated using, for example, OpenSSL: + // openssl x509 -pubkey -in ca.crt openssl rsa -pubin -outform der 2>&/dev/null | openssl dgst -sha256 -hex + DiscoveryTokenCACertHashes []string `json:"discoveryTokenCACertHashes"` + + // DiscoveryTokenUnsafeSkipCAVerification allows token-based discovery + // without CA verification via DiscoveryTokenCACertHashes. This can weaken + // the security of kubeadm since other nodes can impersonate the master. + DiscoveryTokenUnsafeSkipCAVerification bool `json:"discoveryTokenUnsafeSkipCAVerification"` } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index 3c19c02d3ecbd..dbdd252dccb9e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -137,6 +137,17 @@ func ValidateArgSelection(cfg *kubeadm.NodeConfiguration, fldPath *field.Path) f if len(cfg.DiscoveryTokenAPIServers) < 1 && len(cfg.DiscoveryToken) != 0 { allErrs = append(allErrs, field.Required(fldPath, "DiscoveryTokenAPIServers not set")) } + + if len(cfg.DiscoveryFile) != 0 && len(cfg.DiscoveryTokenCACertHashes) != 0 { + allErrs = append(allErrs, field.Invalid(fldPath, "", "DiscoveryTokenCACertHashes cannot be used with DiscoveryFile")) + } + + // TODO: convert this warning to an error after v1.8 + if len(cfg.DiscoveryFile) == 0 && len(cfg.DiscoveryTokenCACertHashes) == 0 && !cfg.DiscoveryTokenUnsafeSkipCAVerification { + fmt.Println("[validation] WARNING: using token-based discovery without DiscoveryTokenCACertHashes can be unsafe (see https://kubernetes.io/docs/admin/kubeadm/#kubeadm-join).") + fmt.Println("[validation] WARNING: Pass --discovery-token-unsafe-skip-ca-verification to disable this warning. This warning will become an error in Kubernetes 1.9.") + } + // TODO remove once we support multiple api servers if len(cfg.DiscoveryTokenAPIServers) > 1 { fmt.Println("[validation] WARNING: kubeadm doesn't fully support multiple API Servers yet") diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 2955ae3fdd41d..a907ab6a3fe45 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -32,6 +32,7 @@ go_library( "//cmd/kubeadm/app/phases/apiconfig:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", + "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/markmaster:go_default_library", @@ -41,6 +42,7 @@ go_library( "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", + "//cmd/kubeadm/app/util/pubkeypin:go_default_library", "//cmd/kubeadm/app/util/token:go_default_library", "//pkg/api:go_default_library", "//pkg/bootstrap/api:go_default_library", diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 6eaa449638fc4..9933ffd064783 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -38,6 +38,7 @@ import ( apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig" clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" @@ -46,6 +47,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/util/version" ) @@ -67,7 +69,7 @@ var ( You can now join any number of machines by running the following on each node as root: - kubeadm join --token {{.Token}} {{.MasterIP}}:{{.MasterPort}} + kubeadm join --token {{.Token}} {{.MasterIP}}:{{.MasterPort}} --discovery-token-ca-cert-hash {{.CAPubKeyPin}} `))) ) @@ -310,10 +312,17 @@ func (i *Init) Run(out io.Writer) error { } } + // Load the CA certificate from so we can pin its public key + caCert, err := pkiutil.TryLoadCertFromDisk(i.cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + ctx := map[string]string{ "KubeConfigPath": filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName), "KubeConfigName": kubeadmconstants.AdminKubeConfigFileName, "Token": i.cfg.Token, + "CAPubKeyPin": pubkeypin.Hash(caCert), "MasterIP": i.cfg.API.AdvertiseAddress, "MasterPort": strconv.Itoa(int(i.cfg.API.BindPort)), } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index a4b6a8e787ef3..acc8a094070be 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -77,6 +77,21 @@ func NewCmdJoin(out io.Writer) *cobra.Command { the discovery information is loaded from a URL, HTTPS must be used and the host installed CA bundle is used to verify the connection. + If you use a shared token for discovery, you should also pass the + --discovery-token-ca-cert-hash flag to validate the public key of the + root certificate authority (CA) presented by the Kubernetes Master. The + value of this flag is specified as ":", + where the supported hash type is "sha256". The hash is calculated over + the bytes of the Subject Public Key Info (SPKI) object (as in RFC7469). + This value is available in the output of "kubeadm init" or can be + calcuated using standard tools. The --discovery-token-ca-cert-hash flag + may be repeated multiple times to allow more than one public key. + + If you cannot know the CA public key hash ahead of time, you can pass + the --discovery-token-unsafe-skip-ca-verification flag to disable this + verification. This weakens the kubeadm security model since other nodes + can potentially impersonate the Kubernetes Master. + The TLS bootstrap mechanism is also driven via a shared token. This is used to temporarily authenticate with the Kubernetes Master to submit a certificate signing request (CSR) for a locally created key pair. By @@ -117,6 +132,13 @@ func NewCmdJoin(out io.Writer) *cobra.Command { cmd.PersistentFlags().StringVar( &cfg.TLSBootstrapToken, "tls-bootstrap-token", "", "A token used for TLS bootstrapping") + cmd.PersistentFlags().StringSliceVar( + &cfg.DiscoveryTokenCACertHashes, "discovery-token-ca-cert-hash", []string{}, + "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").") + cmd.PersistentFlags().BoolVar( + &cfg.DiscoveryTokenUnsafeSkipCAVerification, "discovery-token-unsafe-skip-ca-verification", false, + "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.") + cmd.PersistentFlags().StringVar( &cfg.Token, "token", "", "Use this token for both discovery-token and tls-bootstrap-token") diff --git a/cmd/kubeadm/app/discovery/discovery.go b/cmd/kubeadm/app/discovery/discovery.go index b452285fa41d0..a1bff38644966 100644 --- a/cmd/kubeadm/app/discovery/discovery.go +++ b/cmd/kubeadm/app/discovery/discovery.go @@ -58,7 +58,7 @@ func GetValidatedClusterInfoObject(cfg *kubeadmapi.NodeConfiguration) (*clientcm } return file.RetrieveValidatedClusterInfo(cfg.DiscoveryFile) case len(cfg.DiscoveryToken) != 0: - return token.RetrieveValidatedClusterInfo(cfg.DiscoveryToken, cfg.DiscoveryTokenAPIServers) + return token.RetrieveValidatedClusterInfo(cfg.DiscoveryToken, cfg.DiscoveryTokenAPIServers, cfg.DiscoveryTokenCACertHashes) default: return nil, fmt.Errorf("couldn't find a valid discovery configuration.") } diff --git a/cmd/kubeadm/app/discovery/token/BUILD b/cmd/kubeadm/app/discovery/token/BUILD index 45c345c156afb..a5acdcf4bbc7e 100644 --- a/cmd/kubeadm/app/discovery/token/BUILD +++ b/cmd/kubeadm/app/discovery/token/BUILD @@ -15,6 +15,7 @@ go_library( deps = [ "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", + "//cmd/kubeadm/app/util/pubkeypin:go_default_library", "//cmd/kubeadm/app/util/token:go_default_library", "//pkg/bootstrap/api:go_default_library", "//pkg/controller/bootstrap:go_default_library", diff --git a/cmd/kubeadm/app/discovery/token/token.go b/cmd/kubeadm/app/discovery/token/token.go index f765a4153dc11..a1dc464604f15 100644 --- a/cmd/kubeadm/app/discovery/token/token.go +++ b/cmd/kubeadm/app/discovery/token/token.go @@ -17,6 +17,9 @@ limitations under the License. package token import ( + "bytes" + "crypto/x509" + "encoding/pem" "fmt" "sync" @@ -27,6 +30,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" "k8s.io/kubernetes/pkg/controller/bootstrap" @@ -35,32 +39,40 @@ import ( const BootstrapUser = "token-bootstrap-client" // RetrieveValidatedClusterInfo connects to the API Server and tries to fetch the cluster-info ConfigMap -// It then makes sure it can trust the API Server by looking at the JWS-signed tokens -func RetrieveValidatedClusterInfo(discoveryToken string, tokenAPIServers []string) (*clientcmdapi.Cluster, error) { - +// It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if rootCAPubKeys is not empty) +// validating the cluster CA against a set of pinned public keys +func RetrieveValidatedClusterInfo(discoveryToken string, tokenAPIServers, rootCAPubKeys []string) (*clientcmdapi.Cluster, error) { tokenId, tokenSecret, err := tokenutil.ParseToken(discoveryToken) if err != nil { return nil, err } + // Load the cfg.DiscoveryTokenCACertHashes into a pubkeypin.Set + pubKeyPins := pubkeypin.NewSet() + err = pubKeyPins.Allow(rootCAPubKeys...) + if err != nil { + return nil, err + } + // The function below runs for every endpoint, and all endpoints races with each other. // The endpoint that wins the race and completes the task first gets its kubeconfig returned below baseKubeConfig := runForEndpointsAndReturnFirst(tokenAPIServers, func(endpoint string) (*clientcmdapi.Config, error) { - bootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint) - clusterName := bootstrapConfig.Contexts[bootstrapConfig.CurrentContext].Cluster + insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint) + clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster - client, err := kubeconfigutil.KubeConfigToClientSet(bootstrapConfig) + insecureClient, err := kubeconfigutil.KubeConfigToClientSet(insecureBootstrapConfig) if err != nil { return nil, err } - fmt.Printf("[discovery] Created cluster-info discovery client, requesting info from %q\n", bootstrapConfig.Clusters[clusterName].Server) + fmt.Printf("[discovery] Created cluster-info discovery client, requesting info from %q\n", insecureBootstrapConfig.Clusters[clusterName].Server) - var clusterinfo *v1.ConfigMap + // Make an initial insecure connection to get the cluster-info ConfigMap + var insecureClusterInfo *v1.ConfigMap wait.PollImmediateInfinite(constants.DiscoveryRetryInterval, func() (bool, error) { var err error - clusterinfo, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + insecureClusterInfo, err = insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) if err != nil { fmt.Printf("[discovery] Failed to request cluster info, will try again: [%s]\n", err) return false, nil @@ -68,25 +80,82 @@ func RetrieveValidatedClusterInfo(discoveryToken string, tokenAPIServers []strin return true, nil }) - kubeConfigString, ok := clusterinfo.Data[bootstrapapi.KubeConfigKey] - if !ok || len(kubeConfigString) == 0 { + // Validate the MAC on the kubeconfig from the ConfigMap and load it + insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey] + if !ok || len(insecureKubeconfigString) == 0 { return nil, fmt.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect", bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo) } - detachedJWSToken, ok := clusterinfo.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenId] + detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenId] if !ok || len(detachedJWSToken) == 0 { return nil, fmt.Errorf("there is no JWS signed token in the %s ConfigMap. This token id %q is invalid for this cluster, can't connect", bootstrapapi.ConfigMapClusterInfo, tokenId) } - if !bootstrap.DetachedTokenIsValid(detachedJWSToken, kubeConfigString, tokenId, tokenSecret) { + if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, tokenId, tokenSecret) { return nil, fmt.Errorf("failed to verify JWS signature of received cluster info object, can't trust this API Server") } + insecureKubeconfigBytes := []byte(insecureKubeconfigString) + insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes) + if err != nil { + return nil, fmt.Errorf("couldn't parse the kubeconfig file in the %s configmap: %v", bootstrapapi.ConfigMapClusterInfo, err) + } + + // If no TLS root CA pinning was specified, we're done + if pubKeyPins.Empty() { + fmt.Printf("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q\n", endpoint) + return insecureConfig, nil + } + + // Load the cluster CA from the Config + if len(insecureConfig.Clusters) != 1 { + return nil, fmt.Errorf("expected the kubeconfig file in the %s configmap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters)) + } + var clusterCABytes []byte + for _, cluster := range insecureConfig.Clusters { + clusterCABytes = cluster.CertificateAuthorityData + } + clusterCA, err := parsePEMCert(clusterCABytes) + if err != nil { + return nil, fmt.Errorf("failed to parse cluster CA from the %s configmap: %v", bootstrapapi.ConfigMapClusterInfo, err) + + } + + // Validate the cluster CA public key against the pinned set + err = pubKeyPins.Check(clusterCA) + if err != nil { + return nil, fmt.Errorf("cluster CA found in %s configmap is invalid: %v", bootstrapapi.ConfigMapClusterInfo, err) + } + + // Now that we know the proported cluster CA, connect back a second time validating with that CA + secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes) + secureClient, err := kubeconfigutil.KubeConfigToClientSet(secureBootstrapConfig) + if err != nil { + return nil, err + } + + fmt.Printf("[discovery] Requesting info from %q again to validate TLS against the pinned public key\n", insecureBootstrapConfig.Clusters[clusterName].Server) + var secureClusterInfo *v1.ConfigMap + wait.PollImmediateInfinite(constants.DiscoveryRetryInterval, func() (bool, error) { + var err error + secureClusterInfo, err = secureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + fmt.Printf("[discovery] Failed to request cluster info, will try again: [%s]\n", err) + return false, nil + } + return true, nil + }) - finalConfig, err := clientcmd.Load([]byte(kubeConfigString)) + // Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time + secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey]) + if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) { + return nil, fmt.Errorf("the second kubeconfig from the %s configmap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo) + } + + secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes) if err != nil { return nil, fmt.Errorf("couldn't parse the kubeconfig file in the %s configmap: %v", bootstrapapi.ConfigMapClusterInfo, err) } - fmt.Printf("[discovery] Cluster info signature and contents are valid, will use API Server %q\n", bootstrapConfig.Clusters[clusterName].Server) - return finalConfig, nil + fmt.Printf("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q\n", endpoint) + return secureKubeconfig, nil }) return kubeconfigutil.GetClusterFromKubeConfig(baseKubeConfig), nil @@ -101,6 +170,13 @@ func buildInsecureBootstrapKubeConfig(endpoint string) *clientcmdapi.Config { return bootstrapConfig } +// buildSecureBootstrapKubeConfig makes a KubeConfig object that connects securely to the API Server for bootstrapping purposes (validating with the specified CA) +func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte) *clientcmdapi.Config { + masterEndpoint := fmt.Sprintf("https://%s", endpoint) + bootstrapConfig := kubeconfigutil.CreateBasic(masterEndpoint, "kubernetes", BootstrapUser, caCert) + return bootstrapConfig +} + // runForEndpointsAndReturnFirst loops the endpoints slice and let's the endpoints race for connecting to the master func runForEndpointsAndReturnFirst(endpoints []string, fetchKubeConfigFunc func(string) (*clientcmdapi.Config, error)) *clientcmdapi.Config { stopChan := make(chan struct{}) @@ -131,3 +207,15 @@ func runForEndpointsAndReturnFirst(endpoints []string, fetchKubeConfigFunc func( wg.Wait() return resultingKubeConfig } + +// parsePEMCert decodes a PEM-formatted certificate and returns it as an x509.Certificate +func parsePEMCert(certData []byte) (*x509.Certificate, error) { + pemBlock, trailingData := pem.Decode(certData) + if pemBlock == nil { + return nil, fmt.Errorf("invalid PEM data") + } + if len(trailingData) != 0 { + return nil, fmt.Errorf("trailing data after first PEM block") + } + return x509.ParseCertificate(pemBlock.Bytes) +} diff --git a/cmd/kubeadm/app/discovery/token/token_test.go b/cmd/kubeadm/app/discovery/token/token_test.go index 06012608bd1da..4c59ddfbd48fc 100644 --- a/cmd/kubeadm/app/discovery/token/token_test.go +++ b/cmd/kubeadm/app/discovery/token/token_test.go @@ -25,6 +25,30 @@ import ( kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" ) +// testCertPEM is a simple self-signed test certificate issued with the openssl CLI: +// openssl req -new -newkey rsa:2048 -days 36500 -nodes -x509 -keyout /dev/null -out test.crt +const testCertPEM = ` +-----BEGIN CERTIFICATE----- +MIIDRDCCAiygAwIBAgIJAJgVaCXvC6HkMA0GCSqGSIb3DQEBBQUAMB8xHTAbBgNV +BAMTFGt1YmVhZG0ta2V5cGlucy10ZXN0MCAXDTE3MDcwNTE3NDMxMFoYDzIxMTcw +NjExMTc0MzEwWjAfMR0wGwYDVQQDExRrdWJlYWRtLWtleXBpbnMtdGVzdDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0ba8mHU9UtYlzM1Own2Fk/XGjR +J4uJQvSeGLtz1hID1IA0dLwruvgLCPadXEOw/f/IWIWcmT+ZmvIHZKa/woq2iHi5 ++HLhXs7aG4tjKGLYhag1hLjBI7icqV7ovkjdGAt9pWkxEzhIYClFMXDjKpMSynu+ +YX6nZ9tic1cOkHmx2yiZdMkuriRQnpTOa7bb03OC1VfGl7gHlOAIYaj4539WCOr8 ++ACTUMJUFEHcRZ2o8a/v6F9GMK+7SC8SJUI+GuroXqlMAdhEv4lX5Co52enYaClN ++D9FJLRpBv2YfiCQdJRaiTvCBSxEFz6BN+PtP5l2Hs703ZWEkOqCByM6HV8CAwEA +AaOBgDB+MB0GA1UdDgQWBBRQgUX8MhK2rWBWQiPHWcKzoWDH5DBPBgNVHSMESDBG +gBRQgUX8MhK2rWBWQiPHWcKzoWDH5KEjpCEwHzEdMBsGA1UEAxMUa3ViZWFkbS1r +ZXlwaW5zLXRlc3SCCQCYFWgl7wuh5DAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEB +BQUAA4IBAQCaAUif7Pfx3X0F08cxhx8/Hdx4jcJw6MCq6iq6rsXM32ge43t8OHKC +pJW08dk58a3O1YQSMMvD6GJDAiAfXzfwcwY6j258b1ZlI9Ag0VokvhMl/XfdCsdh +AWImnL1t4hvU5jLaImUUMlYxMcSfHBGAm7WJIZ2LdEfg6YWfZh+WGbg1W7uxLxk6 +y4h5rWdNnzBHWAGf7zJ0oEDV6W6RSwNXtC0JNnLaeIUm/6xdSddJlQPwUv8YH4jX +c1vuFqTnJBPcb7W//R/GI2Paicm1cmns9NLnPR35exHxFTy+D1yxmGokpoPMdife +aH+sfuxT8xeTPb3kjzF9eJTlnEquUDLM +-----END CERTIFICATE-----` + func TestRunForEndpointsAndReturnFirst(t *testing.T) { tests := []struct { endpoints []string @@ -59,3 +83,33 @@ func TestRunForEndpointsAndReturnFirst(t *testing.T) { } } } + +func TestParsePEMCert(t *testing.T) { + for _, testCase := range []struct { + name string + input []byte + expectValid bool + }{ + {"invalid certificate data", []byte{0}, false}, + {"certificate with junk appended", []byte(testCertPEM + "\nABC"), false}, + {"multiple certificates", []byte(testCertPEM + "\n" + testCertPEM), false}, + {"valid", []byte(testCertPEM), true}, + } { + cert, err := parsePEMCert(testCase.input) + if testCase.expectValid { + if err != nil { + t.Errorf("failed TestParsePEMCert(%s): unexpected error %v", testCase.name, err) + } + if cert == nil { + t.Errorf("failed TestParsePEMCert(%s): returned nil", testCase.name) + } + } else { + if err == nil { + t.Errorf("failed TestParsePEMCert(%s): expected an error", testCase.name) + } + if cert != nil { + t.Errorf("failed TestParsePEMCert(%s): expected not to get a certificate back, but got one", testCase.name) + } + } + } +} From 358806e18b91bbcb846fcd7af0f3e2a99adf4769 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 24 Jul 2017 16:39:47 -0500 Subject: [PATCH 3/3] kubeadm: generated deepcopy for `k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm` and `k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1`. --- .../app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go | 5 +++++ cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go index fddd84c7689f2..3750f87b64c1b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go @@ -194,6 +194,11 @@ func (in *NodeConfiguration) DeepCopyInto(out *NodeConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.DiscoveryTokenCACertHashes != nil { + in, out := &in.DiscoveryTokenCACertHashes, &out.DiscoveryTokenCACertHashes + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 6e5eb7b00a138..7fae52156a9fe 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -199,6 +199,11 @@ func (in *NodeConfiguration) DeepCopyInto(out *NodeConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.DiscoveryTokenCACertHashes != nil { + in, out := &in.DiscoveryTokenCACertHashes, &out.DiscoveryTokenCACertHashes + *out = make([]string, len(*in)) + copy(*out, *in) + } return }