server: properly support status port over TLS (pingcap#14785)

server/http_status.go

@@ -17,6 +17,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net"
@@ -264,11 +265,23 @@ func (s *Server) startHTTPServer() {
 	})
 
 	logutil.BgLogger().Info("for status and metrics report", zap.String("listening on addr", addr))
-	s.setupStatuServerAndRPCServer(addr, serverMux)
+	s.setupStatusServerAndRPCServer(addr, serverMux)
 }
 
-func (s *Server) setupStatuServerAndRPCServer(addr string, serverMux *http.ServeMux) {
-	l, err := net.Listen("tcp", addr)
+func (s *Server) setupStatusServerAndRPCServer(addr string, serverMux *http.ServeMux) {
+	tlsConfig, err := s.cfg.Security.ToTLSConfig()
+	if err != nil {
+		logutil.BgLogger().Error("invalid TLS config", zap.Error(err))
+		return
+	}
+
+	var l net.Listener
+	if tlsConfig != nil {
+		// we need to manage TLS here for cmux to distinguish between HTTP and gRPC.
+		l, err = tls.Listen("tcp", addr, tlsConfig)
+	} else {
+		l, err = net.Listen("tcp", addr)
+	}
 	if err != nil {
 		logutil.BgLogger().Info("listen failed", zap.Error(err))
 		return
@@ -287,17 +300,11 @@ func (s *Server) setupStatuServerAndRPCServer(addr string, serverMux *http.Serve
 		logutil.BgLogger().Error("grpc server error", zap.Error(err))
 	}, nil)
 
-	if len(s.cfg.Security.ClusterSSLCA) != 0 {
-		go util.WithRecovery(func() {
-			err := s.statusServer.ServeTLS(httpL, s.cfg.Security.ClusterSSLCert, s.cfg.Security.ClusterSSLKey)
-			logutil.BgLogger().Error("http server error", zap.Error(err))
-		}, nil)
-	} else {
-		go util.WithRecovery(func() {
-			err := s.statusServer.Serve(httpL)
-			logutil.BgLogger().Error("http server error", zap.Error(err))
-		}, nil)
-	}
+	go util.WithRecovery(func() {
+		err := s.statusServer.Serve(httpL)
+		logutil.BgLogger().Error("http server error", zap.Error(err))
+	}, nil)
+
 	err = m.Serve()
 	if err != nil {
 		logutil.BgLogger().Error("start status/rpc server error", zap.Error(err))

server/rpc_server.go

@@ -35,7 +35,6 @@ import (
 	"github.com/pingcap/tidb/util/stringutil"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
 )
 
 // NewRPCServer creates a new rpc server.
@@ -46,16 +45,7 @@ func NewRPCServer(config *config.Config, dom *domain.Domain, sm util.SessionMana
 		}
 	}()
 
-	var s *grpc.Server
-	if len(config.Security.ClusterSSLCA) != 0 {
-		tlsConfig, err := config.Security.ToTLSConfig()
-		if err == nil {
-			s = grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)))
-		}
-	}
-	if s == nil {
-		s = grpc.NewServer()
-	}
+	s := grpc.NewServer()
 	rpcSrv := &rpcServer{
 		DiagnosticsServer: sysutil.NewDiagnosticsServer(config.Log.File.Filename),
 		dom:               dom,

server/server_test.go

@@ -65,21 +65,23 @@ type configOverrider func(*mysql.Config)
 // testServerClient config server connect parameters and provider several
 // method to communicate with server and run tests
 type testServerClient struct {
-	port       uint
-	statusPort uint
+	port         uint
+	statusPort   uint
+	statusScheme string
 }
 
 // newTestServerClient return a testServerClient with unique address
 func newTestServerClient() *testServerClient {
 	return &testServerClient{
-		port:       genPort(),
-		statusPort: genStatusPort(),
+		port:         genPort(),
+		statusPort:   genStatusPort(),
+		statusScheme: "http",
 	}
 }
 
 // statusURL return the full URL of a status path
 func (cli *testServerClient) statusURL(path string) string {
-	return fmt.Sprintf("http://localhost:%d%s", cli.statusPort, path)
+	return fmt.Sprintf("%s://localhost:%d%s", cli.statusScheme, cli.statusPort, path)
 }
 
 // fetchStatus exec http.Get to server status port

server/tidb_test.go

@@ -177,6 +177,43 @@ func (ts *tidbTestSuite) TestStatusAPI(c *C) {
 	ts.runTestStatusAPI(c)
 }
 
+func (ts *tidbTestSuite) TestStatusAPIWithTLS(c *C) {
+	caCert, caKey, err := generateCert(0, "TiDB CA 2", nil, nil, "/tmp/ca-key-2.pem", "/tmp/ca-cert-2.pem")
+	c.Assert(err, IsNil)
+	_, _, err = generateCert(1, "tidb-server-2", caCert, caKey, "/tmp/server-key-2.pem", "/tmp/server-cert-2.pem")
+	c.Assert(err, IsNil)
+
+	defer func() {
+		os.Remove("/tmp/ca-key-2.pem")
+		os.Remove("/tmp/ca-cert-2.pem")
+		os.Remove("/tmp/server-key-2.pem")
+		os.Remove("/tmp/server-cert-2.pem")
+	}()
+
+	cli := newTestServerClient()
+	cli.statusScheme = "https"
+	cfg := config.NewConfig()
+	cfg.Port = cli.port
+	cfg.Status.StatusPort = cli.statusPort
+	cfg.Security.ClusterSSLCA = "/tmp/ca-cert-2.pem"
+	cfg.Security.ClusterSSLCert = "/tmp/server-cert-2.pem"
+	cfg.Security.ClusterSSLKey = "/tmp/server-key-2.pem"
+	server, err := NewServer(cfg, ts.tidbdrv)
+	c.Assert(err, IsNil)
+	go server.Run()
+	time.Sleep(time.Millisecond * 100)
+
+	// https connection should work.
+	ts.runTestStatusAPI(c)
+
+	// but plain http connection should fail.
+	cli.statusScheme = "http"
+	_, err = cli.fetchStatus("/status")
+	c.Assert(err, NotNil)
+
+	server.Close()
+}
+
 func (ts *tidbTestSuite) TestMultiStatements(c *C) {
 	c.Parallel()
 	ts.runTestMultiStatements(c)