Skip to content

Commit

Permalink
executor: add diagnosis rule to check Transparent Huge Pages(THP) ena…
Browse files Browse the repository at this point in the history 
…bled (pingcap#20611)

Signed-off-by: crazycs520 <[email protected]>
  • Loading branch information
@crazycs520
crazycs520 authored Oct 29, 2020
1 parent b1b1da2 commit 3d536a6
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 17 deletions.
35 changes: 24 additions & 11 deletions executor/inspection_result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ func (configInspection) inspectDiffConfig(ctx context.Context, sctx sessionctx.C
// TiKV
"server.addr",
"server.advertise-addr",
"server.advertise-status-addr",
"server.status-addr",
"log-file",
"raftstore.raftdb-path",
Expand Down Expand Up @@ -298,44 +299,56 @@ func (configInspection) inspectDiffConfig(ctx context.Context, sctx sessionctx.C
func (c configInspection) inspectCheckConfig(ctx context.Context, sctx sessionctx.Context, filter inspectionFilter) []inspectionResult {
// check the configuration in reason.
cases := []struct {
table string
tp string
key string
value string
expect string
cond string
detail string
}{
{
tp: "tidb",
table: "cluster_config",
key: "log.slow-threshold",
value: "0",
expect: "> 0",
cond: "type = 'tidb' and `key` = 'log.slow-threshold' and value = '0'",
detail: "slow-threshold = 0 will record every query to slow log, it may affect performance",
},
{
tp: "tikv",

table: "cluster_config",
key: "raftstore.sync-log",
value: "false",
expect: "true",
cond: "type = 'tikv' and `key` = 'raftstore.sync-log' and value = 'false'",
detail: "sync-log should be true to avoid recover region when the machine breaks down",
},
{
table: "cluster_systeminfo",
key: "transparent_hugepage_enabled",
expect: "always madvise [never]",
cond: "system_name = 'kernel' and name = 'transparent_hugepage_enabled' and value not like '%[never]%'",
detail: "Transparent HugePages can cause memory allocation delays during runtime, TiDB recommends that you disable Transparent HugePages on all TiDB servers",
},
}

var results []inspectionResult
for _, cas := range cases {
if !filter.enable(cas.key) {
continue
}
sql := fmt.Sprintf("select instance from information_schema.cluster_config where type = '%s' and `key` = '%s' and value = '%s'",
cas.tp, cas.key, cas.value)
sql := fmt.Sprintf("select type,instance,value from information_schema.%s where %s",
cas.table, cas.cond)
rows, _, err := sctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQLWithContext(ctx, sql)
if err != nil {
sctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("check configuration in reason failed: %v", err))
}

for _, row := range rows {
results = append(results, inspectionResult{
tp: cas.tp,
instance: row.GetString(0),
tp: row.GetString(0),
instance: row.GetString(1),
item: cas.key,
actual: cas.value,
expected: "not " + cas.value,
actual: row.GetString(2),
expected: cas.expect,
severity: "warning",
detail: cas.detail,
})
Expand Down
26 changes: 22 additions & 4 deletions executor/inspection_result_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,14 @@ func (s *inspectionResultSuite) TestInspectionResult(c *C) {
types.MakeDatums("pd", "192.168.1.33:1234", "cpu", "cpu", "cpu-logical-cores", "10"),
},
}
// mock cluster system information
mockData[infoschema.TableClusterSystemInfo] = variable.TableSnapshot{
Rows: [][]types.Datum{
types.MakeDatums("pd", "pd-0", "system", "kernel", "transparent_hugepage_enabled", "always madvise [never]"),
types.MakeDatums("tikv", "tikv-2", "system", "kernel", "transparent_hugepage_enabled", "[always] madvise never"),
types.MakeDatums("tidb", "tidb-0", "system", "kernel", "transparent_hugepage_enabled", "always madvise [never]"),
},
}

datetime := func(str string) types.Time {
return s.parseTime(c, tk.Se, str)
Expand All @@ -120,9 +128,10 @@ func (s *inspectionResultSuite) TestInspectionResult(c *C) {
rows: []string{
"config coprocessor.high tikv inconsistent consistent warning 192.168.3.32:26600,192.168.3.33:26600 config value is 8\n192.168.3.34:26600,192.168.3.35:26600 config value is 7",
"config ddl.lease tidb inconsistent consistent warning 192.168.3.22:4000,192.168.3.24:4000,192.168.3.25:4000 config value is 1\n192.168.3.23:4000 config value is 2",
"config log.slow-threshold tidb 0 not 0 warning slow-threshold = 0 will record every query to slow log, it may affect performance",
"config log.slow-threshold tidb 0 > 0 warning slow-threshold = 0 will record every query to slow log, it may affect performance",
"config log.slow-threshold tidb inconsistent consistent warning 192.168.3.24:4000 config value is 0\n192.168.3.25:4000 config value is 1",
"config raftstore.sync-log tikv false not false warning sync-log should be true to avoid recover region when the machine breaks down",
"config raftstore.sync-log tikv false true warning sync-log should be true to avoid recover region when the machine breaks down",
"config transparent_hugepage_enabled tikv [always] madvise never always madvise [never] warning Transparent HugePages can cause memory allocation delays during runtime, TiDB recommends that you disable Transparent HugePages on all TiDB servers",
"version git_hash pd inconsistent consistent critical the cluster has 3 different pd versions, execute the sql to see more detail: select * from information_schema.cluster_info where type='pd'",
"version git_hash tidb inconsistent consistent critical the cluster has 3 different tidb versions, execute the sql to see more detail: select * from information_schema.cluster_info where type='tidb'",
"version git_hash tikv inconsistent consistent critical the cluster has 2 different tikv versions, execute the sql to see more detail: select * from information_schema.cluster_info where type='tikv'",
Expand All @@ -140,9 +149,10 @@ func (s *inspectionResultSuite) TestInspectionResult(c *C) {
rows: []string{
"config coprocessor.high tikv inconsistent consistent warning 192.168.3.32:26600,192.168.3.33:26600 config value is 8\n192.168.3.34:26600,192.168.3.35:26600 config value is 7",
"config ddl.lease tidb inconsistent consistent warning 192.168.3.22:4000,192.168.3.24:4000,192.168.3.25:4000 config value is 1\n192.168.3.23:4000 config value is 2",
"config log.slow-threshold tidb 0 not 0 warning slow-threshold = 0 will record every query to slow log, it may affect performance",
"config log.slow-threshold tidb 0 > 0 warning slow-threshold = 0 will record every query to slow log, it may affect performance",
"config log.slow-threshold tidb inconsistent consistent warning 192.168.3.24:4000 config value is 0\n192.168.3.25:4000 config value is 1",
"config raftstore.sync-log tikv false not false warning sync-log should be true to avoid recover region when the machine breaks down",
"config raftstore.sync-log tikv false true warning sync-log should be true to avoid recover region when the machine breaks down",
"config transparent_hugepage_enabled tikv [always] madvise never always madvise [never] warning Transparent HugePages can cause memory allocation delays during runtime, TiDB recommends that you disable Transparent HugePages on all TiDB servers",
},
},
{
Expand Down Expand Up @@ -208,6 +218,14 @@ func (s *inspectionResultSuite) setupForInspection(c *C, mockData map[string][][
types.MakeDatums("tikv", "tikv-2", "tikv-2s", "4.0", "a234c", "", ""),
},
}
// mock cluster system information
configurations[infoschema.TableClusterSystemInfo] = variable.TableSnapshot{
Rows: [][]types.Datum{
types.MakeDatums("pd", "pd-0", "system", "kernel", "transparent_hugepage_enabled", "always madvise [never]"),
types.MakeDatums("tikv", "tikv-2", "system", "kernel", "transparent_hugepage_enabled", "always madvise [never]"),
types.MakeDatums("tidb", "tidb-0", "system", "kernel", "transparent_hugepage_enabled", "always madvise [never]"),
},
}
}
fpName := "github.com/pingcap/tidb/executor/mockMergeMockInspectionTables"
c.Assert(failpoint.Enable(fpName, "return"), IsNil)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ require (
github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463
github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3
github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a
github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2
github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible
github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f
github.com/prometheus/client_golang v1.5.1
Expand Down
3 changes: 2 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,8 +449,9 @@ github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463/go.mod h1:4rbK1p9ILyIf
github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3 h1:dfdPB1Ot9cNki/hVUgWFUiM8b05b5JCw7Oq9x6HaDeM=
github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3/go.mod h1:74+OEdwM4B/jMpBRl92ch6CSmSYkQtv2TNxIjFdT/GE=
github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a h1:i2RElJ2aykSqZKeY+3SK18NHhajil8cQdG77wHe+P1Y=
github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 h1:b2G/eqDeywtdJF3w9nIUdqMmXChsmpLvf4FzUxJ9Vmk=
github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible h1:qPppnsXVh3KswqRZdSAShGLLPd7dB+5w4lXDnpYn0SQ=
github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f h1:J+0TAI+7Hvebz4bM4GnRCRT4MpjYnUxbyi9ky5ZQUsU=
Expand Down

0 comments on commit 3d536a6

Please sign in to comment.