-
Notifications
You must be signed in to change notification settings - Fork 1
/
stats_loader.go
284 lines (245 loc) · 10.9 KB
/
stats_loader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
//go:build rocksdb
// +build rocksdb
package opendb
import (
"fmt"
"strconv"
)
const (
sum = "SUM"
count = "COUNT"
p50 = "P50"
p95 = "P95"
p99 = "P99"
p100 = "P100"
)
type statLoader struct {
// statMap contains map of stat objects returned by parseSerializedStats function
// example of stats:
// #1: rocksdb.block.cache.miss COUNT : 5
// #2: rocksdb.compaction.times.micros P50 : 21112 P95 : 21112 P99 : 21112 P100 : 21112 COUNT : 1 SUM : 21112
// #1 case will be cast into int64
// #2 case will be cast into float64Histogram
statMap map[string]*stat
// NOTE: some methods accumulate errors instead of returning them, these methods are private and not intended to use outside
errors []error
}
func newStatLoader(statMap map[string]*stat) *statLoader {
return &statLoader{
statMap: statMap,
errors: make([]error, 0),
}
}
type stats struct {
NumberKeysWritten int64
NumberKeysRead int64
NumberKeysUpdated int64
// total block cache misses
// BLOCK_CACHE_MISS == BLOCK_CACHE_INDEX_MISS +
// BLOCK_CACHE_FILTER_MISS +
// BLOCK_CACHE_DATA_MISS;
// BLOCK_CACHE_INDEX_MISS: # of times cache miss when accessing index block from block cache.
// BLOCK_CACHE_FILTER_MISS: # of times cache miss when accessing filter block from block cache.
// BLOCK_CACHE_DATA_MISS: # of times cache miss when accessing data block from block cache.
BlockCacheMiss int64
// total block cache hit
// BLOCK_CACHE_HIT == BLOCK_CACHE_INDEX_HIT +
// BLOCK_CACHE_FILTER_HIT +
// BLOCK_CACHE_DATA_HIT;
// BLOCK_CACHE_INDEX_HIT: # of times cache hit when accessing index block from block cache.
// BLOCK_CACHE_FILTER_HIT: # of times cache hit when accessing filter block from block cache.
// BLOCK_CACHE_DATA_HIT: # of times cache hit when accessing data block from block cache.
BlockCacheHit int64
// # of blocks added to block cache.
BlockCacheAdd int64
// # of failures when adding blocks to block cache.
BlockCacheAddFailures int64
BlockCacheIndexMiss int64
BlockCacheIndexHit int64
BlockCacheIndexBytesInsert int64
BlockCacheFilterMiss int64
BlockCacheFilterHit int64
BlockCacheFilterBytesInsert int64
BlockCacheDataMiss int64
BlockCacheDataHit int64
BlockCacheDataBytesInsert int64
CompactReadBytes int64 // Bytes read during compaction
CompactWriteBytes int64 // Bytes written during compaction
CompactionTimesMicros *float64Histogram
CompactionTimesCPUMicros *float64Histogram
NumFilesInSingleCompaction *float64Histogram
// Read amplification statistics.
// Read amplification can be calculated using this formula
// (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES)
//
// REQUIRES: ReadOptions::read_amp_bytes_per_bit to be enabled
// TODO(yevhenii): seems not working?
ReadAmpEstimateUsefulBytes int64 // Estimate of total bytes actually used.
ReadAmpTotalReadBytes int64 // Total size of loaded data blocks.
NumberFileOpens int64
NumberFileErrors int64
// # of times bloom filter has avoided file reads, i.e., negatives.
BloomFilterUseful int64
// # of times bloom FullFilter has not avoided the reads.
BloomFilterFullPositive int64
// # of times bloom FullFilter has not avoided the reads and data actually
// exist.
BloomFilterFullTruePositive int64
// # of memtable hits.
MemtableHit int64
// # of memtable misses.
MemtableMiss int64
// # of Get() queries served by L0
GetHitL0 int64
// # of Get() queries served by L1
GetHitL1 int64
// # of Get() queries served by L2 and up
GetHitL2AndUp int64
// The number of uncompressed bytes issued by DB::Put(), DB::Delete(),
// DB::Merge(), and DB::Write().
BytesWritten int64
// The number of uncompressed bytes read from DB::Get(). It could be
// either from memtables, cache, or table files.
// For the number of logical bytes read from DB::MultiGet(),
// please use NUMBER_MULTIGET_BYTES_READ.
BytesRead int64
// Writer has to wait for compaction or flush to finish.
StallMicros int64
DBWriteStallHistogram *float64Histogram
// Last level and non-last level read statistics
LastLevelReadBytes int64
LastLevelReadCount int64
NonLastLevelReadBytes int64
NonLastLevelReadCount int64
DBGetMicros *float64Histogram
DBWriteMicros *float64Histogram
// Value size distribution in each operation
BytesPerRead *float64Histogram
BytesPerWrite *float64Histogram
BytesPerMultiget *float64Histogram
// Time spent flushing memtable to disk
FlushMicros *float64Histogram
}
type float64Histogram struct {
Sum float64
Count float64
P50 float64
P95 float64
P99 float64
P100 float64
}
func (l *statLoader) error() error {
if len(l.errors) != 0 {
return fmt.Errorf("%v", l.errors)
}
return nil
}
func (l *statLoader) load() (*stats, error) {
stats := &stats{
NumberKeysWritten: l.getInt64StatValue("rocksdb.number.keys.written", count),
NumberKeysRead: l.getInt64StatValue("rocksdb.number.keys.read", count),
NumberKeysUpdated: l.getInt64StatValue("rocksdb.number.keys.updated", count),
BlockCacheMiss: l.getInt64StatValue("rocksdb.block.cache.miss", count),
BlockCacheHit: l.getInt64StatValue("rocksdb.block.cache.hit", count),
BlockCacheAdd: l.getInt64StatValue("rocksdb.block.cache.add", count),
BlockCacheAddFailures: l.getInt64StatValue("rocksdb.block.cache.add.failures", count),
BlockCacheIndexMiss: l.getInt64StatValue("rocksdb.block.cache.index.miss", count),
BlockCacheIndexHit: l.getInt64StatValue("rocksdb.block.cache.index.hit", count),
BlockCacheIndexBytesInsert: l.getInt64StatValue("rocksdb.block.cache.index.bytes.insert", count),
BlockCacheFilterMiss: l.getInt64StatValue("rocksdb.block.cache.filter.miss", count),
BlockCacheFilterHit: l.getInt64StatValue("rocksdb.block.cache.filter.hit", count),
BlockCacheFilterBytesInsert: l.getInt64StatValue("rocksdb.block.cache.filter.bytes.insert", count),
BlockCacheDataMiss: l.getInt64StatValue("rocksdb.block.cache.data.miss", count),
BlockCacheDataHit: l.getInt64StatValue("rocksdb.block.cache.data.hit", count),
BlockCacheDataBytesInsert: l.getInt64StatValue("rocksdb.block.cache.data.bytes.insert", count),
CompactReadBytes: l.getInt64StatValue("rocksdb.compact.read.bytes", count),
CompactWriteBytes: l.getInt64StatValue("rocksdb.compact.write.bytes", count),
CompactionTimesMicros: l.getFloat64HistogramStatValue("rocksdb.compaction.times.micros"),
CompactionTimesCPUMicros: l.getFloat64HistogramStatValue("rocksdb.compaction.times.cpu_micros"),
NumFilesInSingleCompaction: l.getFloat64HistogramStatValue("rocksdb.numfiles.in.singlecompaction"),
ReadAmpEstimateUsefulBytes: l.getInt64StatValue("rocksdb.read.amp.estimate.useful.bytes", count),
ReadAmpTotalReadBytes: l.getInt64StatValue("rocksdb.read.amp.total.read.bytes", count),
NumberFileOpens: l.getInt64StatValue("rocksdb.no.file.opens", count),
NumberFileErrors: l.getInt64StatValue("rocksdb.no.file.errors", count),
BloomFilterUseful: l.getInt64StatValue("rocksdb.bloom.filter.useful", count),
BloomFilterFullPositive: l.getInt64StatValue("rocksdb.bloom.filter.full.positive", count),
BloomFilterFullTruePositive: l.getInt64StatValue("rocksdb.bloom.filter.full.true.positive", count),
MemtableHit: l.getInt64StatValue("rocksdb.memtable.hit", count),
MemtableMiss: l.getInt64StatValue("rocksdb.memtable.miss", count),
GetHitL0: l.getInt64StatValue("rocksdb.l0.hit", count),
GetHitL1: l.getInt64StatValue("rocksdb.l1.hit", count),
GetHitL2AndUp: l.getInt64StatValue("rocksdb.l2andup.hit", count),
BytesWritten: l.getInt64StatValue("rocksdb.bytes.written", count),
BytesRead: l.getInt64StatValue("rocksdb.bytes.read", count),
StallMicros: l.getInt64StatValue("rocksdb.stall.micros", count),
DBWriteStallHistogram: l.getFloat64HistogramStatValue("rocksdb.db.write.stall"),
LastLevelReadBytes: l.getInt64StatValue("rocksdb.last.level.read.bytes", count),
LastLevelReadCount: l.getInt64StatValue("rocksdb.last.level.read.count", count),
NonLastLevelReadBytes: l.getInt64StatValue("rocksdb.non.last.level.read.bytes", count),
NonLastLevelReadCount: l.getInt64StatValue("rocksdb.non.last.level.read.count", count),
DBGetMicros: l.getFloat64HistogramStatValue("rocksdb.db.get.micros"),
DBWriteMicros: l.getFloat64HistogramStatValue("rocksdb.db.write.micros"),
BytesPerRead: l.getFloat64HistogramStatValue("rocksdb.bytes.per.read"),
BytesPerWrite: l.getFloat64HistogramStatValue("rocksdb.bytes.per.write"),
BytesPerMultiget: l.getFloat64HistogramStatValue("rocksdb.bytes.per.multiget"),
FlushMicros: l.getFloat64HistogramStatValue("rocksdb.db.flush.micros"),
}
err := l.error()
if err != nil {
return nil, err
}
return stats, nil
}
// getFloat64HistogramStatValue converts stat object into float64Histogram
func (l *statLoader) getFloat64HistogramStatValue(statName string) *float64Histogram {
return &float64Histogram{
Sum: l.getFloat64StatValue(statName, sum),
Count: l.getFloat64StatValue(statName, count),
P50: l.getFloat64StatValue(statName, p50),
P95: l.getFloat64StatValue(statName, p95),
P99: l.getFloat64StatValue(statName, p99),
P100: l.getFloat64StatValue(statName, p100),
}
}
// getInt64StatValue converts property of stat object into int64
func (l *statLoader) getInt64StatValue(statName, propName string) int64 {
stringVal := l.getStatValue(statName, propName)
if stringVal == "" {
l.errors = append(l.errors, fmt.Errorf("can't get stat by name: %v", statName))
return 0
}
intVal, err := strconv.ParseInt(stringVal, 10, 64)
if err != nil {
l.errors = append(l.errors, fmt.Errorf("can't parse int: %v", err))
return 0
}
return intVal
}
// getFloat64StatValue converts property of stat object into float64
func (l *statLoader) getFloat64StatValue(statName, propName string) float64 {
stringVal := l.getStatValue(statName, propName)
if stringVal == "" {
l.errors = append(l.errors, fmt.Errorf("can't get stat by name: %v", statName))
return 0
}
floatVal, err := strconv.ParseFloat(stringVal, 64)
if err != nil {
l.errors = append(l.errors, fmt.Errorf("can't parse float: %v", err))
return 0
}
return floatVal
}
// getStatValue gets property of stat object
func (l *statLoader) getStatValue(statName, propName string) string {
stat, ok := l.statMap[statName]
if !ok {
l.errors = append(l.errors, fmt.Errorf("stat %v doesn't exist", statName))
return ""
}
prop, ok := stat.props[propName]
if !ok {
l.errors = append(l.errors, fmt.Errorf("stat %v doesn't have %v property", statName, propName))
return ""
}
return prop
}