forked from vectordotdev/vector
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parse-criterion-output.awk
118 lines (106 loc) · 4.05 KB
/
parse-criterion-output.awk
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
###
# This script matches criterion output to output JSON for use for formatting output in CI
#
# We'd prefer to use machine output from cargo-criterion but there is an
# outstanding issue preventing this:
# https://github.com/bheisler/cargo-criterion/issues/17
#
# It matches output like:
# ```
# Benchmarking transforms/transforms
# Benchmarking transforms/transforms: Warming up for 3.0000 s
# Benchmarking transforms/transforms: Collecting 100 samples in estimated 7.6949 s (200 iterations)
# Benchmarking transforms/transforms: Analyzing
# transforms/transforms time: [31.562 ms 31.932 ms 32.307 ms]
# thrpt: [32.471 MiB/s 32.852 MiB/s 33.238 MiB/s]
# change:
# time: [-0.5510% +1.0918% +2.8006%] (p = 0.19 > 0.05)
# thrpt: [-2.7243% -1.0800% +0.5540%]
# No change in performance detected.
# Found 2 outliers among 100 measurements (2.00%)
# 1 (1.00%) low mild
# 1 (1.00%) high severe
#
# Benchmarking complex/complex
# Benchmarking complex/complex: Warming up for 3.0000 s
#
# Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 182.3s, or reduce sample count to 10.
# Benchmarking complex/complex: Collecting 100 samples in estimated 182.26 s (100 iterations)
# Benchmarking complex/complex: Analyzing
# complex/complex time: [1.8272 s 1.8391 s 1.8498 s]
# change: [+0.1481% +0.8758% +1.5410%] (p = 0.01 < 0.05)
# Change within noise threshold.
# Found 5 outliers among 100 measurements (5.00%)
# 3 (3.00%) low severe
# 2 (2.00%) low mild
#
# ```
#
# To output records like:
# ```
# { "throughput": "32.852 MiB/s", "change": "none", "throughput_change": "-1.0800%", "name": "transforms/transforms", "time_change": "+1.0918%", "time": "31.932 ms", "p": "0.00"}
# { "throughput": null, "change": "none", "throughput_change": null, "name": "complex/complex", "time_change": "+0.8758%", "time": "1.8391 s", "p": "0.02"}
# ```
###
BEGIN {
# match time: [1.8272 s 1.8391 s 1.8498 s]
measurement_regex = "\\[(\\S+ \\S+) (\\S+ \\S+) (\\S+ \\S+)\\]"
# match time change: [+0.1481% +0.8758% +1.5410%] (p = 0.01 < 0.05)
time_change_regex = "\\[(\\S+) (\\S+) (\\S+)\\] \\(p = (\\S+) [<>] \\S+\\)"
# match thrpt change: [+0.1481% +0.8758% +1.5410%]
thrpt_change_regex = "\\[(\\S+) (\\S+) (\\S+)\\]"
}
function finish_benchmark(benchmark, change)
{
benchmark["change"] = change
separator = ""
printf "{"
for (field in benchmark) {
value = benchmark[field]
gsub(/\\/, "\\\\", value)
gsub(/"/, "\\\"", value)
if (value != "null") value = "\""value"\""
field = "\""field"\""
printf "%s %s: %s", separator, field, value
separator = ", "
}
printf "}\n"
delete benchmark
in_change = 0
in_bench = 0
}
/^Benchmarking .*: Analyzing$/ {
match($0, /^Benchmarking (.*): Analyzing$/, arr)
num_fields = split("name time time_change throughput throughput_change p change", fields)
for (i=1; i<=num_fields; i++) benchmark[fields[i]] = "null"
benchmark["name"] = arr[1]
benchmark["change"] = "unknown"
in_bench = 1
}
in_bench && /change:$/ { in_change = 1 }
in_change && /time: / {
match($0, time_change_regex, arr)
benchmark["time_change"] = arr[2]
benchmark["p"] = arr[4]
}
in_change && /\s+thrpt: / {
match($0, thrpt_change_regex, arr)
benchmark["throughput_change"] = arr[2]
}
!in_change && /time: / {
match($0, measurement_regex, arr)
benchmark["time"] = arr[2]
}
!in_change && /thrpt: / {
match($0, measurement_regex, arr)
benchmark["throughput"] = arr[2]
}
!in_change && /change: / {
match($0, time_change_regex, arr)
benchmark["time_change"] = arr[2]
benchmark["p"] = arr[4]
}
in_bench && /Performance has improved./ { finish_benchmark(benchmark, "improved") }
in_bench && /Performance has regressed./ { finish_benchmark(benchmark, "regressed") }
in_bench && /Change within noise threshold./ { finish_benchmark(benchmark, "none") }
in_bench && /No change in performance detected./ { finish_benchmark(benchmark, "none") }