forked from automl/SMAC3
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathobjective.py
146 lines (117 loc) · 4.48 KB
/
objective.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import numpy as np
from smac.runhistory.runhistory import RunKey, RunHistory
from smac.configspace import Configuration
"""Define overall objectives.
Overall objectives are functions or callables that calculate the overall
objective of a configuration on the instances and seeds it already ran on."""
def _runtime(config: Configuration, run_history: RunHistory,
instance_seed_pairs=None):
"""Return array of all runtimes for the given config for further calculations.
Parameters
----------
config : Configuration
configuration to calculate objective for
run_history : RunHistory
RunHistory object from which the objective value is computed.
instance_seed_pairs : list, optional (default=None)
list of tuples of instance-seeds pairs. If None, the run_history is
queried for all runs of the given configuration.
Returns
----------
list
"""
try:
id_ = run_history.config_ids[config]
except KeyError: # challenger was not running so far
return []
if instance_seed_pairs is None:
instance_seed_pairs = run_history.get_runs_for_config(config)
runtimes = []
for i, r in instance_seed_pairs:
k = RunKey(id_, i, r)
runtimes.append(run_history.data[k].time)
return runtimes
def total_runtime(config: Configuration, run_history: RunHistory,
instance_seed_pairs=None):
"""Return the total cost of a configuration.
This is the sum of costs of all instance-seed pairs.
Parameters
----------
config : Configuration
Configuration to calculate objective for
run_history : RunHistory
RunHistory object from which the objective value is computed.
instance_seed_pairs : list, optional (default=None)
List of tuples of instance-seeds pairs. If None, the run_history is
queried for all runs of the given configuration.
Returns
-------
Runtime: float
Sum of all costs
"""
return np.sum(_runtime(config, run_history, instance_seed_pairs))
def _cost(config: Configuration, run_history: RunHistory,
instance_seed_pairs=None):
"""Return array of all costs for the given config for further calculations.
Parameters
----------
config : Configuration
Configuration to calculate objective for
run_history : RunHistory
RunHistory object from which the objective value is computed.
instance_seed_pairs : list, optional (default=None)
List of tuples of instance-seeds pairs. If None, the run_history is
queried for all runs of the given configuration.
Returns
-------
Costs: list
Array of all costs
"""
try:
id_ = run_history.config_ids[config]
except KeyError: # challenger was not running so far
return []
if instance_seed_pairs is None:
instance_seed_pairs = run_history.get_runs_for_config(config)
costs = []
for i, r in instance_seed_pairs:
k = RunKey(id_, i, r)
costs.append(run_history.data[k].cost)
return costs
def average_cost(config, run_history, instance_seed_pairs=None):
"""Return the average cost of a configuration.
This is the mean of costs of all instance-seed pairs.
Parameters
----------
config : Configuration
Configuration to calculate objective for
run_history : RunHistory
RunHistory object from which the objective value is computed.
instance_seed_pairs : list, optional (default=None)
List of tuples of instance-seeds pairs. If None, the run_history is
queried for all runs of the given configuration.
Returns
----------
Cost: float
Average cost
"""
return np.mean(_cost(config, run_history, instance_seed_pairs))
def sum_cost(config: Configuration, run_history: RunHistory,
instance_seed_pairs=None):
"""Return the sum of costs of a configuration.
This is the sum of costs of all instance-seed pairs.
Parameters
----------
config : Configuration
Configuration to calculate objective for
run_history : RunHistory
RunHistory object from which the objective value is computed.
instance_seed_pairs : list, optional (default=None)
List of tuples of instance-seeds pairs. If None, the run_history is
queried for all runs of the given configuration.
Returns
----------
sum_cost: float
Sum of costs of config
"""
return np.sum(_cost(config, run_history, instance_seed_pairs))