Skip to content

Commit

Permalink
ADD auto-generation of scenario options in docs (along with some upda…
Browse files Browse the repository at this point in the history 
…te on help-content)
  • Loading branch information
@shukon
shukon committed Jun 9, 2017
1 parent f7b7683 commit 15f4553
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 64 deletions.
4 changes: 4 additions & 0 deletions doc/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

sys.path.insert(0, '..')
import smac
from smac.scenario.scenario import Scenario


# If extensions (or modules to document with autodoc) are in another directory,
Expand Down Expand Up @@ -302,3 +303,6 @@
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Create Scenario-object to update the list of options in the docs
scenario = Scenario({'run_obj':'runtime', 'cutoff_time':1})
scenario.write_options_to_doc()
25 changes: 2 additions & 23 deletions doc/options.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,30 +35,9 @@ For boolean options "1" or "true" both evaluate to True.
The following assumes that the scenario is created via a scenario-file. If it is
generated within custom code, you might not need *algo* or *paramfile*.

Required:
* *run_obj* in [runtime, quality]. Defines what metric to optimize. When optimizing runtime, *cutoff_time* is required as well.
* *cutoff_time* is the maximum runtime, after which the target algorithm is cancelled. **Required if *run_obj* is runtime.**
* *algo* specifies the target algorithm call that *SMAC* will optimize. Interpreted as a bash-command.
* *paramfile* specifies the path to the PCS-file
Options:

Optional:
* *abort_on_first_run_crash* in [true, false]. If true, *SMAC* will abort if the first run of the target algorithm crashes. Default: true.
* *execdir* specifies the path to the execution-directory. Default: ".".
* *deterministic* in [true, false]. If true, the optimization process will be repeatable. Default: false
* *overall_obj* is PARX, where X is an integer defining the penalty imposed on timeouts (i.e. runtimes that exceed the *cutoff-time*). Default: PAR10.
* *memory_limit* is the maximum available memory the target algorithm can occupy before being cancelled.
* *tuner-timeout* is the maximum amount of CPU-time used for optimization. Default: inf.
* *wallclock_limit* is the maximum amount of wallclock-time used for optimization. Default: inf.
* *runcount_limit* is the maximum number of algorithm-calls during optimization. Default: inf.
* *minR* is the minimum number of calls per configuration. Default: 1
* *maxR* is the maximum number of calls per configuration. Default: 2000
* *always_race_default* indicates that new incumbents are always raced against default configuration. Default: false
* *instance_file* specifies the file with the training-instances.
* *test_instance_file* specifies the file with the test-instances.
* *feature_file* specifies the file with the instance-features
* *output_dir* specifies the output-directory for all emerging files, such as logging and results. Default: "smac3-output_YEAR-MONTH-DAY_HOUR:MINUTE:SECOND"
* *shared_model*: Default: false
* *initial_incumbent*: in [DEFAULT, RANDOM]. DEFAULT is the default from the PCS. Default: DEFAULT.
.. include:: scenario_options.rst

.. _paramcs:

Expand Down
100 changes: 59 additions & 41 deletions smac/scenario/scenario.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,12 @@ def __init__(self, scenario, cmd_args=None, run_id=1):
setattr(self, arg_name, arg_value)

self._transform_arguments()

if self.output_dir:
self.output_dir += "_run%d" %(run_id)

self.out_writer.write_scenario_file(self)

self.logger.debug("Scenario Options:")
for arg_name, arg_value in parsed_arguments.items():
if isinstance(arg_value,(int,str,float)):
Expand Down Expand Up @@ -234,36 +234,36 @@ def _add_arguments(self):
# Add allowed arguments
self.add_argument(name='abort_on_first_run_crash',
help="If true, *SMAC* will abort if the first run of "
"the target algorithm crashes. Default: true.",
default='1', callback=_is_truthy)
self.add_argument(name='always_race_default',
help="Race new incumbents always against default configuration",
default=False,
"the target algorithm crashes.",
default=True, callback=_is_truthy)
self.add_argument(name='always_race_default', default=False,
help="Race new incumbents always against default "
"configuration.",
callback=_is_truthy, dest="always_race_default")
self.add_argument(name='algo',
help="Specifies the target algorithm call that *SMAC*"
"will optimize. Interpreted as a bash-command.",
dest='ta', callback=shlex.split)
self.add_argument(name='algo', dest='ta', callback=shlex.split,
help="Specifies the target algorithm call that *SMAC* "
"will optimize. Interpreted as a bash-command.")
self.add_argument(name='execdir', default='.',
help="specifies the path to the execution-directory."
"Default: \".\".")
self.add_argument(name='deterministic', default='0',
help="If true, the optimization process will be"
"repeatable. Default: false.",
callback=_is_truthy)
help="Specifies the path to the execution-directory.")
self.add_argument(name='deterministic', default=False,
help="If true, the optimization process will be "
"repeatable.", callback=_is_truthy)
self.add_argument(name='intensification_percentage', default=0.5,
help=None, callback=float)
self.add_argument(name='paramfile', help="specifies the path to the PCS-file",
help="The fraction of time to be used on "
"intensification (versus choice of next "
"Configurations).", callback=float)
self.add_argument(name='paramfile', help="Specifies the path to the "
"PCS-file.",
dest='pcs_fn', mutually_exclusive_group='cs')
self.add_argument(name='run_obj', default='runtime',
self.add_argument(name='run_obj',
help="Defines what metric to optimize. When "
"optimizing runtime, *cutoff_time* is "
"required as well. Default: runtime.",
"required as well.",
required=True, choice=['runtime', 'quality'])
self.add_argument(name='overall_obj',
help="PARX, where X is an integer defining the "
"penalty imposed on timeouts (i.e. runtimes that "
"exceed the *cutoff-time*). Default: PAR10.",
"exceed the *cutoff-time*).",
default='par10')
self.add_argument(name='cost_for_crash', default=float(MAXINT),
help="Defines the cost-value for crashed runs "
Expand All @@ -278,28 +278,24 @@ def _add_arguments(self):
help="Maximum available memory the target algorithm "
"can occupy before being cancelled.")
self.add_argument(name='tuner-timeout',
help="Maximum amount of CPU-time used for optimization."
"Default: inf.", default=numpy.inf,
help="Maximum amount of CPU-time used for optimization.",
default=numpy.inf,
dest='algo_runs_timelimit', callback=float)
self.add_argument(name='wallclock_limit',
help="Maximum amount of wallclock-time used for "
"optimization. Default: inf.",
help="Maximum amount of wallclock-time used for optimization.",
default=numpy.inf, callback=float)
self.add_argument(name='always_race_default',
help="Race new incumbents always against default configuration",
help="Race new incumbents always against default configuration.",
default=False,
callback=_is_truthy, dest="always_race_default")
self.add_argument(name='runcount_limit',
help="Maximum number of algorithm-calls during "
"optimization. Default: inf.",
help="Maximum number of algorithm-calls during optimization.",
default=numpy.inf, callback=float, dest="ta_run_limit")
self.add_argument(name='minR',
help="Minimum number of calls per configuration. "
"Default: 1",
help="Minimum number of calls per configuration.",
default=1, callback=int, dest='minR')
self.add_argument(name='maxR',
help="Maximum number of calls per configuration. "
"Default: 2000",
help="Maximum number of calls per configuration.",
default=2000, callback=int, dest='maxR')
self.add_argument(name='instance_file',
help="Specifies the file with the training-instances.",
Expand All @@ -312,23 +308,23 @@ def _add_arguments(self):
dest='feature_fn')
self.add_argument(name='output_dir',
help="Specifies the output-directory for all emerging "
"files, such as logging and results. Default: "
"\"smac3-output_YEAR-MONTH-DAY_HOUR:MINUTE:SECOND\"",
"files, such as logging and results.",
default="smac3-output_%s" % (
datetime.datetime.fromtimestamp(
time.time()).strftime(
'%Y-%m-%d_%H:%M:%S_(%f)')))
self.add_argument(name='input_psmac_dirs', help=None,
default=None)
self.add_argument(name='shared_model', help=None, default='0',
callback=_is_truthy)
self.add_argument(name='input_psmac_dirs', default=None,
help="For parallel SMAC, multiple output-directories "
"are used.")
self.add_argument(name='shared_model',
help="Whether to run SMAC in parallel mode.",
default=False, callback=_is_truthy)
self.add_argument(name='instances', default=[[None]], help=None,
dest='train_insts')
self.add_argument(name='test_instances', default=[[None]], help=None,
dest='test_insts')
self.add_argument(name='initial_incumbent', default="DEFAULT",
help="In [DEFAULT, RANDOM]. DEFAULT is the default "
"from the PCS. Default: DEFAULT.",
help="DEFAULT is the default from the PCS.",
dest='initial_incumbent',
choice=['DEFAULT', 'RANDOM'])
# instance name -> feature vector
Expand Down Expand Up @@ -451,3 +447,25 @@ def _to_str_and_warn(self, l: typing.List[typing.Any]):
if warn_:
self.logger.warn("All instances were casted to str.")
return l

def write_options_to_doc(self, path='scenario_options.rst'):
"""
Writes the option-list to file for autogeneration in documentation.
The list is created in doc/conf.py and read in doc/options.rst.
Parameters
----------
path: string
where to write to (possibly relativ to doc-folder)
"""
exclude = ['cs', 'features', 'instances', 'test_instances']
with open(path, 'w') as fh:
for arg in sorted(self._arguments.keys()):
if arg in exclude:
continue
fh.write(" * *{}*: {}".format(arg, self._arguments[arg]['help']))
if self._arguments[arg]['default']:
fh.write(" Default: {}.".format(self._arguments[arg]['default']))
if self._arguments[arg]['choice']:
fh.write(" Must be from: {}.".format(self._arguments[arg]['choice']))
fh.write("\n")

0 comments on commit 15f4553

Please sign in to comment.