From b91c1665bc6e324f6c8134cffab212f67345eaac Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 22 Oct 2020 17:09:22 +0200 Subject: [PATCH 1/5] Add better error messages (#694) * add better error messages * fix bugs --- smac/optimizer/smbo.py | 12 +++++------- smac/tae/execute_func.py | 14 ++++++++++++-- test/test_smbo/test_smbo.py | 3 ++- test/test_tae/test_parallel_runner.py | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/smac/optimizer/smbo.py b/smac/optimizer/smbo.py index 5faf4e3db..3d3c5ba8d 100644 --- a/smac/optimizer/smbo.py +++ b/smac/optimizer/smbo.py @@ -464,13 +464,11 @@ def _incorporate_run_results(self, run_info: RunInfo, result: RunValue, if self.scenario.abort_on_first_run_crash : # type: ignore[attr-defined] # noqa F821 if self.stats.finished_ta_runs == 1 and result.status == StatusType.CRASHED: - raise FirstRunCrashedException("First run crashed, abort. " - "Please check your setup -- " - "we assume that your default" - "configuration does not crashes. " - "(To deactivate this exception," - " use the SMAC scenario option " - "'abort_on_first_run_crash')") + raise FirstRunCrashedException( + "First run crashed, abort. Please check your setup -- we assume that your default " + "configuration does not crashes. (To deactivate this exception, use the SMAC scenario option " + "'abort_on_first_run_crash'). Additional run info: %s" % result.additional_info + ) # Update the intensifier with the result of the runs self.incumbent, inc_perf = self.intensifier.process_results( diff --git a/smac/tae/execute_func.py b/smac/tae/execute_func.py index 8c6b59418..0c19bf038 100644 --- a/smac/tae/execute_func.py +++ b/smac/tae/execute_func.py @@ -2,6 +2,7 @@ import math import time import json +import traceback import typing import numpy as np @@ -165,8 +166,17 @@ def run(self, config: Configuration, } # call ta - obj = pynisher.enforce_limits(**arguments)(self._ta) - rval = self._call_ta(obj, config, obj_kwargs) + try: + obj = pynisher.enforce_limits(**arguments)(self._ta) + rval = self._call_ta(obj, config, obj_kwargs) + except Exception as e: + exception_traceback = traceback.format_exc() + error_message = repr(e) + additional_info = { + 'traceback': exception_traceback, + 'error': error_message + } + return StatusType.CRASHED, self.cost_for_crash, 0.0, additional_info if isinstance(rval, tuple): result = rval[0] diff --git a/test/test_smbo/test_smbo.py b/test/test_smbo/test_smbo.py index 16fcaee5e..a71aa134e 100644 --- a/test/test_smbo/test_smbo.py +++ b/test/test_smbo/test_smbo.py @@ -108,7 +108,8 @@ def target(x): 'abort_on_first_run_crash': True}) self.output_dirs.append(scen.output_dir) smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver - self.assertRaises(FirstRunCrashedException, smbo.run) + with self.assertRaisesRegex(FirstRunCrashedException, "in _mock_call"): + smbo.run() # should not raise an error if abort_on_first_run_crash is False patch.side_effect = FirstRunCrashedException() diff --git a/test/test_tae/test_parallel_runner.py b/test/test_tae/test_parallel_runner.py index 53a436ee6..79720c9ec 100644 --- a/test/test_tae/test_parallel_runner.py +++ b/test/test_tae/test_parallel_runner.py @@ -168,7 +168,7 @@ def target_nonpickable(x, seed, instance): self.assertIn( # We expect the problem to occur in the run wrapper # So traceback should show this! - 'in run_wrapper', + 'target_nonpickable', result.additional_info['traceback']) # Make sure the error message is included From f78986cf55583c8aa430074e2ec8d12ea5e347dc Mon Sep 17 00:00:00 2001 From: Francisco Rivera Valverde <44504424+franchuterivera@users.noreply.github.com> Date: Thu, 22 Oct 2020 18:39:40 +0200 Subject: [PATCH 2/5] fix_695 (#700) * fix_695 * Incorporated comment feedback * Mypy fixing --- smac/intensification/successive_halving.py | 94 +++++-- smac/utils/validate.py | 6 +- test/test_intensify/test_eval_utils.py | 2 + test/test_intensify/test_hyperband.py | 8 + .../test_intensify/test_successive_halving.py | 251 +++++++++++++++++- 5 files changed, 339 insertions(+), 22 deletions(-) diff --git a/smac/intensification/successive_halving.py b/smac/intensification/successive_halving.py index d4773e06b..d287a80bc 100644 --- a/smac/intensification/successive_halving.py +++ b/smac/intensification/successive_halving.py @@ -213,7 +213,7 @@ def __init__(self, # run history, does not have this information and so we track locally. That way, # when we access the complete list of configs from the run history, we filter # the ones launched by the current succesive halver using self.run_tracker - self.run_tracker = [] # type: typing.List[typing.Tuple[Configuration, str, int]] + self.run_tracker = {} # type: typing.Dict[typing.Tuple[Configuration, str, int, float], bool] def _init_sh_params(self, initial_budget: typing.Optional[float], @@ -328,6 +328,9 @@ def process_results(self, empirical performance of incumbent configuration """ + # Mark the fact that we processed this configuration + self.run_tracker[(run_info.config, run_info.instance, run_info.seed, run_info.budget)] = True + # If The incumbent is None and it is the first run, we use the challenger if not incumbent and self.first_run: self.logger.info( @@ -336,14 +339,9 @@ def process_results(self, incumbent = run_info.config self.first_run = False - # selecting instance-seed subset for this budget, depending on the kind of budget - curr_budget = self.all_budgets[self.stage] - if self.instance_as_budget: - prev_budget = int(self.all_budgets[self.stage - 1]) if self.stage > 0 else 0 - curr_insts = self.inst_seed_pairs[int(prev_budget):int(curr_budget)] - else: - curr_insts = self.inst_seed_pairs - n_insts_remaining = len(curr_insts) - self.curr_inst_idx - 1 + # Account for running instances across configurations, not only on the + # running configuration + n_insts_remaining = self._get_pending_instances_for_stage(run_history) # Make sure that there is no Budget exhausted if result.status == StatusType.CAPPED: @@ -367,8 +365,18 @@ def process_results(self, else: self.fail_challengers.add(run_info.config) # capped/crashed/do not advance configs + # We need to update the incumbent if this config we are processing + # completes all scheduled instance-seed pairs. + # Here, a config/seed/instance is going to be processed for the first time + # (it has been previously scheduled by get_next_run and marked False, indicating + # that it has not been processed yet. Entering process_results() this config/seed/instance + # is marked as TRUE as an indication that it has finished and should be processed) + # so if all configurations runs are marked as TRUE it means that this new config + # was the missing piece to have everything needed to compare against the incumbent + update_incumbent = all([v for k, v in self.run_tracker.items() if k[0] == run_info.config]) + # get incumbent if all instances have been evaluated - if n_insts_remaining <= 0: + if n_insts_remaining <= 0 or update_incumbent: incumbent = self._compare_configs(challenger=run_info.config, incumbent=incumbent, run_history=run_history, @@ -582,7 +590,9 @@ def get_next_run(self, if (self.cutoff is not None) and (cutoff < self.cutoff): # type: ignore[operator] # noqa F821 capped = True - self.run_tracker.append((challenger, instance, seed)) + budget = 0.0 if self.instance_as_budget else curr_budget + + self.run_tracker[(challenger, instance, seed, budget)] = False return RunInfoIntent.RUN, RunInfo( config=challenger, instance=instance, @@ -590,7 +600,7 @@ def get_next_run(self, seed=seed, cutoff=cutoff, capped=capped, - budget=0.0 if self.instance_as_budget else curr_budget, + budget=budget, source_id=self.identifier, ) @@ -674,7 +684,7 @@ def _update_stage(self, run_history: RunHistory) -> None: self.iteration_done = True self.sh_iters += 1 self.stage = 0 - self.run_tracker = [] + self.run_tracker = {} self.configs_to_run = [] self.fail_chal_offset = 0 @@ -881,14 +891,18 @@ def _top_k(self, for c in configs: # ensuring that all configurations being compared are run on the same set of instance, seed & budget cur_run_key = run_history.get_runs_for_config(c, only_max_observed_budget=True) - if cur_run_key != run_key: + + # Move to compare set -- get_runs_for_config queries form a dictionary + # which is not an ordered structure. Some queries to that dictionary returned unordered + # list which wrongly trigger the below if + if set(cur_run_key) != set(run_key): raise ValueError( 'Cannot compare configs that were run on different instances-seeds-budgets: %s vs %s' % (run_key, cur_run_key) ) config_costs[c] = run_history.get_cost(c) - configs_sorted = sorted(config_costs, key=config_costs.get) + configs_sorted = [k for k, v in sorted(config_costs.items(), key=lambda item: item[1])] # select top configurations only top_configs = configs_sorted[:k] return top_configs @@ -913,6 +927,38 @@ def _count_running_instances_for_challenger(self, run_history: RunHistory) -> in return running_instances + def _get_pending_instances_for_stage(self, run_history: RunHistory) -> int: + """ + When running SH, M configs might require N instances. Before moving to the + next stage, we need to make sure that all MxN jobs are completed + + We use the run tracker to make sure we processed all configurations. + + Parameters + ---------- + run_history : RunHistory + stores all runs we ran so far + + Returns + ------- + int: All the instances that have not yet been processed + """ + curr_budget = self.all_budgets[self.stage] + if self.instance_as_budget: + prev_budget = int(self.all_budgets[self.stage - 1]) if self.stage > 0 else 0 + curr_insts = self.inst_seed_pairs[int(prev_budget):int(curr_budget)] + else: + curr_insts = self.inst_seed_pairs + + # The minus one here accounts for the fact that len(curr_insts) is a length starting at 1 + # and self.curr_inst_idx is a zero based index + # But when all configurations have been launched and are running in run history + # n_insts_remaining becomes -1, which is confusing. Cap to zero + n_insts_remaining = max(len(curr_insts) - self.curr_inst_idx - 1, 0) + # If there are pending runs from a past config, wait for them + pending_to_process = [k for k, v in self.run_tracker.items() if not v] + return n_insts_remaining + len(pending_to_process) + def _launched_all_configs_for_current_stage(self, run_history: RunHistory) -> bool: """ This procedure queries if the addition of currently finished configs @@ -943,9 +989,25 @@ def _launched_all_configs_for_current_stage(self, run_history: RunHistory) -> bo n_insts_remaining = len(curr_insts) - (self.curr_inst_idx + running_instances) # Check which of the current configs is running - my_configs = [c for c, i, s in self.run_tracker] + my_configs = [c for c, i, s, b in self.run_tracker] running_configs = set() + tracked_configs = self.success_challengers.union( + self.fail_challengers).union(self.do_not_advance_challengers) for k, v in run_history.data.items(): + # Our goal here is to account for number of challengers available + # We care if the challenger is running only if is is not tracked in + # success/fails/do not advance + # In other words, in each SH iteration we have to run N configs on + # M instance/seed pairs. This part of the code makes sure that N different + # configurations are launched (we only move to a new config after M + # instance-seed pairs on that config are launched) + # Notice that this number N of configs tracked in num_chal_available + # is a set of processed configurations + the running challengers + # so we do not want to double count configurations + # n_insts_remaining variable above accounts for the last active configuration only + if run_history.ids_config[k.config_id] in tracked_configs: + continue + if v.status == StatusType.RUNNING: if run_history.ids_config[k.config_id] in my_configs: running_configs.add(k.config_id) diff --git a/smac/utils/validate.py b/smac/utils/validate.py index 34cd6d93f..f3045b855 100644 --- a/smac/utils/validate.py +++ b/smac/utils/validate.py @@ -445,9 +445,9 @@ def _get_runs(self, if isinstance(configs, str): configs = self._get_configs(configs) if isinstance(insts, str): - instances = self._get_instances(insts) # type: typing.Sequence[typing.Union[str, None]] + instances = sorted(self._get_instances(insts)) # type: typing.Sequence[typing.Union[str, None]] elif insts is not None: - instances = insts + instances = sorted(insts) else: instances = [None] # If no instances are given, fix the instances to one "None" instance @@ -470,7 +470,7 @@ def _get_runs(self, # If we reuse runs, we want to return them as well new_rh = RunHistory() - for i in sorted(instances): + for i in instances: for rep in range(repetitions): # First, find a seed and add all the data we can take from the # given runhistory to "our" validation runhistory. diff --git a/test/test_intensify/test_eval_utils.py b/test/test_intensify/test_eval_utils.py index ce68f56dc..8b7efd444 100644 --- a/test/test_intensify/test_eval_utils.py +++ b/test/test_intensify/test_eval_utils.py @@ -8,6 +8,7 @@ def eval_challenger( taf: ExecuteTAFuncDict, stats: Stats, runhistory: RunHistory, + force_update=False, ): """ Wrapper over challenger evaluation @@ -29,6 +30,7 @@ def eval_challenger( instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, + force_update=force_update, ) stats.n_configs = len(runhistory.config_ids) return result diff --git a/test/test_intensify/test_hyperband.py b/test/test_intensify/test_hyperband.py index c01ef602e..0a1e69bfb 100644 --- a/test/test_intensify/test_hyperband.py +++ b/test/test_intensify/test_hyperband.py @@ -510,12 +510,20 @@ def target(x): self.assertEqual(intensifier.s, 1) self.assertEqual(intensifier.s_max, 1) + # We assume now that process results was called with below successes. + # We track closely run execution through run_tracker, so this also + # has to be update -- the fact that the succesive halving inside hyperband + # processed the given configurations self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, seed=0, budget=1) + intensifier.sh_intensifier.run_tracker[(self.config1, None, 0, 1)] = True self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) + intensifier.sh_intensifier.run_tracker[(self.config2, None, 0, 0.5)] = True self.rh.add(config=self.config3, cost=3, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) + intensifier.sh_intensifier.run_tracker[(self.config3, None, 0, 0.5)] = True + intensifier.sh_intensifier.success_challengers = {self.config2, self.config3} intensifier.sh_intensifier._update_stage(self.rh) intent, run_info = intensifier.get_next_run( diff --git a/test/test_intensify/test_successive_halving.py b/test/test_intensify/test_successive_halving.py index cc9faba07..38b804a07 100644 --- a/test/test_intensify/test_successive_halving.py +++ b/test/test_intensify/test_successive_halving.py @@ -1422,8 +1422,8 @@ def target(x): total_configs_in_stage = 4 instances_per_stage = 2 - # get all configs and add them to list - run_tracker = [] + # get all configs and add them to the dict + run_tracker = {} challengers = [self.config1, self.config2, self.config3, self.config4] for i in range(total_configs_in_stage * instances_per_stage): intent, run_info = intensifier.get_next_run( @@ -1438,7 +1438,7 @@ def target(x): # Remove from the challengers, the launched configs challengers = [c for c in challengers if c != run_info.config] - run_tracker.append((run_info.config, run_info.instance, run_info.seed)) + run_tracker[(run_info.config, run_info.instance, run_info.seed)] = False self.rh.add(config=run_info.config, instance_id=run_info.instance, seed=run_info.seed, @@ -1461,6 +1461,251 @@ def target(x): # For all runs to be completed before moving to the next stage self.assertEqual(intent, RunInfoIntent.WAIT) + def _exhaust_stage_execution(self, intensifier, taf, challengers, incumbent): + """ + Exhaust configuration/instances seed and returns the + run_info that were not launched. + + The idea with this procedure is to emulate the fact that some + configurations will finish while others won't. We need to be + robust against this scenario + """ + pending_processing = [] + stage = 0 if not hasattr(intensifier, 'stage') else intensifier.stage + curr_budget = intensifier.all_budgets[stage] + prev_budget = int(intensifier.all_budgets[stage - 1]) if stage > 0 else 0 + if intensifier.instance_as_budget: + total_runs = int(curr_budget - prev_budget) * int( + intensifier.n_configs_in_stage[stage]) + toggle = np.random.choice([True, False], total_runs).tolist() + while not np.any(toggle) or not np.any(np.invert(toggle)): + # make sure we have both true and false! + toggle = np.random.choice([True, False], total_runs).tolist() + else: + # If we directly use the budget, then there are no instances to wait + # But we still want to mimic pending configurations. That is, we don't + # advance to the next stage until all configurations are done for a given + # budget. + # Here if we do not launch a configuration because toggle was false, is + # like this configuration never exited as there is only 1 instance in this + # and if toggle is false, it is never run. So we cannot do a random toggle + toggle = [False, True, False, True] + + while True: + intent, run_info = intensifier.get_next_run( + challengers=challengers, + chooser=None, + run_history=self.rh, + incumbent=incumbent, + ) + + # Update the challengers + challengers = [c for c in challengers if c != run_info.config] + + if intent == RunInfoIntent.WAIT: + break + + # Add this configuration as running + self.rh.add(config=run_info.config, + instance_id=run_info.instance, + seed=run_info.seed, + budget=run_info.budget, + cost=1000, + time=1000, + status=StatusType.RUNNING, + additional_info=None) + + if toggle.pop(): + result = eval_challenger(run_info, taf, self.stats, self.rh, + force_update=True) + incumbent, inc_value = intensifier.process_results( + run_info=run_info, + incumbent=incumbent, + run_history=self.rh, + time_bound=np.inf, + result=result, + log_traj=False, + ) + else: + pending_processing.append(run_info) + + # In case a iteration is done, break + # This happens if the configs per stage is 1 + if intensifier.iteration_done: + break + + return pending_processing, incumbent + + def test_iteration_done_only_when_all_configs_processed_instance_as_budget(self): + """ + Makes sure that iteration done for a given stage is asserted ONLY after all + configurations AND instances are completed, when instance is used as budget + """ + def target(x): + return 1 + taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj='quality') + + taf.runhistory = self.rh + # select best on any budget + intensifier = _SuccessiveHalving( + stats=self.stats, traj_logger=None, + rng=np.random.RandomState(12345), run_obj_time=False, + deterministic=True, + instances=list(range(5)), initial_budget=2, max_budget=5, eta=2) + + # we want to test instance as budget + self.assertTrue(intensifier.instance_as_budget) + + # Run until there are no more configurations to be proposed + # Skip running some configurations to emulate the fact that runs finish on different time + # We need this because there was a bug where not all instances had finished, yet + # the SH instance assumed all configurations finished + challengers = [self.config1, self.config2, self.config3, self.config4] + incumbent = None + pending_processing, incumbent = self._exhaust_stage_execution(intensifier, taf, + challengers, incumbent) + + # We have configurations pending, so iteration should NOT be done + self.assertFalse(intensifier.iteration_done) + + # Make sure we launched all configurations we were meant to: + # all_budgets=[2.5 5. ] n_configs_in_stage=[2.0, 1.0] + # We need 2 configurations in the run history + configurations = set([k.config_id for k, v in self.rh.data.items()]) + self.assertEqual(configurations, {1, 2}) + # We need int(2.5) instances in the run history per config + config_inst_seed = set([k for k, v in self.rh.data.items()]) + self.assertEqual(len(config_inst_seed), 4) + + # Go to the last stage. Notice that iteration should not be done + # as we are in stage 1 out of 2 + for run_info in pending_processing: + result = eval_challenger(run_info, taf, self.stats, self.rh, + force_update=True) + incumbent, inc_value = intensifier.process_results( + run_info=run_info, + incumbent=self.config1, + run_history=self.rh, + time_bound=np.inf, + result=result, + log_traj=False, + ) + self.assertFalse(intensifier.iteration_done) + + # we transition to stage 1, where the budget is 5 + self.assertEqual(intensifier.stage, 1) + + pending_processing, incumbent = self._exhaust_stage_execution(intensifier, taf, + challengers, incumbent) + + # Because budget is 5, BUT we previously ran 2 instances in stage 0 + # we expect that the run history will be populated with 3 new instances for 1 + # config more 4 (stage0, 2 config on 2 instances) + 3 (stage1, 1 config 3 instances) = 7 + config_inst_seed = [k for k, v in self.rh.data.items()] + self.assertEqual(len(config_inst_seed), 7) + + # All new runs should be on the same config + self.assertEqual(len(set([c.config_id for c in config_inst_seed[4:]])), 1) + # We need 3 new instance seed pairs + self.assertEqual(len(set(config_inst_seed[4:])), 3) + + # because there are configurations pending, no iteration should be done + self.assertFalse(intensifier.iteration_done) + + # Finish the pending runs + for run_info in pending_processing: + result = eval_challenger(run_info, taf, self.stats, self.rh, + force_update=True) + incumbent, inc_value = intensifier.process_results( + run_info=run_info, + incumbent=incumbent, + run_history=self.rh, + time_bound=np.inf, + result=result, + log_traj=False, + ) + + # Finally, all stages are done, so iteration should be done!! + self.assertTrue(intensifier.iteration_done) + + def test_iteration_done_only_when_all_configs_processed_no_instance_as_budget(self): + """ + Makes sure that iteration done for a given stage is asserted ONLY after all + configurations AND instances are completed, when instance is NOT used as budget + """ + def target(x): + return 1 + taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj='quality') + + taf.runhistory = self.rh + # select best on any budget + intensifier = _SuccessiveHalving( + stats=self.stats, traj_logger=None, + rng=np.random.RandomState(12345), run_obj_time=False, + deterministic=True, + instances=[0], initial_budget=2, max_budget=5, eta=2) + + # we do not want to test instance as budget + self.assertFalse(intensifier.instance_as_budget) + + # Run until there are no more configurations to be proposed + # Skip running some configurations to emulate the fact that runs finish on different time + # We need this because there was a bug where not all instances had finished, yet + # the SH instance assumed all configurations finished + challengers = [self.config1, self.config2, self.config3, self.config4] + incumbent = None + pending_processing, incumbent = self._exhaust_stage_execution(intensifier, taf, + challengers, incumbent) + + # We have configurations pending, so iteration should NOT be done + self.assertFalse(intensifier.iteration_done) + + # Make sure we launched all configurations we were meant to: + # all_budgets=[2.5 5. ] n_configs_in_stage=[2.0, 1.0] + # We need 2 configurations in the run history + configurations = set([k.config_id for k, v in self.rh.data.items()]) + self.assertEqual(configurations, {1, 2}) + # There is only one instance always -- so we only have 2 configs for 1 instances each + config_inst_seed = set([k for k, v in self.rh.data.items()]) + self.assertEqual(len(config_inst_seed), 2) + + # Go to the last stage. Notice that iteration should not be done + # as we are in stage 1 out of 2 + for run_info in pending_processing: + result = eval_challenger(run_info, taf, self.stats, self.rh, + force_update=True) + incumbent, inc_value = intensifier.process_results( + run_info=run_info, + incumbent=incumbent, + run_history=self.rh, + time_bound=np.inf, + result=result, + log_traj=False, + ) + self.assertFalse(intensifier.iteration_done) + + # we transition to stage 1, where the budget is 5 + self.assertEqual(intensifier.stage, 1) + + pending_processing, incumbent = self._exhaust_stage_execution(intensifier, taf, + challengers, incumbent) + + # The next configuration per stage is just one (n_configs_in_stage=[2.0, 1.0]) + # We ran previously 2 configs and with this new, we should have 3 total + config_inst_seed = [k for k, v in self.rh.data.items()] + self.assertEqual(len(config_inst_seed), 3) + + # Because it is only 1 config, the iteration is completed + self.assertTrue(intensifier.iteration_done) + + # We make sure the proper budget got allocated on the whole run: + # all_budgets=[2.5 5. ] + # We ran 2 configs in small budget and 1 in full budget + self.assertEqual( + [k.budget for k in self.rh.data.keys()], + [2.5, 2.5, 5] + ) + if __name__ == "__main__": unittest.main() From faf0e5279804fd42be07a1c1412d7b38dbad8b68 Mon Sep 17 00:00:00 2001 From: Katharina Eggensperger Date: Wed, 28 Oct 2020 13:31:46 +0100 Subject: [PATCH 3/5] Fix initial hb budget (#702) * FIX numerical issues when comuting initial budget for SH * FIX flake * FIX ignoring intensifier arg for BOHB facade * FIX unittest * FIX mypy * ADD tests * FIX assert n_config_in_stage being int * FIX tests,flake8 --- smac/facade/smac_bohb_facade.py | 3 +- smac/intensification/hyperband.py | 6 ++ smac/intensification/successive_halving.py | 40 ++++++++++--- test/test_intensify/test_hyperband.py | 58 +++++++++++++++++++ .../test_intensify/test_successive_halving.py | 47 +++++++++++++++ 5 files changed, 144 insertions(+), 10 deletions(-) diff --git a/smac/facade/smac_bohb_facade.py b/smac/facade/smac_bohb_facade.py index 515ac1126..448ec03ae 100644 --- a/smac/facade/smac_bohb_facade.py +++ b/smac/facade/smac_bohb_facade.py @@ -42,7 +42,8 @@ def __init__(self, **kwargs: typing.Any): # Intensification parameters # select Hyperband as the intensifier ensure respective parameters are provided - kwargs['intensifier'] = Hyperband + if kwargs.get('intensifier') is None: + kwargs['intensifier'] = Hyperband # set Hyperband parameters if not given intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) diff --git a/smac/intensification/hyperband.py b/smac/intensification/hyperband.py index f1c6fdcba..ecff2f829 100644 --- a/smac/intensification/hyperband.py +++ b/smac/intensification/hyperband.py @@ -272,6 +272,10 @@ def _update_stage(self, run_history: RunHistory = None) -> None: # sample challengers for next iteration (based on HpBandster package) n_challengers = int(np.floor((self.s_max + 1) / (self.s + 1)) * self.eta ** self.s) + # Compute this for the next round + n_configs_in_stage = n_challengers * np.power(self.eta, -np.linspace(0, self.s, self.s + 1)) + n_configs_in_stage = np.array(np.round(n_configs_in_stage), dtype=int).tolist() + self.logger.info('Hyperband iteration-step: %d-%d with initial budget: %d' % ( self.hb_iters + 1, self.s_max - self.s + 1, sh_initial_budget)) @@ -287,6 +291,8 @@ def _update_stage(self, run_history: RunHistory = None) -> None: initial_budget=sh_initial_budget, max_budget=self.max_budget, eta=self.eta, + _all_budgets=self.all_budgets[(-self.s - 1):], + _n_configs_in_stage=n_configs_in_stage, num_initial_challengers=n_challengers, run_obj_time=self.run_obj_time, n_seeds=self.n_seeds, diff --git a/smac/intensification/successive_halving.py b/smac/intensification/successive_halving.py index d287a80bc..6980002bc 100644 --- a/smac/intensification/successive_halving.py +++ b/smac/intensification/successive_halving.py @@ -71,6 +71,10 @@ class _SuccessiveHalving(AbstractRacer): maximum budget allowed for 1 run of successive halving eta : float 'halving' factor after each iteration in a successive halving run. Defaults to 3 + _all_budgets: typing.Optional[typing.List[float]] = None + Used internally when HB uses SH as a subrouting + _n_configs_in_stage: typing.Optional[typing.List[int]] = None + Used internally when HB uses SH as a subrouting num_initial_challengers : typing.Optional[int] number of challengers to consider for the initial budget. If None, calculated internally run_obj_time : bool @@ -111,6 +115,8 @@ def __init__(self, initial_budget: typing.Optional[float] = None, max_budget: typing.Optional[float] = None, eta: float = 3, + _all_budgets: typing.Optional[typing.List[float]] = None, + _n_configs_in_stage: typing.Optional[typing.List[int]] = None, num_initial_challengers: typing.Optional[int] = None, run_obj_time: bool = True, n_seeds: typing.Optional[int] = None, @@ -175,7 +181,9 @@ def __init__(self, self.inst_seed_pairs = inst_seed_pairs # successive halving parameters - self._init_sh_params(initial_budget, max_budget, eta, num_initial_challengers) + self._init_sh_params(initial_budget=initial_budget, max_budget=max_budget, eta=eta, + num_initial_challengers=num_initial_challengers, + _all_budgets=_all_budgets, _n_configs_in_stage=_n_configs_in_stage) # adaptive capping if self.instance_as_budget and self.instance_order != 'shuffle' and self.run_obj_time: @@ -219,7 +227,10 @@ def _init_sh_params(self, initial_budget: typing.Optional[float], max_budget: typing.Optional[float], eta: float, - num_initial_challengers: typing.Optional[int]) -> None: + num_initial_challengers: typing.Optional[int] = None, + _all_budgets: typing.Optional[typing.List[float]] = None, + _n_configs_in_stage: typing.Optional[typing.List[int]] = None, + ) -> None: """ initialize Successive Halving parameters @@ -233,6 +244,10 @@ def _init_sh_params(self, 'halving' factor after each iteration in a successive halving run num_initial_challengers : typing.Optional[int] number of challengers to consider for the initial budget + _all_budgets: typing.Optional[typing.List[float]] = None + Used internally when HB uses SH as a subrouting + _n_configs_in_stage: typing.Optional[typing.List[int]] = None + Used internally when HB uses SH as a subrouting """ if eta <= 1: @@ -280,14 +295,21 @@ def _init_sh_params(self, # max. no. of SH iterations possible given the budgets max_sh_iter = int(np.floor(np.log(self.max_budget / self.initial_budget) / np.log(self.eta))) # initial number of challengers to sample - if not num_initial_challengers: + if num_initial_challengers is None: num_initial_challengers = int(self.eta ** max_sh_iter) - # budgets to consider in each stage - self.all_budgets = self.max_budget * np.power(self.eta, -np.linspace(max_sh_iter, 0, max_sh_iter + 1)) - # number of challengers to consider in each stage - self.n_configs_in_stage = num_initial_challengers * np.power(self.eta, - -np.linspace(0, max_sh_iter, max_sh_iter + 1)) - self.n_configs_in_stage = self.n_configs_in_stage.tolist() + + if _all_budgets is not None and _n_configs_in_stage is not None: + # Assert we use the given numbers to avoid rounding issues, see #701 + self.all_budgets = _all_budgets + self.n_configs_in_stage = _n_configs_in_stage + else: + # budgets to consider in each stage + self.all_budgets = self.max_budget * np.power(self.eta, -np.linspace(max_sh_iter, 0, + max_sh_iter + 1)) + # number of challengers to consider in each stage + n_configs_in_stage = num_initial_challengers * \ + np.power(self.eta, -np.linspace(0, max_sh_iter, max_sh_iter + 1)) + self.n_configs_in_stage = np.array(np.round(n_configs_in_stage), dtype=int).tolist() def process_results(self, run_info: RunInfo, diff --git a/test/test_intensify/test_hyperband.py b/test/test_intensify/test_hyperband.py index 0a1e69bfb..2b871b7ef 100644 --- a/test/test_intensify/test_hyperband.py +++ b/test/test_intensify/test_hyperband.py @@ -550,5 +550,63 @@ def target(x): self.assertEqual(self.stats.inc_changed, 1) +class Test__Hyperband(unittest.TestCase): + + def test_budget_initialization(self): + """ + Check computing budgets (only for non-instance cases) + """ + intensifier = _Hyperband( + stats=None, traj_logger=None, + rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, + instances=None, initial_budget=1, max_budget=81, eta=3 + ) + self.assertListEqual([1, 3, 9, 27, 81], intensifier.all_budgets.tolist()) + self.assertListEqual([81, 27, 9, 3, 1], intensifier.n_configs_in_stage) + + to_check = [ + # minb, maxb, eta, n_configs_in_stage, all_budgets + [1, 81, 3, [81, 27, 9, 3, 1], [1, 3, 9, 27, 81]], + [1, 600, 3, [243, 81, 27, 9, 3, 1], + [2.469135, 7.407407, 22.222222, 66.666666, 200, 600]], + [1, 100, 10, [100, 10, 1], [1, 10, 100]], + [0.001, 1, 3, [729, 243, 81, 27, 9, 3, 1], + [0.001371, 0.004115, 0.012345, 0.037037, 0.111111, 0.333333, 1.0]], + [1, 1000, 3, [729, 243, 81, 27, 9, 3, 1], + [1.371742, 4.115226, 12.345679, 37.037037, 111.111111, 333.333333, 1000.0]], + [0.001, 100, 10, [100000, 10000, 1000, 100, 10, 1], + [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]], + ] + + for minb, maxb, eta, n_configs_in_stage, all_budgets in to_check: + intensifier = _Hyperband( + stats=None, traj_logger=None, + rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, + instances=None, initial_budget=minb, max_budget=maxb, eta=eta + ) + intensifier._init_sh_params(initial_budget=minb, + max_budget=maxb, + eta=eta, + _all_budgets=None, + _n_configs_in_stage=None, + ) + for i in range(len(all_budgets) + 10): + intensifier._update_stage() + comp_budgets = intensifier.sh_intensifier.all_budgets.tolist() + comp_configs = intensifier.sh_intensifier.n_configs_in_stage + + self.assertIsInstance(comp_configs, list) + for c in comp_configs: + self.assertIsInstance(c, int) + + # all_budgets for SH is always a subset of all_budgets of HB + np.testing.assert_array_almost_equal(all_budgets[i % len(all_budgets):], + comp_budgets, decimal=5) + + # The content of these lists might differ + self.assertEqual(len(n_configs_in_stage[i % len(n_configs_in_stage):]), + len(comp_configs)) + + if __name__ == "__main__": unittest.main() diff --git a/test/test_intensify/test_successive_halving.py b/test/test_intensify/test_successive_halving.py index 38b804a07..eb15f889f 100644 --- a/test/test_intensify/test_successive_halving.py +++ b/test/test_intensify/test_successive_halving.py @@ -1707,5 +1707,52 @@ def target(x): ) +class Test__SuccessiveHalving(unittest.TestCase): + + def test_budget_initialization(self): + """ + Check computing budgets (only for non-instance cases) + """ + intensifier = _SuccessiveHalving( + stats=None, traj_logger=None, + rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, + instances=None, initial_budget=1, max_budget=81, eta=3 + ) + self.assertListEqual([1, 3, 9, 27, 81], intensifier.all_budgets.tolist()) + self.assertListEqual([81, 27, 9, 3, 1], intensifier.n_configs_in_stage) + + to_check = [ + # minb, maxb, eta, n_configs_in_stage, all_budgets + [1, 81, 3, [81, 27, 9, 3, 1], [1, 3, 9, 27, 81]], + [1, 600, 3, [243, 81, 27, 9, 3, 1], + [2.469135, 7.407407, 22.222222, 66.666666, 200, 600]], + [1, 100, 10, [100, 10, 1], [1, 10, 100]], + [0.001, 1, 3, [729, 243, 81, 27, 9, 3, 1], + [0.001371, 0.004115, 0.012345, 0.037037, 0.111111, 0.333333, 1.0]], + [1, 1000, 3, [729, 243, 81, 27, 9, 3, 1], + [1.371742, 4.115226, 12.345679, 37.037037, 111.111111, 333.333333, 1000.0]], + [0.001, 100, 10, [100000, 10000, 1000, 100, 10, 1], + [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]], + ] + + for minb, maxb, eta, n_configs_in_stage, all_budgets in to_check: + intensifier._init_sh_params(initial_budget=minb, + max_budget=maxb, + eta=eta, + _all_budgets=None, + _n_configs_in_stage=None, + ) + comp_budgets = intensifier.all_budgets.tolist() + comp_configs = intensifier.n_configs_in_stage + + self.assertEqual(len(all_budgets), len(comp_budgets)) + self.assertEqual(comp_budgets[-1], maxb) + np.testing.assert_array_almost_equal(all_budgets, comp_budgets, decimal=5) + + self.assertEqual(comp_configs[-1], 1) + self.assertEqual(len(n_configs_in_stage), len(comp_configs)) + np.testing.assert_array_almost_equal(n_configs_in_stage, comp_configs, decimal=5) + + if __name__ == "__main__": unittest.main() From 8f9dae84e850c5bcbb0fac17c1ed42126a58107c Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 28 Oct 2020 16:25:36 +0100 Subject: [PATCH 4/5] Add callback mechanism (#703) * Add callback mechanism * allow registering callbacks with the SMAC facade * implements a callback for _incorporate_run_results * add the actual file * improve callback call syntax * respond to feedback --- doc/api.rst | 1 + smac/callbacks.py | 38 ++++++++++++++++++++++++++++ smac/facade/smac_ac_facade.py | 25 ++++++++++++++++++ smac/optimizer/smbo.py | 13 ++++++++++ test/test_facade/test_smac_facade.py | 20 ++++++++++++++- test/test_smbo/test_smbo.py | 37 ++++++++++++++++++++++++++- 6 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 smac/callbacks.py diff --git a/doc/api.rst b/doc/api.rst index 1a2f8e1a0..eb41124f3 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -8,6 +8,7 @@ API Documentation .. toctree:: + apidoc/smac.callbacks apidoc/smac.configspace apidoc/smac.epm apidoc/smac.facade diff --git a/smac/callbacks.py b/smac/callbacks.py new file mode 100644 index 000000000..b93bccf06 --- /dev/null +++ b/smac/callbacks.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from smac.optimizer.smbo import SMBO +from smac.runhistory.runhistory import RunInfo, RunValue + +"""Callbacks for SMAC. + +Callbacks allow customizing the behavior of SMAC to ones needs. Currently, the list of implemented callbacks is +very limited, but they can easily be added. + +How to add a new callback +========================= + +1. Implement a callback class in this module. There are no restrictions on how such a callback must look like, + but it is recommended to implement the main logic inside the `__call__` function, such as for example in + ``IncorporateRunResultCallback``. +2. Add your callback to ``smac.smbo.optimizer.SMBO._callbacks``, using the name of your callback as the key, + and an empty list as the value. +3. Add your callback to ``smac.smbo.optimizer.SMBO._callback_to_key``, using the callback class as the key, + and the name as value (the name used in 2.). +4. Implement calling all registered callbacks at the correct place. This is as simple as + ``for callback in self._callbacks['your_callback']: callback(*args, **kwargs)``, where you obviously need to + change the callback name and signature. +""" + + +class IncorporateRunResultCallback: + + """Callback to react on a new run result. Called after the finished run is added to the runhistory.""" + + def __call__( + self, smbo: 'SMBO', + run_info: RunInfo, + result: RunValue, + time_left: float, + ) -> None: + pass diff --git a/smac/facade/smac_ac_facade.py b/smac/facade/smac_ac_facade.py index 9655b9d33..f31e36f74 100644 --- a/smac/facade/smac_ac_facade.py +++ b/smac/facade/smac_ac_facade.py @@ -692,3 +692,28 @@ def get_trajectory(self) -> List[TrajEntry]: raise ValueError('SMAC was not fitted yet. Call optimize() prior ' 'to accessing the runhistory.') return self.trajectory + + def register_callback(self, callback: Callable) -> None: + """Register a callback function. + + Callbacks must implement a class in ``smac.callbacks`` and be instantiated objects. + They will automatically be registered within SMAC based on which callback class from + ``smac.callbacks`` they implement. + + Parameters + ---------- + callback - Callable + + Returns + ------- + None + """ + types_to_check = callback.__class__.__mro__ + key = None + for type_to_check in types_to_check: + key = self.solver._callback_to_key.get(type_to_check) + if key is not None: + break + if key is None: + raise ValueError('Cannot register callback of type %s' % type(callback)) + self.solver._callbacks[key].append(callback) diff --git a/smac/optimizer/smbo.py b/smac/optimizer/smbo.py index 3d3c5ba8d..aea87c215 100644 --- a/smac/optimizer/smbo.py +++ b/smac/optimizer/smbo.py @@ -4,6 +4,7 @@ import time import typing +from smac.callbacks import IncorporateRunResultCallback from smac.configspace import Configuration from smac.epm.rf_with_instances import RandomForestWithInstances from smac.initial_design.initial_design import InitialDesign @@ -147,6 +148,15 @@ def __init__(self, # Internal variable - if this is set to True it will gracefully stop SMAC self._stop = False + # Callbacks. All known callbacks have a key. If something does not have a key here, there is + # no callback available. + self._callbacks = { + '_incorporate_run_results': list() + } # type: typing.Dict[str, typing.List[typing.Callable]] + self._callback_to_key = { + IncorporateRunResultCallback: '_incorporate_run_results', + } # type: typing.Dict[typing.Type, str] + def start(self) -> None: """Starts the Bayesian Optimization loop. Detects whether the optimization is restored from a previous state. @@ -479,4 +489,7 @@ def _incorporate_run_results(self, run_info: RunInfo, result: RunValue, result=result, ) + for callback in self._callbacks['_incorporate_run_results']: + callback(smbo=self, run_info=run_info, result=result, time_left=time_left) + return diff --git a/test/test_facade/test_smac_facade.py b/test/test_facade/test_smac_facade.py index 46dba53f9..e6346a318 100644 --- a/test/test_facade/test_smac_facade.py +++ b/test/test_facade/test_smac_facade.py @@ -8,8 +8,8 @@ from ConfigSpace.hyperparameters import UniformFloatHyperparameter from ConfigSpace.util import get_one_exchange_neighbourhood +from smac.callbacks import IncorporateRunResultCallback from smac.configspace import ConfigurationSpace - from smac.epm.random_epm import RandomEPM from smac.epm.rf_with_instances import RandomForestWithInstances from smac.epm.uncorrelated_mo_rf_with_instances import UncorrelatedMultiObjectiveRandomForestWithInstances @@ -433,3 +433,21 @@ def test_no_output(self): scen1 = Scenario(test_scenario_dict) smac = SMAC4AC(scenario=scen1, run_id=1) self.assertFalse(os.path.isdir(smac.output_dir)) + + def test_register_callback(self): + smac = SMAC4AC(scenario=self.scenario, run_id=1) + + with self.assertRaisesRegex(ValueError, "Cannot register callback of type "): + smac.register_callback(lambda: 1) + + with self.assertRaisesRegex(ValueError, "Cannot register callback of type "): + smac.register_callback(IncorporateRunResultCallback) + + smac.register_callback(IncorporateRunResultCallback()) + self.assertEqual(len(smac.solver._callbacks['_incorporate_run_results']), 1) + + class SubClass(IncorporateRunResultCallback): + pass + + smac.register_callback(SubClass()) + self.assertEqual(len(smac.solver._callbacks['_incorporate_run_results']), 2) diff --git a/test/test_smbo/test_smbo.py b/test/test_smbo/test_smbo.py index a71aa134e..0ea11d172 100644 --- a/test/test_smbo/test_smbo.py +++ b/test/test_smbo/test_smbo.py @@ -8,14 +8,16 @@ from ConfigSpace.hyperparameters import UniformFloatHyperparameter +from smac.callbacks import IncorporateRunResultCallback from smac.configspace import ConfigurationSpace from smac.epm.rf_with_instances import RandomForestWithInstances +import smac.facade.smac_ac_facade from smac.facade.smac_ac_facade import SMAC4AC from smac.facade.smac_hpo_facade import SMAC4HPO from smac.intensification.abstract_racer import RunInfoIntent from smac.optimizer.acquisition import EI, LogEI from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost, RunHistory2EPM4LogCost -from smac.runhistory.runhistory import RunInfo +from smac.runhistory.runhistory import RunInfo, RunValue from smac.scenario.scenario import Scenario from smac.tae import FirstRunCrashedException, StatusType from smac.tae.execute_func import ExecuteTAFuncArray @@ -374,6 +376,39 @@ def mock_get_next_run(**kwargs): X, Y, X_config = smbo.epm_chooser._collect_data_to_train_model() self.assertEqual(X.shape[0], len(all_configs)) + @unittest.mock.patch.object(smac.facade.smac_ac_facade.Intensifier, 'process_results') + def test_incorporate_run_results_callback(self, process_results_mock): + + process_results_mock.return_value = None, None + + class TestCallback(IncorporateRunResultCallback): + def __init__(self): + self.num_call = 0 + + def __call__(self, smbo, run_info, result, time_left) -> None: + self.num_call += 1 + self.config = run_info.config + + callback = TestCallback() + + self.scenario.output_dir = None + smac = SMAC4AC(self.scenario) + smac.register_callback(callback) + + self.output_dirs.append(smac.output_dir) + smbo = smac.solver + + config = self.scenario.cs.sample_configuration() + + run_info = RunInfo(config=config, instance=None, instance_specific=None, seed=1, + cutoff=None, capped=False, budget=0.0, source_id=0) + result = RunValue(1.2345, 2.3456, 'status', 'starttime', 'endtime', 'additional_info') + time_left = 10 + + smbo._incorporate_run_results(run_info=run_info, result=result, time_left=time_left) + self.assertEqual(callback.num_call, 1) + self.assertEqual(callback.config, config) + if __name__ == "__main__": unittest.main() From 3c463dc3c109e37effaacd1d622b5069d3724f5a Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 29 Oct 2020 08:22:39 +0100 Subject: [PATCH 5/5] create new minor release (#704) --- changelog.md | 10 ++++++++++ smac/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index c20c9232d..b28cfc73d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,13 @@ +# 0.13.1 + +## Minor Changes +* Improve error message for first run crashed (#694). +* Experimental: add callback mechanism (#703). + +## Bug fixes +* Fix a bug which could make successive halving fail if run in parallel (#695). +* Fix a bug which could cause hyperband to ignore the lowest budget (#701). + # 0.13.0 ## Major Changes diff --git a/smac/__init__.py b/smac/__init__.py index 9599f031e..248c8de1b 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -5,7 +5,7 @@ import lazy_import from smac.utils import dependencies -__version__ = '0.13.0' +__version__ = '0.13.1' __author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, ' \ 'André Biedenkapp, Francisco Rivera, Ashwin Raaghav, Aaron Klein, Stefan Falkner ' \ 'and Frank Hutter'