diff --git a/.travis.yml b/.travis.yml index 021e9c9f0..85b3f22f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,17 +55,10 @@ before_cache: - rm -f $HOME/.cache/pip/log/debug.log before_install: - - wget $MINICONDA_URL -O miniconda.sh - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - if [[ `which conda` ]]; then echo 'Conda installation successful'; else exit 1; fi - - conda create -n testenv --yes python=$PYTHON_VERSION pip wheel pytest gxx_linux-64 gcc_linux-64 swig - - source activate testenv + - source ci_scripts/install_env.sh install: - - pip install pep8 codecov mypy flake8 pytest-cov - - cat requirements.txt | xargs -n 1 -L 1 pip install - - pip install .[all] + - source ci_scripts/install.sh script: - ci_scripts/$TESTSUITE diff --git a/changelog.md b/changelog.md index 55afcc25c..c588ebba0 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,13 @@ -# 0.12.1 +# 0.12.2 -## Major Changes +## Bug Fixes + +* Fixes the docstring of SMAC's default acquisition function optimizer (#653) +* Correctly attributes the configurations' origin if using the `FixedSet` acquisition function optimizer (#653) +* Fixes an infinite loop which could occur if using only a single configuration per iteration (#654) +* Fixes a bug in the kernel construction of the `BOFacade` (#655) + +# 0.12.1 ## Minor Changes diff --git a/ci_scripts/install.sh b/ci_scripts/install.sh new file mode 100644 index 000000000..68b6eb4bb --- /dev/null +++ b/ci_scripts/install.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +pip install pep8 codecov mypy flake8 pytest-cov +cat requirements.txt | xargs -n 1 -L 1 pip install +pip install .[all] \ No newline at end of file diff --git a/ci_scripts/install_env.sh b/ci_scripts/install_env.sh new file mode 100644 index 000000000..1e37d1ae6 --- /dev/null +++ b/ci_scripts/install_env.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +python --version + +wget $MINICONDA_URL -O miniconda.sh +bash miniconda.sh -b -p $HOME/miniconda +export PATH="$HOME/miniconda/bin:$PATH" +if [[ `which conda` ]]; then echo 'Conda installation successful'; else exit 1; fi +conda create -n testenv --yes python=$PYTHON_VERSION pip wheel pytest gxx_linux-64 gcc_linux-64 swig +source activate testenv \ No newline at end of file diff --git a/smac/__init__.py b/smac/__init__.py index d77626440..e06ea171e 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -5,7 +5,7 @@ import lazy_import from smac.utils import dependencies -__version__ = '0.12.1' +__version__ = '0.12.2' __author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein,'\ 'Stefan Falkner and Frank Hutter' diff --git a/smac/facade/smac_ac_facade.py b/smac/facade/smac_ac_facade.py index 105ee0c57..ac4c831c9 100644 --- a/smac/facade/smac_ac_facade.py +++ b/smac/facade/smac_ac_facade.py @@ -34,7 +34,7 @@ # optimizer from smac.optimizer.smbo import SMBO from smac.optimizer.acquisition import EI, LogEI, AbstractAcquisitionFunction, IntegratedAcquisitionFunction -from smac.optimizer.ei_optimization import InterleavedLocalAndRandomSearch, \ +from smac.optimizer.ei_optimization import LocalAndSortedRandomSearch, \ AcquisitionFunctionMaximizer from smac.optimizer.random_configuration_chooser import RandomConfigurationChooser, ChooserProb # epm @@ -135,7 +135,7 @@ def __init__(self, hyperparameters (i.e. GaussianProcessMCMC). acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`. - Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set. + Will use :class:`smac.optimizer.ei_optimization.LocalAndSortedRandomSearch` if not set. acquisition_function_optimizer_kwargs: Optional[Dict] Arguments passed to constructor of '~acquisition_function_optimizer' model : AbstractEPM @@ -332,7 +332,7 @@ def __init__(self, if key not in acq_func_opt_kwargs: acq_func_opt_kwargs[key] = value acquisition_function_optimizer_instance = ( - InterleavedLocalAndRandomSearch(**acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821 + LocalAndSortedRandomSearch(**acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821 ) # type: AcquisitionFunctionMaximizer elif inspect.isclass(acquisition_function_optimizer): acquisition_function_optimizer_instance = acquisition_function_optimizer(**acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821 diff --git a/smac/facade/smac_bo_facade.py b/smac/facade/smac_bo_facade.py index d2bd22c9b..7679c0fa8 100644 --- a/smac/facade/smac_bo_facade.py +++ b/smac/facade/smac_bo_facade.py @@ -86,8 +86,8 @@ def __init__(self, model_type: str = 'gp_mcmc', **kwargs: typing.Any): prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) - cont_dims = np.nonzero(types == 0)[0] - cat_dims = np.nonzero(types != 0)[0] + cont_dims = np.where(np.array(types) == 0)[0] + cat_dims = np.where(np.array(types) != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( @@ -104,6 +104,8 @@ def __init__(self, model_type: str = 'gp_mcmc', **kwargs: typing.Any): operate_on=cat_dims, ) + assert len(cont_dims + len(cat_dims)) == len(scenario.cs.get_hyperparameters()) + noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), diff --git a/smac/intensification/intensification.py b/smac/intensification/intensification.py index 9cba0273a..497685d9a 100644 --- a/smac/intensification/intensification.py +++ b/smac/intensification/intensification.py @@ -126,6 +126,7 @@ def __init__(self, tae_runner: ExecuteTARun, # challenger related variables self._chall_indx = 0 + self.num_chall_run = 0 self.current_challenger = None self.continue_challenger = False self.configs_to_run = iter([]) # type: _config_to_run_type @@ -225,7 +226,13 @@ def eval_challenger(self, self.elapsed_time += time.time() - start_time # check if 1 intensification run is complete - line 18 - if self.stage == IntensifierStage.RUN_INCUMBENT and self._chall_indx >= self.min_chall: + # this is different to regular SMAC as it requires at least successful challenger run, + # which is necessary to work on a fixed grid of configurations. + if ( + self.stage == IntensifierStage.RUN_INCUMBENT + and self._chall_indx >= self.min_chall + and self.num_chall_run > 0 + ): if self.num_run > self.run_limit: self.logger.info("Maximum #runs for intensification reached") self._next_iteration() @@ -294,6 +301,7 @@ def _add_inc_run(self, next_seed = self.rs.randint(low=0, high=MAXINT, size=1)[0] if available_insts: + self.num_run += 1 # Line 5 (here for easier code) next_instance = self.rs.choice(available_insts) # Line 7 @@ -305,7 +313,6 @@ def _add_inc_run(self, cutoff=self.cutoff, instance_specific=self.instance_specifics.get(next_instance, "0")) self._ta_time += dur - self.num_run += 1 else: self.logger.debug("No further instance-seed pairs for " "incumbent available.") @@ -392,6 +399,9 @@ def _race_challenger(self, self.logger.debug("Add run of challenger") try: + self.num_run += 1 + self.num_chall_run += 1 + status, cost, dur, res = self.tae_runner.start( config=challenger, instance=instance, @@ -401,7 +411,6 @@ def _race_challenger(self, # Cutoff might be None if self.cutoff is None, but then the first if statement prevents # evaluation of the second if statement capped=(self.cutoff is not None) and (cutoff < self.cutoff)) # type: ignore[operator] # noqa F821 - self.num_run += 1 self._ta_time += dur except CappedRunException: @@ -625,6 +634,7 @@ def _next_iteration(self) -> None: # reset for a new iteration self.num_run = 0 + self.num_chall_run = 0 self._chall_indx = 0 self.elapsed_time = 0 self._ta_time = 0.0 diff --git a/smac/optimizer/ei_optimization.py b/smac/optimizer/ei_optimization.py index efa4d49b8..5967c9aaf 100644 --- a/smac/optimizer/ei_optimization.py +++ b/smac/optimizer/ei_optimization.py @@ -574,13 +574,13 @@ def _maximize( return [(0, rand_configs[i]) for i in range(len(rand_configs))] -class InterleavedLocalAndRandomSearch(AcquisitionFunctionMaximizer): +class LocalAndSortedRandomSearch(AcquisitionFunctionMaximizer): """Implements SMAC's default acquisition function optimization. This optimizer performs local search from the previous best points according, to the acquisition function, uses the acquisition function to - sort randomly sampled configurations and interleaves unsorted, randomly - sampled configurations in between. + sort randomly sampled configurations. Random configurations are + interleaved by the main SMAC code. Parameters ---------- @@ -751,4 +751,6 @@ def _maximize( num_points: int, ) -> List[Tuple[float, Configuration]]: configurations = copy.deepcopy(self.configurations) + for config in configurations: + config.origin = 'Fixed Set' return self._sort_configs_by_acq_value(configurations) diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index e14b7f83f..1ac1eaaf9 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -168,7 +168,7 @@ def add( status: StatusType, instance_id: typing.Optional[str] = None, seed: typing.Optional[int] = None, - budget: float = 0, + budget: float = 0.0, additional_info: typing.Optional[typing.Dict] = None, origin: DataOrigin = DataOrigin.INTERNAL, ) -> None: diff --git a/smac/utils/io/traj_logging.py b/smac/utils/io/traj_logging.py index d25f986f3..5945694ff 100644 --- a/smac/utils/io/traj_logging.py +++ b/smac/utils/io/traj_logging.py @@ -323,7 +323,7 @@ def _convert_dict_to_config(config_list: typing.List[str], cs: ConfigurationSpac continue # Second, check if it's in the choices / the correct type. - legal = {l for l in interpretations if hp.is_legal(l)} + legal = {interpretation for interpretation in interpretations if hp.is_legal(interpretation)} # Third, issue warnings if the interpretation is ambigious if len(legal) != 1: diff --git a/test/test_intensify/test_intensify.py b/test/test_intensify/test_intensify.py index 55840ffb7..f1864dca1 100644 --- a/test/test_intensify/test_intensify.py +++ b/test/test_intensify/test_intensify.py @@ -78,6 +78,8 @@ def target(x): run_history=self.rh) self.assertEqual(inc, self.config2) + self.assertEqual(intensifier.num_run, 1) + self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_2(self): """ @@ -108,8 +110,9 @@ def target(x): incumbent=self.config1, run_history=self.rh,) - # self.assertTrue(False) self.assertEqual(inc, self.config1) + self.assertEqual(intensifier.num_run, 1) + self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_3(self): """ @@ -176,6 +179,9 @@ def target(config: Configuration, seed: int, instance: str): self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) + self.assertEqual(intensifier.num_run, 3) + self.assertEqual(intensifier.num_chall_run, 3) + def test_race_challenger_large(self): """ test _race_challenger using solution_quality @@ -221,6 +227,9 @@ def target(x): runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) + self.assertEqual(intensifier.num_run, 10) + self.assertEqual(intensifier.num_chall_run, 10) + def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs @@ -269,6 +278,9 @@ def target(x): seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) + self.assertEqual(intensifier.num_run, 10) + self.assertEqual(intensifier.num_chall_run, 10) + def test_add_inc_run_det(self): """ test _add_inc_run() @@ -295,6 +307,11 @@ def target(x): intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) + # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run + # as it is the first evaluation of this intensifier + self.assertEqual(intensifier.num_run, 0) + self.assertEqual(intensifier.num_chall_run, 0) + def test_add_inc_run_nondet(self): """ test _add_inc_run() @@ -324,6 +341,9 @@ def target(x): intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) + self.assertEqual(intensifier.num_run, 2) + self.assertEqual(intensifier.num_chall_run, 0) + def test_get_next_challenger(self): """ test get_next_challenger() @@ -604,3 +624,76 @@ def target(x): self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(len(self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2) + + def test_no_new_intensification_wo_challenger_run(self): + """ + This test ensures that no new iteration is started if no challenger run was conducted + """ + def target(x): + return 2 * x['a'] + x['b'] + + taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") + taf.runhistory = self.rh + + intensifier = Intensifier( + tae_runner=taf, stats=self.stats, + traj_logger=TrajLogger(output_dir=None, stats=self.stats), + rng=np.random.RandomState(12345), + instances=[1], run_obj_time=False, + deterministic=True, always_race_against=None, run_limit=1, + min_chall=1, + ) + + self.assertEqual(intensifier.n_iters, 0) + self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) + + config, _ = intensifier.get_next_challenger(challengers=[self.config3], + chooser=None) + self.assertEqual(config, self.config3) + self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) + inc, _ = intensifier.eval_challenger(challenger=config, incumbent=None, run_history=self.rh, ) + self.assertEqual(inc, self.config3) + self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) + self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! + + # regular intensification begins - run incumbent + config, _ = intensifier.get_next_challenger(challengers=None, # since incumbent is run, no configs required + chooser=None) + self.assertEqual(config, inc) + self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) + inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, ) + self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) + self.assertEqual(intensifier.n_iters, 1) + + # Check that we don't walk into the next iteration if the challenger is passed again + config, _ = intensifier.get_next_challenger(challengers=[self.config3], + chooser=None) + inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, ) + self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) + self.assertEqual(intensifier.n_iters, 1) + + intensifier._next_iteration() + + # Add a configuration, then try to execute it afterwards + self.assertEqual(intensifier.n_iters, 2) + self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, + instance_id=1, seed=0, additional_info=None) + intensifier.stage = IntensifierStage.RUN_CHALLENGER + config, _ = intensifier.get_next_challenger(challengers=[self.config1], + chooser=None) + inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, ) + self.assertEqual(intensifier.n_iters, 2) + self.assertEqual(intensifier.num_chall_run, 0) + + # This returns the config evaluating the incumbent again + config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) + inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, ) + # This doesn't return a config because the array of configs is exhausted + config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) + self.assertIsNone(config) + # This finally gives a runable configuration + config, _ = intensifier.get_next_challenger(challengers=[self.config2], + chooser=None) + inc, _ = intensifier.eval_challenger(challenger=config, incumbent=inc, run_history=self.rh, ) + self.assertEqual(intensifier.n_iters, 3) + self.assertEqual(intensifier.num_chall_run, 1) diff --git a/test/test_utils/io/test_traj_logging.py b/test/test_utils/io/test_traj_logging.py index 160346cdd..aacfe6de9 100644 --- a/test/test_utils/io/test_traj_logging.py +++ b/test/test_utils/io/test_traj_logging.py @@ -96,9 +96,9 @@ def test_add_entries(self, mock_stats): with open(os.path.join(tmpdir, 'traj_old.csv')) as to: data = to.read().split('\n') with open(os.path.join(tmpdir, 'traj_aclib2.json')) as js_aclib: - json_dicts_aclib2 = [json.loads(l) for l in js_aclib.read().splitlines()] + json_dicts_aclib2 = [json.loads(line) for line in js_aclib.read().splitlines()] with open(os.path.join(tmpdir, 'traj.json')) as js: - json_dicts_alljson = [json.loads(l) for l in js.read().splitlines()] + json_dicts_alljson = [json.loads(line) for line in js.read().splitlines()] # Check old format header = data[0].split(',')